o
    Zh;                     @   s`  d Z ddlmZmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZ eZg ZzddlmZ W n ey>   ed Y nw zddlmZ W n eyU   ed Y nw zddlmZmZmZmZmZmZmZm Z m!Z!m"Z" W n ey~   ed	 Y n
w dd
l#Z#e#$e  dd Z%dd Z&	dddZ'	dddZ(	
ddddddZ)G dd deZ*d
S )zO
FileSystem abstraction to interact with various local and remote filesystems.
    )_is_path_like_stringify_path)FileSelectorFileTypeFileInfo
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystem_copy_files_copy_files_selector)HadoopFileSystemr   )GcsFileSystemr   )
AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyS3FileSystem
S3LogLevelS3RetryStrategyensure_s3_initializedfinalize_s3ensure_s3_finalizedinitialize_s3resolve_s3_regionr   Nc                 C   s$   | t v rtd| td| )Nz<The pyarrow installation is not built with support for '{0}'z*module 'pyarrow.fs' has no attribute '{0}')_not_importedImportErrorformatAttributeError)name r    A/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/fs.py__getattr__E   s   r"   c                 C   s^   t | \}}||}|r-||gd }|jtjkr(td|jj	|j
| t||}|S )Nr   zThe path component of the filesystem URI must point to a directory but it has a type: `{}`. The path component is `{}` and the given filesystem URI is `{}`)r   from_urinormalize_pathget_file_infotyper   	Directory
ValueErrorr   r   pathr	   )uri
filesystemprefixZprefix_infor    r    r!   _filesystem_from_strQ   s   

r-   Fc                 C   s   t | tr| S t | tr|rtdt| S zdd l}W n	 ty%   Y nw t | |jr>t| j	dkr8t
|dS tt| S dd lm} t | |j
rOt
|dS |rYt | |jrY| S tdt| )NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   r   use_mmapzlUnrecognized filesystem: {}. `filesystem` argument must be a FileSystem instance or a valid file system URI')
isinstancer   strr(   r-   fsspecr   ZAbstractFileSystemr&   __name__r   r   FSSpecHandlerZpyarrow.filesystemr+   	TypeErrorr   )r+   r/   allow_legacy_filesystemr2   Zlegacyfsr    r    r!   _ensure_filesysteme   s8   



r7   c              
   C   s@  t | s|durtd|| fS |dur9t|||d}t|tr%t| } n	t| ts.td|s5|| } || fS t| } t|d}z|	| }W n tyV   d}d}Y nw |j
tjk}|szt| \}} W || fS  ty } zdt|vrdt|vr W Y d}~|| fS W Y d}~|| fS d}~ww || } || fS )	z`
    Return filesystem/path from path which could be an URI or a plain
    filesystem path.
    Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.)r/   r6   zPExpected string path; path-like objects are only allowed with a local filesystemr.   Fzempty schemezCannot parse URI)r   r(   r7   r0   r   r   r1   r5   r$   r%   r&   r   NotFoundr   r#   )r)   r+   r6   Z
memory_map	file_infoZexists_locallyer    r    r!   _resolve_filesystem_and_path   sZ   







r;   i   T)
chunk_sizeuse_threadsc                C   sj   t | |\}}t ||\}}	||}
|
jtjkr*t|dd}t||||	|| dS t||||	|| dS )a  
    Copy files between FileSystems.

    This functions allows you to recursively copy directories of files from
    one file system to another, such as from S3 to your local machine.

    Parameters
    ----------
    source : string
        Source file path or URI to a single file or directory.
        If a directory, files will be copied recursively from this path.
    destination : string
        Destination file path or URI. If `source` is a file, `destination`
        is also interpreted as the destination file (not directory).
        Directories will be created as necessary.
    source_filesystem : FileSystem, optional
        Source filesystem, needs to be specified if `source` is not a URI,
        otherwise inferred.
    destination_filesystem : FileSystem, optional
        Destination filesystem, needs to be specified if `destination` is not
        a URI, otherwise inferred.
    chunk_size : int, default 1MB
        The maximum size of block to read before flushing to the
        destination file. A larger chunk_size will use more memory while
        copying but may help accommodate high latency FileSystems.
    use_threads : bool, default True
        Whether to use multiple threads to accelerate copying.

    Examples
    --------
    Inspect an S3 bucket's files:

    >>> s3, path = fs.FileSystem.from_uri(
    ...            "s3://registry.opendata.aws/roda/ndjson/")
    >>> selector = fs.FileSelector(path)
    >>> s3.get_file_info(selector)
    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

    Copy one file from S3 bucket to a local directory:

    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path))

    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
    ...                                    '/index_copy.ndjson')
    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

    Copy file using a FileSystem object:

    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               "file:///{}/index_copy.ndjson".format(local_path),
    ...               source_filesystem=fs.S3FileSystem())
    T	recursiveN)r;   r%   r&   r   r'   r   r   r   )sourceZdestinationZsource_filesystemZdestination_filesystemr<   r=   Z	source_fssource_pathZdestination_fsZdestination_pathr9   Z
source_selr    r    r!   
copy_files   s"   8
rB   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*S )+r4   a  
    Handler for fsspec-based Python filesystems.

    https://filesystem-spec.readthedocs.io/en/latest/index.html

    Parameters
    ----------
    fs : FSSpec-compliant filesystem instance

    Examples
    --------
    >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
    c                 C   s
   || _ d S N)fs)selfrD   r    r    r!   __init__(  s   
zFSSpecHandler.__init__c                 C   s   t |tr| j|jkS tS rC   r0   r4   rD   NotImplementedrE   otherr    r    r!   __eq__+     
zFSSpecHandler.__eq__c                 C   s   t |tr| j|jkS tS rC   rG   rI   r    r    r!   __ne__0  rL   zFSSpecHandler.__ne__c                 C   s$   | j j}t|tr|d }d|S )Nr   z
fsspec+{0})rD   protocolr0   listr   )rE   rN   r    r    r!   get_type_name5  s   

zFSSpecHandler.get_type_namec                 C   s   |S rC   r    rE   r)   r    r    r!   r$   ;  s   zFSSpecHandler.normalize_pathc                 C   sR   |d }|d dkrt j}n|d dkrt j}d }nt j}t| |||dd dS )Nsizer&   file	directorymtime)rR   rU   )r   Filer'   Unknownr   get)r)   inforR   Zftyper    r    r!   _create_file_info>  s   zFSSpecHandler._create_file_infoc              	   C   sZ   g }|D ]&}z| j |}W n ty    |t|tj Y qw || || q|S rC   )rD   rY   FileNotFoundErrorappendr   r   r8   rZ   )rE   pathsinfosr)   rY   r    r    r!   r%   K  s   zFSSpecHandler.get_file_infoc           	      C   s   | j |js| j |jrt|j|jrg S t|j|jr#d }nd}g }| j j|j|ddd}|	 D ]\}}|
d}|j
d}||krR|| || q6|S )N   T)maxdepthZwithdirsdetail/)rD   isdirbase_direxistsNotADirectoryErrorZallow_not_foundr[   r?   finditemsstripr\   rZ   )	rE   selectorr`   r^   Zselected_filesr)   rY   _pathrd   r    r    r!   get_file_info_selectorV  s(   



z$FSSpecHandler.get_file_info_selectorc                 C   s,   z| j j||d W d S  ty   Y d S w )N)Zcreate_parents)rD   mkdirFileExistsError)rE   r)   r?   r    r    r!   
create_dirs  s
   zFSSpecHandler.create_dirc                 C   s   | j j|dd d S NTr>   )rD   rmrQ   r    r    r!   
delete_dirz  s   zFSSpecHandler.delete_dirc                 C   st   z
| j j|dd}W n ty   |rY d S  w |D ]}| j |r+| j j|dd q| j |r7| j | qd S )NF)ra   Tr>   )rD   listdirr[   rc   rq   isfile)rE   r)   missing_dir_okZsubpathssubpathr    r    r!   _delete_dir_contents}  s   z"FSSpecHandler._delete_dir_contentsc                 C   s*   | ddkrtd|d| || d S )Nrb    z$delete_dir_contents called on path '')ri   r(   rw   )rE   r)   ru   r    r    r!   delete_dir_contents  s
   z!FSSpecHandler.delete_dir_contentsc                 C   s   |  d d S )Nrb   )rw   )rE   r    r    r!   delete_root_dir_contents  s   z&FSSpecHandler.delete_root_dir_contentsc                 C   s$   | j |s
t|| j | d S rC   )rD   re   r[   rq   rQ   r    r    r!   delete_file  s   zFSSpecHandler.delete_filec                 C   s   | j j||dd d S rp   )rD   mvrE   srcdestr    r    r!   move  s   zFSSpecHandler.movec                 C   s   | j || d S rC   )rD   copyr~   r    r    r!   	copy_file  s   zFSSpecHandler.copy_filec                 C   8   ddl m} | j|st||| jj|ddddS Nr   
PythonFilerbmoderpyarrowr   rD   rt   r[   openrE   r)   r   r    r    r!   open_input_stream     zFSSpecHandler.open_input_streamc                 C   r   r   r   r   r    r    r!   open_input_file  r   zFSSpecHandler.open_input_filec                 C   $   ddl m} || jj|ddddS )Nr   r   wbr   wr   r   rD   r   rE   r)   metadatar   r    r    r!   open_output_stream     z FSSpecHandler.open_output_streamc                 C   r   )Nr   r   abr   r   r   r   r    r    r!   open_append_stream  r   z FSSpecHandler.open_append_streamN)r3   
__module____qualname____doc__rF   rK   rM   rP   r$   staticmethodrZ   r%   rl   ro   rr   rw   rz   r{   r|   r   r   r   r   r   r   r    r    r    r!   r4     s.    
r4   )FF)NFF)NN)+r   Zpyarrow.utilr   r   Zpyarrow._fsr   r   r   r   r   r	   r
   r   r   r   r   Z	FileStatsr   Zpyarrow._hdfsr   r   r\   Zpyarrow._gcsfsr   Zpyarrow._s3fsr   r   r   r   r   r   r   r   r   r   atexitregisterr"   r-   r7   r;   rB   r4   r    r    r    r!   <module>   sB   44

*
AK