o
    Zh8                     @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl	Z
d dlmZmZmZmZ edddZG dd dZG d	d
 d
eZG dd deZG dd deZdd Zdd ZdddZdS )    N)join)doc_stringify_path_is_path_like	_DEPR_MSGzfilesystem.LocalFileSystemz2.0.0zfs.LocalFileSystemc                   @   s   e Zd ZdZdd Zdd Zd(ddZd	d
 Zdd Zdd Z	d(ddZ
dd Zdd Zd)ddZdd Zdd Zdd Zdd Z	 	d*d!d"Zd+d$d%Zed&d' Zd S ),
FileSystemz(
    Abstract filesystem interface.
    c                 C   s8   |  |d}| W  d   S 1 sw   Y  dS )z
        Return contents of file as a bytes object.

        Parameters
        ----------
        path : str
            File path to read content from.

        Returns
        -------
        contents : bytes
        rbN)openread)selfpathf r   I/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/filesystem.pycat)   s   $zFileSystem.catc                 C      t )z
        Return list of file paths.

        Parameters
        ----------
        path : str
            Directory to list contents from.
        NotImplementedErrorr   r   r   r   r   ls9      	zFileSystem.lsFc                 C   r   )z
        Delete the indicated file or directory.

        Parameters
        ----------
        path : str
            Path to delete.
        recursive : bool, default False
            If True, also delete child paths for directories.
        r   r   r   	recursiver   r   r   deleteD      zFileSystem.deletec           	      C   sl   t |}| |}|d dkr|d S d}| |D ]\}}}|D ]}| ||}|| |d 7 }q!q|S )z
        Compute bytes used by all contents under indicated path in file tree.

        Parameters
        ----------
        path : str
            Can be a file path or directory.

        Returns
        -------
        usage : int
        kindfilesizer   )r   statwalk
_path_join)	r   r   Z	path_infototalrootdirectoriesfilesZ
child_pathabspathr   r   r   
disk_usageQ   s   
zFileSystem.disk_usagec                 G   s   | j |S N)pathsepr   )r   argsr   r   r   r    k   s   zFileSystem._path_joinc                 C      t d)zl
        Information about a filesystem entry.

        Returns
        -------
        stat : dict
        FileSystem.statr   r   r   r   r   r   n   s   r+   c                 C   s   | j ||dS )z.
        Alias for FileSystem.delete.
        r   )r   r   r   r   r   rmx   s   zFileSystem.rmc                 C   s   |  ||S )z.
        Alias for FileSystem.rename.
        )renamer   r   new_pathr   r   r   mv~   s   zFileSystem.mvc                 C   r*   )z
        Rename file, like UNIX mv command.

        Parameters
        ----------
        path : str
            Path to alter.
        new_path : str
            Path to move to.
        FileSystem.renamer   r/   r   r   r   r.      s   r2   Tc                 C   r   )z
        Create a directory.

        Parameters
        ----------
        path : str
            Path to the directory.
        create_parents : bool, default True
            If the parent directories don't exists create them as well.
        r   r   r   Zcreate_parentsr   r   r   mkdir   r   zFileSystem.mkdirc                 C   r   )z
        Return True if path exists.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   r   r   r   exists   r   zFileSystem.existsc                 C   r   )z
        Return True if path is a directory.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   r   r   r   isdir   r   zFileSystem.isdirc                 C   r   )z
        Return True if path is a file.

        Parameters
        ----------
        path : str
            Path to check.
        r   r   r   r   r   isfile   r   zFileSystem.isfilec                 C   r   )zf
        Returns True if this FileSystem is a unix-style file store with
        directories.
        r   r   r   r   r   _isfilestore   s   zFileSystem._isfilestoreNc           	      C   s,   ddl m} ||||| d}|j|||dS )a]  
        Read Parquet data from path in file system. Can read from a single file
        or a directory of files.

        Parameters
        ----------
        path : str
            Single file path or directory
        columns : List[str], optional
            Subset of columns to read.
        metadata : pyarrow.parquet.FileMetaData
            Known metadata to validate files against.
        schema : pyarrow.parquet.Schema
            Known schema to validate files against. Alternative to metadata
            argument.
        use_threads : bool, default True
            Perform multi-threaded column reads.
        use_pandas_metadata : bool, default False
            If True and file has custom pandas schema metadata, ensure that
            index columns are also loaded.

        Returns
        -------
        table : pyarrow.Table
        r   )ParquetDataset)schemametadata
filesystem)columnsuse_threadsuse_pandas_metadata)Zpyarrow.parquetr:   r
   )	r   r   r>   r<   r;   r?   r@   r:   Zdatasetr   r   r   read_parquet   s   zFileSystem.read_parquetr   c                 C   r   )3
        Open file for reading or writing.
        r   r   r   moder   r   r   r	      s   zFileSystem.openc                 C      dS )N/r   r8   r   r   r   r(         zFileSystem.pathsepFT)NNNTFr   )__name__
__module____qualname____doc__r   r   r   r&   r    r   r-   r1   r.   r4   r5   r6   r7   r9   rA   r	   propertyr(   r   r   r   r   r   $   s,    





!r   c                       s   e Zd ZdZ fddZedd Zedd Zee	j
dd	 Z
ee	jdddZee	jdd Zee	jdd Zee	jdd Zee	jdd Zee	jdddZedd Zdd Z  ZS )LocalFileSystemNc                    s   t jttdd t   d S N   
stacklevel)warningswarn_FS_DEPR_MSGFutureWarningsuper__init__r8   	__class__r   r   rZ      s   zLocalFileSystem.__init__c                 C   sN   | j d u r$t  td t | _ W d    | j S 1 sw   Y  | j S )Nignore)	_instancerU   catch_warningssimplefilterrP   clsr   r   r   _get_instance   s   




zLocalFileSystem._get_instancec                 C   s   t jttdd |  S rQ   )rU   rV   rW   rX   rc   ra   r   r   r   get_instance  s   zLocalFileSystem.get_instancec                    s$   t   t fddt D S )Nc                 3   s    | ]}t  |V  qd S r'   )pjoin.0xr   r   r   	<genexpr>
  s    z%LocalFileSystem.ls.<locals>.<genexpr>)r   sortedoslistdirr   r   ri   r   r     s   zLocalFileSystem.lsTc                 C   s(   t |}|rt| d S t| d S r'   )r   rl   makedirsr4   r3   r   r   r   r4     s   zLocalFileSystem.mkdirc                 C      t |}tj|S r'   )r   rl   r   r6   r   r   r   r   r6        zLocalFileSystem.isdirc                 C   ro   r'   )r   rl   r   r7   r   r   r   r   r7     rp   zLocalFileSystem.isfilec                 C   rE   )NTr   r8   r   r   r   r9     rG   zLocalFileSystem._isfilestorec                 C   ro   r'   )r   rl   r   r5   r   r   r   r   r5   "  rp   zLocalFileSystem.existsr   c                 C   s   t |}t||dS rB   )rD   )r   r	   rC   r   r   r   r	   '  s   zLocalFileSystem.openc                 C   s   t jjS r'   )rl   r   sepr8   r   r   r   r(   /     zLocalFileSystem.pathsepc                 C   s   t |}t|S )z8
        Directory tree generator, see os.walk.
        )r   rl   r   r   r   r   r   r   3  s   
zLocalFileSystem.walkrI   rJ   )rK   rL   rM   r^   rZ   classmethodrc   rd   r   r   r   r4   r6   r7   r9   r5   r	   rO   r(   r   __classcell__r   r   r[   r   rP      s0    







rP   c                   @   s   e Zd ZdZdd Zeejdd Zeejdd Zeej	dd	 Z	eej
dddZ
eejdd ZeejdddZeejdddZdddZdd ZdS )DaskFileSystemzJ
    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
    c                 C   s   t jdtdd || _d S )Nz~The pyarrow.filesystem.DaskFileSystem/S3FSWrapper are deprecated as of pyarrow 3.0.0, and will be removed in a future version.rR   rS   )rU   rV   rX   fs)r   rw   r   r   r   rZ   @  s
   
zDaskFileSystem.__init__c                 C   r*   NzUnsupported file system APIr   r   r   r   r   r6   G  rs   zDaskFileSystem.isdirc                 C   r*   rx   r   r   r   r   r   r7   K  rs   zDaskFileSystem.isfilec                 C   rE   )zh
        Object Stores like S3 and GCSFS are based on key lookups, not true
        file-paths.
        Fr   r8   r   r   r   r9   O  s   zDaskFileSystem._isfilestoreFc                 C      t |}| jj||dS )Nr,   )r   rw   r-   r   r   r   r   r   W  s   zDaskFileSystem.deletec                 C      t |}| j|S r'   )r   rw   r5   r   r   r   r   r5   \  rp   zDaskFileSystem.existsTc                 C   s$   t |}|r| j|S | j|S r'   )r   rw   Zmkdirsr4   r3   r   r   r   r4   a  s   zDaskFileSystem.mkdirr   c                 C   ry   rq   )r   rw   r	   rC   r   r   r   r	   i  s   zDaskFileSystem.openc                 C   ry   )N)detail)r   rw   r   )r   r   r{   r   r   r   r   q  s   zDaskFileSystem.lsc                 C   rz   )z9
        Directory tree generator, like os.walk.
        )r   rw   r   r   r   r   r   r   u  s   zDaskFileSystem.walkNrH   rI   rJ   )rK   rL   rM   rN   rZ   r   r   r6   r7   r9   r   r5   r4   r	   r   r   r   r   r   r   rv   ;  s&    




rv   c                   @   s:   e Zd Zeejdd Zeejdd Zd	ddZdS )
S3FSWrapperc                 C   sR   t t|}z| j|}t|dkr|d |krW dS W dS  ty(   Y dS w )N   r   FT_sanitize_s3r   rw   r   lenOSErrorr   r   contentsr   r   r   r6     s   zS3FSWrapper.isdirc                 C   sH   t t|}z| j|}t|dko|d |kW S  ty#   Y dS w )Nr}   r   Fr~   r   r   r   r   r7     s   zS3FSWrapper.isfileFc                 #   s    t t|}t  t }t| jj||dD ]}|d }|d dkr) | q|d dkr0q|| qt fdd|D }tdd  D  | |fV   D ]}| j||dE d	H  qRd	S )
z
        Directory tree generator, like os.walk.

        Generator version of what is in s3fs, which yields a flattened list of
        files.
        )refreshKeyZStorageClassZ	DIRECTORYZBUCKETc                    s"   g | ]}| vrt |d  qS r}   	posixpathsplit)rg   r   r#   r   r   
<listcomp>  s    z$S3FSWrapper.walk.<locals>.<listcomp>c                 S   s   g | ]	}t |d  qS r   r   rf   r   r   r   r     s    N)	r   r   setlistrw   Z_lsaddrk   r   )r   r   r   r$   key	directoryr   r   r   r     s&   zS3FSWrapper.walkNrH   )rK   rL   rM   r   r   r6   r7   r   r   r   r   r   r|   }  s    

r|   c                 C   s   |  dr| ddS | S )Nzs3:// )
startswithreplaceri   r   r   r   r     s   
r   c                 C   sH   t | }t|ts"dtjv rtjd }t| |jr| S td|| S )NfsspeczUnrecognized filesystem: {})	type
issubclassr   sysmodules
isinstanceZAbstractFileSystemr   format)rw   Zfs_typer   r   r   r   _ensure_filesystem  s   


r   c           	      C   s:  t | s|durtd|| fS |dur4t|}t|tr%t| }||fS t| ts.td| }||fS t| }tj	
|}|jdksH|jdkr|jd}|d }|dkrYd	}n|jd
 | }d}t|dkrt|d  rtt|d }tjj||d}|j}||fS |jdkrt }|j}||fS t }|}||fS )zn
    Return filesystem from path which could be an HDFS URI, a local URI,
    or a plain filesystem path.
    NzVfilesystem passed but where is file-like, so there is nothing to open with filesystem.zPExpected string path; path-like objects are only allowed with a local filesystemhdfsZviewfs:r   r   defaultz://rR   r}   )hostportr   )r   
ValueErrorr   r   rP   r   str	TypeErrorurllibparseurlparseschemenetlocr   r   	isnumericintpar   _connectr   rc   )	wherer=   r   Z
parsed_uriZnetloc_splitr   r   rw   Zfs_pathr   r   r   resolve_filesystem_and_path  sH   



r   r'   )rl   r   r   urllib.parser   rU   Zos.pathr   re   Zpyarrowr   Zpyarrow.utilr   r   r   r   r   rW   r   rP   rv   r|   r   r   r   r   r   r   r   <module>   s&    OIB7