o NZh[W@sfddlmZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZmZmZddlmZddlmZmZdd lmZmZmZmZed ZGd d d ZGd ddeZ          d2ddZ ddZ!ddZ"      d3ddZ# d4d5d"d#Z$d$d%Z%d&d'Z&d(d)Z'd*d+Z(      d6d,d-Z)d.d/Z*Gd0d1d1ej+Z,dS)7) annotationsN) has_magic)Path) BaseCache BlockCache BytesCache MMapCacheReadAheadCachecaches)compr) filesystemget_filesystem_class)_unstrip_protocolbuild_name_functioninfer_compressionstringify_pathZfsspecc@s`eZdZdZ     dddZddZdd Zd d Zd d Ze ddZ ddZ ddZ dS)OpenFileaF File-like object to be used in a context Can layer (buffered) text-mode and compression over any file-system, which are typically binary-only. These instances are safe to serialize, as the low-level file object is not created until invoked using ``with``. Parameters ---------- fs: FileSystem The file system to use for opening the file. Should be a subclass or duck-type with ``fsspec.spec.AbstractFileSystem`` path: str Location to open mode: str like 'rb', optional Mode of the opened file compression: str or None, optional Compression to apply encoding: str or None, optional The encoding to use if opened in text mode. errors: str or None, optional How to handle encoding errors if opened in text mode. newline: None or str Passed to TextIOWrapper in text mode, how to handle line endings. autoopen: bool If True, calls open() immediately. Mostly used by pickle pos: int If given and autoopen is True, seek to this location immediately rbNcCs:||_||_||_t|||_||_||_||_g|_dSN) fspathmodeget_compression compressionencodingerrorsnewlinefobjects)selfrrrrrrrr B/var/www/html/lang_env/lib/python3.10/site-packages/fsspec/core.py__init__@s   zOpenFile.__init__cCs$t|j|j|j|j|j|j|jffSr)rrrrrrrrrr r r! __reduce__SszOpenFile.__reduce__cCsd|jdS)Nz )rr#r r r!__repr__aszOpenFile.__repr__cCs|jddddd}|jj|j|d}|g|_|jdur2t|j}|||dd}|j|d|jvrHt ||j |j |j d}|j||jdS)Ntb)rr)rrr) rreplaceropenrrrr appendPickleableTextIOWrapperrrr)rrfcompressr r r! __enter__ds      zOpenFile.__enter__cGs |dSr)close)rargsr r r!__exit__ys zOpenFile.__exit__cCst|j|jSr)rrrr#r r r! full_name|szOpenFile.full_namecCs|S)aMaterialise this as a real open file without context The OpenFile object should be explicitly closed to avoid enclosed file instances persisting. You must, therefore, keep a reference to the OpenFile during the life of the file-like it generates. r0r#r r r!r+sz OpenFile.opencCs>t|jD]}d|jvr|js||q|jdS)z#Close all encapsulated file objectsrN)reversedrrclosedflushr1clear)rr.r r r!r1s  zOpenFile.close)rNNNN) __name__ __module__ __qualname____doc__r"r$r%r0r3propertyr4r+r1r r r r!rs $   rcsLeZdZdZdddfdd ZddZd d Zfd d Zd dZZ S) OpenFilesaList of OpenFile instances Can be used in a single context, which opens and closes all of the contained files. Normal list access to get the elements works as normal. A special case is made for caching filesystems - the files will be down/uploaded together at the start or end of the context, and this may happen concurrently, if the target filesystem supports it. rNrrcs"||_||_g|_tj|dSr)rrfilessuperr")rrrr2 __class__r r!r"szOpenFiles.__init__cCsd|jdur td|j} t|dr|||_|jSt|dr)|jdur)|j}nnq dd|DS)NzContext has already been usedT open_manyrcSsg|]}|qSr r5.0sr r r! z'OpenFiles.__enter__..)r ValueErrorhasattrrFrB)rrr r r!r0s    zOpenFiles.__enter__csd|j}fdd|Dd|jvr0 t|dr||jdSt|dr-|jdur-|j}ndSqdS)Ncsg|]}|jqSr )r3rGr2r r!rJz&OpenFiles.__exit__..r6TrFr)rrrMZ commit_manyrB)rr2rr rNr!r3s   zOpenFiles.__exit__cs,t|}t|trt||j|jdS|S)NrA)rC __getitem__ isinstanceslicer@rr)ritemoutrDr r!rPs  zOpenFiles.__getitem__cCsdt|dS)Nz )lenr#r r r!r%szOpenFiles.__repr__) r;r<r=r>r"r0r3rPr% __classcell__r r rDr!r@s  r@rutf8Tc  st|||| || d\} } jdkr| _n&dvr=| r=fdd| D}|D]}z j|ddWq(ty<Yq(wtfdd | Dd S) av Given a path or paths, return a list of ``OpenFile`` objects. For writing, a str path must contain the "*" character, which will be filled in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. For either reading or writing, can instead provide explicit list of paths. Parameters ---------- urlpath: string or list Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` to read from alternative filesystems. To read from multiple files you can pass a globstring or a list of paths, with the caveat that they must all have the same protocol. mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode name_function: function or None if opening a set of files for writing, those files do not yet exist, so we need to generate their names by formatting the urlpath for each sequence number num: int [1] if writing mode, number of files we expect to create (passed to name+function) protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. auto_mkdir: bool (True) If in write mode, this will ensure the target directory exists before writing, by calling ``fs.mkdirs(exist_ok=True)``. expand: bool **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> files = open_files('2015-*-*.csv') # doctest: +SKIP >>> files = open_files( ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' ... ) # doctest: +SKIP Returns ------- An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can be used as a single context Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations )num name_functionstorage_optionsprotocolexpandfiler6csh|]}|qSr )_parentrHrrr r! 'rOzopen_files..T)exist_okc s"g|] }t|dqS))rrrrr)rr_rrrrrrr r!rJ.s zopen_files..rA)get_fs_token_pathsr[ auto_mkdirmakedirsPermissionErrorr@)urlpathrrrrrYrXr[rrer\kwargsZfs_tokenpathsparentsparentr rcr! open_filess4O     rmc stdd|vrfdd|dDn|g}g}d}|}t|D]L}|ddp3t|dp3d}t|}||}||i} ||durN| |t d i|| } | |}|dvrfd | vrf|}| ||| f|}q$t t|}|S) Nz .*[^a-z]+.*z::cs*g|]}d|vs |r|n|dqS)://)matchrHpxr r!rJBs*z_un_chain..r[rr]>Z simplecache filecacheZ blockcachetarget_protocolr )recompilesplitcopyr7popsplit_protocolrZ_get_kwargs_from_urlsupdatedict_strip_protocolr,list) rribitsrTZ previous_bitbitr[cls extra_kwargskwskwr rrr! _un_chain?s0        rc  shdfdd|D}t||}i}tt|D].\}}|\}}}|t|dkr7td i||}qtd i|||d<||d<||d<q|d\} }} t|fi|} | | fS) a Turn fully-qualified and potentially chained URL into filesystem instance Parameters ---------- url : str The fsspec-compatible URL **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Returns ------- filesystem : FileSystem The new filesystem discovered from ``url`` and created with ``**kwargs``. urlpath : str The file-systems-specific URL for ``url``. >rYrXrr\rrrrcsi|] \}}|vr||qSr r )rHkvZ known_kwargsr r! ~szurl_to_fs..rtarget_optionsruforNr )itemsr enumerater7rUr}r ) urlrichaininkwargsichurlsr[rrh_rr rr! url_to_fs^s   rc Ks6td|g||||||dd|}|st||dS)aGiven a path or paths, return one ``OpenFile`` object. Parameters ---------- urlpath: string or list Absolute or relative filepath. Prefix with a protocol like ``s3://`` to read from alternative filesystems. Should not include glob character(s). mode: 'rb', 'wt', etc. compression: string or None If given, open file using compression codec. Can either be a compression name (a key in ``fsspec.compression.compr``) or "infer" to guess the compression from the filename suffix. encoding: str For text mode only errors: None or str Passed to TextIOWrapper in text mode protocol: str or None If given, overrides the protocol found in the URL. newline: bytes or None Used for line terminator in text mode. If None, uses system default; if blank, uses no translation. **kwargs: dict Extra options that make sense to a particular storage connection, e.g. host, port, username, password, etc. Examples -------- >>> openfile = open('2015-01-01.csv') # doctest: +SKIP >>> openfile = open( ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' ... ) # doctest: +SKIP >>> with openfile as f: ... df = pd.read_csv(f) # doctest: +SKIP ... Returns ------- ``OpenFile`` object. Notes ----- For a full list of the available protocols and the implementations that they map across to see the latest online documentation: - For implementations built into ``fsspec`` see https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations - For implementations in separate packages see https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations F)rhrrrrr[rr\rNr )rmFileNotFoundError) rhrrrrr[rrirTr r r!r+s<  r+r#str | list[str] | Path | list[Path]rstrrZr}returnstr | list[str]cKsd|vrtdt|fd|i|}t|djddstd|}dd |D}Wd n1s3wYt|trAt|rFt|trJ|dS|S) aKOpen file(s) which can be resolved to local For files which either are local, or get downloaded upon open (e.g., by file caching) Parameters ---------- url: str or list(str) mode: str Must be read mode storage_options: passed on to FS for or used by open_files (e.g., compression) r6z(Can only ensure local files when readingrr local_fileFzOopen_local can only be used on a filesystem which has attribute local_file=TruecSsg|]}|jqSr )namerHr.r r r!rJszopen_local..N)rLrmgetattrrrQrrr)rrrZZofrBrjr r r! open_localsrcCs4|dkrt|}|dur|tvrtd|d|S)NZinferzCompression type z not supported)rr rL)rhrr r r!rs rcCsRt|}d|vr|dd\}}t|dkr||fS|dr%|ddSd|fS)zReturn protocol, path pairrnrzdata::N)rrxrU startswith)rhr[rr r r!r{s   r{cCst|\}}t|}||S)zCReturn only path part of full URL, according to appropriate backend)r{rr~)rhr[rrr r r!strip_protocols  rcCsg}t|}d|vrFtdd|Ddkrtdt|t|}|D]}d|vr2|t|||q"||q"t||krD|d|}|S|D]}t|rW|| |qH||qH|S)a(Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` in them (read mode). :param paths: list of paths mode: str Mode in which to open files. num: int If opening in writing mode, number of files we expect to create. fs: filesystem object name_function: callable If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. :return: list of paths wcSsg|]}d|vrdqS)*rr rpr r r!rJ,sz*expand_paths_if_needed..rz;When writing data, only one filename mask can be specified.rN) rsumrLmaxrUextend _expand_pathsr,rglob)rjrrXrrYZexpanded_pathsZ curr_pathr r r!expand_paths_if_neededs(     rcst|tttfr|stdtt|d}nt|}pi|r%|d<t|p*i}i} tt|D].\} } | \} } }| t |dkrOt di|| } q4t di|| | d<| | d<| | d<q4|d\}}}t |fi| t|tttfrfdd |D}t d d |Ddkrtd |d d |D}n |}t|tttfrt ||||}n0d|vr|rt|||}n#d|vr|rt|||}nd|vrއfdd t|D}n|g}j|fS)aFilesystem, deterministic token, and paths from a urlpath and options. Parameters ---------- urlpath: string or iterable Absolute or relative filepath, URL (may include protocols like ``s3://``), or globstring pointing to data. mode: str, optional Mode in which to open files. num: int, optional If opening in writing mode, number of files we expect to create. name_function: callable, optional If opening in writing mode, this callable is used to generate path names. Names are generated for each partition by ``urlpath.replace('*', name_function(partition_index))``. storage_options: dict, optional Additional keywords to pass to the filesystem class. protocol: str or None To override the protocol specifier in the URL expand: bool Expand string paths for writing, assuming the path is a directory zempty urlpath sequencerr[rrrurcs"g|] }tt|p idqSr)rr)rHu)rZr r!rJ}sz&get_fs_token_paths..cSsh|]}|dqS)rr rHZpcr r r!rarKz%get_fs_token_paths..z$Protocol mismatch getting fs from %scSsg|]}|dqSrr rr r r!rJrKrrsrcsg|] }|s|qSr )isdirrr`r r!rJsNr )rQrtuplesetrLrrrr7rUr}r r~rrsortedrZ _fs_token)rhrrXrYrZr[r\Zurlpath0rrrrrZnested_protocolrrjrZpchainsr )rrZr!rdGsL        rdcsttr>ddkrtddvrtjddur%t|dfddt|D}|t |kr.zqIn order to preserve order between partitions paths created with ``name_function`` should sort to partition orderzPath should be either 1. A list of paths: ['foo.json', 'bar.json', ...] 2. A directory: 'foo/ 3. A path with a '*' in it: 'foo.*.json')rQrcountrLosrjoinrrangerloggerwarningrrrU)rrYrXrjr rr!rs(   rcs4eZdZdZ     dfdd ZddZZS) r-zTextIOWrapper cannot be pickled. This solves it. Requires that ``buffer`` be pickleable, which all instances of AbstractBufferedFile are. NFcs$||||||f|_tj|jdSr)r2rCr")rbufferrrrline_buffering write_throughrDr r!r"s z PickleableTextIOWrapper.__init__cCs t|jfSr)r-r2r#r r r!r$s z"PickleableTextIOWrapper.__reduce__)NNNFF)r;r<r=r>r"r$rVr r rDr!r-s  r-) rNrWNNrNNTT)rNrWNNN)r)rrrrrZr}rr)rrNNNT)- __future__riologgingrrvrrpathlibrZcachingrrrr r r rr registryr rutilsrrrr getLoggerrrrr@rmrrr+rrr{rrrdr TextIOWrapperr-r r r r!sb      s< s3 N ! 1 M