o TZh@sddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddlm Z ddl mZddlmZmZddlmZmZmZmZmZmZmZmZmZddlmZddlZddl m!Z!dd l"m#Z#dd l$m%Z%d d l&m'Z'd d l(m)Z)d dl*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0d dl1m2Z2d dl3m4Z4ddl5m6Z6e2e7Z8gdZ9idde)DddiZ:dde)DZ;e?dde>?dde>?dde>?dde>?dd e>?d!d"e>?d#d$e>?d%d&iZ@d'd(iZAeBd)d*ee@eADZCGd+d,d,eDZEd-d.ZFd/d0ZGdzd1eHd2ee6fd3d4ZId5d6ZJd7d8ZKd9d:ZLdzd2ee6d;eMfdd?ZPdzd2ee6d;eMfd@dAZQdzdBdCZRdDdEZSdFeHd;eHfdGdHZTd;eeHfdIdJZUdzd1eHd2ee6d;eeHfdKdLZV dzd1eHd2ee6d;eeHeeHeeHeffffdMdNZW dzd1eHd2ee6d;eeHeeHeeHeffffdOdPZXd{ddRdSeHd2ee6fdTdUZYdzdFeHd2ee6d;eeHfdVdWZZdXddYd2ee6fdZd[Z[dzd2ee6fd\d]Z\Gd^d_d_e]eZ^dFeeHee^ffd`daZ_ddRd2ee6fdbdcZ`ddRd2ee6fdddeZadzd2ee6fdfdgZbdzd2ee6fdhdiZcdzd2ee6fdjdkZddzd2ee6fdldmZed|d2ee6fdndoZfdzd2ee6fdpdqZgGdrdsdseZhGdtduduehZiGdvdwdwehZjGdxdydyZkdS)}N) TimeoutError)BytesIO)chain)Path PurePosixPath) AnyCallableDict GeneratorIterableListOptionalTupleUnion) ElementTree) ClientError)EntryNotFoundError)version)config)COMPRESSION_FILESYSTEMS)"get_authentication_headers_for_urlget_datasets_user_agent http_head is_local_pathis_relative_pathurl_or_path_join) get_logger) map_nested)DownloadConfig) txtcsvjsonZjsonlZtsvZconllZconlluorigparquetZpklpicklerelxmlcCsi|] }|jd|jqS).) extensionlstripprotocol.0Zfs_classr/c/var/www/html/lang_env/lib/python3.10/site-packages/datasets/download/streaming_download_manager.py 8sr1zipcCsh|]}|jqSr/)r,r-r/r/r0 <sr3z(?Ms  r:c@s eZdZdS)NonStreamableDatasetErrorN)__name__ __module__ __qualname__r/r/r/r0r;Ssr;cGsPt|d^}}t|rtjj|g|RStj|g|R}d|g|S)u# This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xjoin function allows you to apply the join on the first path of the chain. Example:: >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") zip://folder1/file.txt::https://host.com/archive.zip ::)strsplitrospathjoin posixpath)apbr/r/r0xjoinWs rIcCs\t|d^}}t|rtjt|}nt|}| dr&|d7}d |g|S)u# This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xdirname function allows you to apply the dirname on the first path of the chain. Example:: >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") zip://folder1::https://host.com/archive.zip r?://) r@rArrBrCdirnameras_posixrEendswithrDrFrHr/r/r0xdirnamers  rPurlpathdownload_configcCs`t|d^}}t|rtj|St||d\}}|d^}}tj||d^}}||S)aExtend `os.path.exists` function to support both local and remote files. Args: urlpath (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r?rRstorage_options) _as_strrArrBrCexists!_prepare_path_and_storage_optionsfsspecget_fs_token_paths)rQrRmain_hop rest_hopsrUfs_r/r/r0xexistss   r_cCs8t|d^}}t|rtjt|St|S)u This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xbasename function allows you to apply the basename on the first path of the chain. Example:: >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") file.txt r?) r@rArrBrCbasenamerrMrErOr/r/r0 xbasenames racCsbt|d^}}t|rtjt|St|\}}d| dr)|dn|g||fS)u, This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplit function allows you to apply the xsplit on the first path of the chain. Example:: >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1::https://host.com/archive.zip', 'file.txt') r?rJrK) r@rArrBrCrrMrErDrN)rFrHtailr/r/r0xsplits &rccCsPt|d^}}t|rtjt|St|\}}d |g||fS)u8 This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplitext function allows you to apply the splitext on the first path of the chain. Example:: >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1/file::https://host.com/archive.zip', '.txt') r?) r@rArrBrCsplitextrrMrErD)rFrHextr/r/r0 xsplitexts rfreturncCs`t|d^}}t|rtj|St||d\}}|d^}}tj||d^}}||S)zExtend `os.path.isfile` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r?rSrT) r@rArrBrCisfilerXrYrZ)rCrRr[r\rUr]r^r/r/r0xisfiles   ric Cst|d^}}t|rtj|St||d\}}|d^}}tj||d^}}z| |}Wnt y@t d|w|durbt ||d}t |}Wd|S1s]wY|S)zExtend `os.path.getsize` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `int`: optional r?rSrTzNo such file: N)r@rArrBrCgetsizerXrYrZsizerFileNotFoundErrorxopenr8read) rCrRr[r\rUr]r^rkfr/r/r0xgetsize s$    rpcCs|t|d^}}t|rtj|St||d\}}|d^}}tj||d^}}|dd}| ds9dS||S)zExtend `os.path.isdir` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` r?rSrT:///T) r@rArrBrCisdirrXrYrZstrip)rCrRr[r\rUr]r^ inner_pathr/r/r0xisdir&s    rwcCsft|d^}}t|r|rtjj||dStj|S|r-tj|t|dddStj|S)zExtend `os.path.relpath` function to support remote files. Args: path (`str`): URL path. start (`str`): Start URL directory path. Returns: `str` r?)startr)r@rArrBrCrelpathrE)rCrxr[r\r/r/r0xrelpath=s  ,rzcs$|jtjfdd}||_dS)Nc sd}tddD]9}z |i|}W|SttfyB}z|}tdtjd|ddttjWYd}~q d}~wwt d|)Nrz4Got disconnected from remote data host. Retrying in zsec [rs]zServer Disconnected) rangerrloggerwarningrZSTREAMING_READ_RETRY_INTERVALtimesleepConnectionError)argskwargsZdisconnect_errretryouterr max_retriesrnr/r0read_with_retriesRs  z?_add_retries_to_file_obj_read_method..read_with_retries)rnrZSTREAMING_READ_MAX_RETRIES)file_objrr/rr0$_add_retries_to_file_obj_read_methodNs rrCcCs*|dd}dD] }||d}q |S)Nr)rrz?-_r)rA)rCr*Zsymbr/r/r0_get_path_extensionesrc Csz|dWn ttjfyYdSw|t}|dttD],}t|dt|}|dur8|St |dt|}|durOt d|dq#dS)zQread the magic number from a file-like object and return the compression protocolrNzCompression protocol 'z' not implemented.) seekAttributeErrorioUnsupportedOperationrnMAGIC_NUMBER_MAX_LENGTHr|$MAGIC_NUMBER_TO_COMPRESSION_PROTOCOLget0MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOLNotImplementedError)ror9i compressionr/r/r0*_get_extraction_protocol_with_magic_numberos    rcCst|}|dd}t|}|tvs|dvs|drdS|tvr&t|St||d\}}z#tj|fi|p7i }t |WdWS1sJwYWdSt yf| t j ret |ddw)Nr?rtgztarz.tar.gzz.tar.bz2z.tar.xzrSS If the repo is private or gated, make sure to log in with `huggingface-cli login`.)r@rArBASE_KNOWN_EXTENSIONSrN!COMPRESSION_EXTENSION_TO_PROTOCOLrXrYopenrrl startswithr HF_ENDPOINT)rQrRrCr*rUror/r/r0_get_extraction_protocols0(  rcCsJg}i}|dD]}t||d\}}||||q d||fS)Nr?rS)rA,_prepare_single_hop_path_and_storage_optionsappendupdaterD)rQrRZprepared_urlpathZprepared_storage_optionsZhoprUr/r/r0rXs  rXc Cs|durdn|j}|tjr%d|vr%d|ttjddddd}d|vr0|ddnd}|durA||jvrA|j|}n|durU||jvrUd d |jD}ni}|r]||i}|d vrit ||d d t iddid| |i||<d|vrt |}d}|j D]\}}|dr|d|7}|j }d|i| |i||<qd|vrd|vr|d7}|drd||dd<||fS|dkr|tjd| |i||<tjtdkrd||d<||fS) a\ Prepare the URL and the kwargs that must be passed to the HttpFileSystem or to requests.get/head In particular it resolves google drive URLs It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. Storage options are formatted in the form {protocol: storage_options_for_protocol} Nz /resolve/zhf://r@rqrfilecSs"i|] \}}|tvr||qSr/)rYZavailable_protocols)r.Z option_nameZ option_valuer/r/r0r1s  z@_prepare_single_hop_path_and_storage_options..)httphttps)tokenz user-agent trust_envT)headersZ client_kwargszdrive.google.comZdownload_warningz &confirm=cookieszconfirm=z &confirm=tz"https://raw.githubusercontent.com/identityrzAccept-EncodingZhf)rZendpointz0.21.0default block_size)rrrrr8replacerArUitemsrrrrrZHF_HUB_VERSIONrparse) rQrRrr,rUresponserkvr/r/r0rs\ $         rrrSrc Ost|}|d^}}t|r!|ddt||g|Ri|St||d\}}i||p/i}ztj|g|Rd|i|} Wn,ty[} z t| dkrVt d| d} ~ wt yo| t j rnt |ddwt| | S) aExtend `open` function to support remote files using `fsspec`. It also has a retry mechanism in case connection fails. The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co Args: file (`str`): Path name of the file to be opened. mode (`str`, *optional*, default "r"): Mode in which the file is opened. *args: Arguments to be passed to `fsspec.open`. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Keyword arguments to be passed to `fsspec.open`. Returns: file object r?rNrSmodezCannot seek streaming HTTP filezStreaming is not possible for this dataset because data host server doesn't support HTTP range requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)r)rVrArpoprrXrY ValueErrorr@r;rlrrrr) rrrRrrZfile_strr[r\rUrer/r/r0rms: $   rmc Cst|d^}}t|rt|St||d\}}|d^}}tj||d^}}|dd}|drB| |sBt d||j|dd }d d |DS) zExtend `os.listdir` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `list` of `str` r?rSrTrqrrrszDirectory doesn't exist: F)detailcSsg|] }tj|dqS)rs)rBrCr`rstrip)r.rCr/r/r0 *szxlistdir..) rVrArrBlistdirrXrYrZrurtrl) rCrRr[r\rUr]r^rvpathsr/r/r0xlistdirs  rF) recursiverRc st|d^}t|rtj||dSt||d\}}|d^}tj||d^}}|dd}||}t|jt rA|jn|jdfdd |DS) aExtend `glob.glob` function to support remote files. Args: urlpath (`str`): URL path with shell-style wildcard patterns. recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more directories or subdirectories. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `list` of `str` r?)rrSrTrqrrrcs&g|]}dd|gqS)r?rq)rD)r. globbed_pathr,r\r/r0rHs&zxglob..) rVrArglobrXrYrZ isinstancer,r@) rQrrRr[rUr]r^rv globbed_pathsr/rr0xglob-s  rc kst|d^}}t|rtj|fi|EdHdSt||d\}}|d^}}tj||d^}}|dd}|drG| |sGgSt |j t rP|j n|j d} |j|fi|D]\} } } d | d| g|| | fVq^dS)auExtend `os.walk` function to support remote files. Args: urlpath (`str`): URL root path. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Additional keyword arguments forwarded to the underlying filesystem. Yields: `tuple`: 3-tuple (dirpath, dirnames, filenames). r?NrSrTrqrrrs)rVrArrBwalkrXrYrZrurtrr,r@rD) rQrRrr[r\rUr]r^rvr,dirpathdirnames filenamesr/r/r0xwalkKs $rcseZdZdZfddZddeefddZddeefdd Zd d Z e d d dZ e d e fddZ e d e fddZe d e fddZddZdee dfd dfddZde d dfddZfddZZS)!xPathzHExtension of `pathlib.Path` to support both local paths and remote URLs.cs\t}|d^}}t|r|S|dd}td|}||dr)d7}|Sd7}|S)Nr?\rsrqrJrK)super__str__rArr#SINGLE_SLASH_AFTER_PROTOCOL_PATTERNsubrN)selfZpath_strr[r\Z path_as_posix __class__r/r0rjs   z xPath.__str__NrRcCstt||dS)zExtend `pathlib.Path.exists` method to support both local and remote files. Args: download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rS)r_r@)rrRr/r/r0rWts z xPath.existsc cs|}|d^}}t|rt||EdHdS|rA|d}t||d\}}|dd|i}d||g|dd}nd}tjt |||d^}} |t ||} | D]} t |d|j d| g|VqYdS)a]Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. download_config : mainly use token or storage_options to support different platforms and auth types. Yields: [`xPath`] r?NrrSrqrrT) rMrArrrrXrDrYrZrItyper,) rpatternrRZ posix_pathr[r\rQrUr]r^rrr/r/r0rs  (z xPath.globcKs|jd|fi|S)zRglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. Yields: [`xPath`] z**/)r)rrrr/r/r0rglobs z xPath.rglobrgcCst|t|S)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: [`xPath`] )rrPrMrr/r/r0parentsz xPath.parentcCt|ddjS)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r?r)rrMrAnamerr/r/r0rz xPath.namecCr)zStem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r?r)rrMrAstemrr/r/r0rrz xPath.stemcCr)zSuffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` r?r)rrMrAsuffixrr/r/r0rrz xPath.suffixcOstt|g|Ri|S)aExtend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. Args: **args: Arguments passed to :func:`fsspec.open`. **kwargs: Keyword arguments passed to :func:`fsspec.open`. Returns: `io.FileIO`: File-like object. )rmr@)rrrr/r/r0rs z xPath.openrG.cGst|t|g|RS)zExtend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. Args: *p (`tuple` of `str`): Other path components. Returns: [`xPath`] )rrIrMrrGr/r/r0joinpaths zxPath.joinpathcCs ||Sr7)rrr/r/r0 __truediv__s zxPath.__truediv__cs`t|d^}}t|rt|tt|St|dt|t||g|S)Nr?) r@rArrr with_suffixrDrrM)rrr[r\rr/r0rs.zxPath.with_suffixr7)rgr)r<r=r>__doc__rr r rWrrpropertyrr@rrrrrrrr __classcell__r/r/rr0rgs$       rcCs"t|tr t|Sttt|Sr7)rrr@)rCr/r/r0rVs"rVcOTddl}t|dr|j|g|Ri|St|}|jt|d|dg|Ri|SNrrnrbrS)r5hasattrrr@rm)filepath_or_bufferrRrrr5r/r/r0 xgzip_open  "rcOrr)numpyrloadr@rm)rrRrrnpr/r/r0 xnumpy_loadrrcKshddl}t|dr|j|fi|St|}|dddkr&t||d|d<|jt|d|dfi|S)NrrnrZinferrSr)pandasrZread_csvr@rrrmrrRrpdr/r/r0xpandas_read_csvs rc Ksddl}t|dr*z |j|fi|WSty)|jt|fi|YSwt|}z|jt|d|dfi|WStyX|jtt|d|dfi|YSwr)rrZ read_excelrrrnr@rmrr/r/r0xpandas_read_excels"    rcKsLddlm}t|dr|j|fi|St|}|jt|d|dfi|S)Nrrnr)rrR)Zpyarrow.parquetr%rZ read_tabler@rm)rrRrZpqr/r/r0xpyarrow_parquet_read_table%s  rcKsDddlm}t|dr|j|fi|S|jt|d|dfi|Sr)Zscipy.iorrZloadmatrm)rrRrsior/r/r0 xsio_loadmat/s  rcCsXt|dr tj||dSt|d|d}tj||dWdS1s%wYdS)aExtend `xml.etree.ElementTree.parse` function to support remote files. Args: source: File path or file object. parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `xml.etree.ElementTree.Element`: Root element of the given source document. rn)parserrrSN)rETrrm)sourcerrRror/r/r0 xet_parse8s  $rcKsht|drtjjj|fi|St|d|d}tjjj|fi|WdS1s-wYdS)aExtend `xml.dom.minidom.parse` function to support remote files. Args: filename_or_file (`str` or file): File path or file object. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. Returns: :obj:`xml.dom.minidom.Document`: Parsed document. rnrrSN)rr(domZminidomrrm)Zfilename_or_filerRrror/r/r0xxml_dom_minidom_parseJs $rc@s&eZdZdZdefddZddZdS)_IterableFromGeneratorzkUtility class to create an iterable from a generator function, in order to reset the generator when needed. generatorcOs||_||_||_dSr7rrr)rrrrr/r/r0__init___s z_IterableFromGenerator.__init__ccs |j|ji|jEdHdSr7rrr/r/r0__iter__dsz_IterableFromGenerator.__iter__N)r<r=r>rrrrr/r/r/r0r\s rc @seZdZdZeddZeddZedee ddffdd Z e dd e d e e dee ddffd d ZedddZedd e e ddfddZdS)ArchiveIterablezIAn iterable of (path, fileobj) from a TAR archive, used by `iter_archive`ccshtj|dd}|D]&}|j}|sq |durq tj|dr#q ||}||fVg|_ q ~dS)Nzr|*)fileobjrr)__) tarfilerrisregrBrCr`r extractfilemembers)rostreamtarinfo file_pathrr/r/r0 _iter_tarks  zArchiveIterable._iter_tarccs`t|}|D]#}|j}|rq |durq tj|dr#q | |}||fVq dS)Nr ) zipfileZipFileinfolistfilenameis_dirrBrCr`rr)roZzipfmemberrrr/r/r0 _iter_zip|s    zArchiveIterable._iter_ziprgNccs:t|}|dkr||EdHdS||EdHdS)Nr2)rrr)clsrorr/r/r0_iter_from_fileobjs z"ArchiveIterable._iter_from_fileobjrQrRccs~t||d}t|d|dd&}|dkr||EdHn||EdHWddSWddS1s8wYdS)NrSrr)rRrr2)rrmrr)rrQrRrror/r/r0_iter_from_urlpaths "z"ArchiveIterable._iter_from_urlpathcCs ||j|Sr7)r)rr r/r/r0from_bufs zArchiveIterable.from_bufcC||j||Sr7)r)rurlpath_or_bufrRr/r/r0 from_urlpathzArchiveIterable.from_urlpathr7)rgr)r<r=r>r staticmethodrr classmethodr rrr@r r rrr!r/r/r/r0rhs*     rc @sdeZdZdZe d deeeefdee de eddffddZ ed dee ddfdd Z dS) FilesIterablez8An iterable of paths from a list of directories or filesNurlpathsrRrgccst|ts |g}|D]J}t||dr|Vq t||drRt||dD]-\}}}tdd|D|dd<t|dr=q#t|D]}|drIqAt||VqAq#q t |dS)NrScSsg|] }|ds|qS)r )r)r.rLr/r/r0rsz5FilesIterable._iter_from_urlpaths..r ) rlistrirwrsortedrarrIrl)rr&rRrQrrrrr/r/r0_iter_from_urlpathss&      z!FilesIterable._iter_from_urlpathscCrr7)r))rr&rRr/r/r0 from_urlpathsr"zFilesIterable.from_urlpathsr7) r<r=r>rr$rr@r r r r r)r*r/r/r/r0r%s  r%c @seZdZdZdZ    ddeedeedeedeefdd Ze d d Z d d Z dedefddZ ddZ dedefddZddZdeeejfdeefddZdeeeefdeefddZdS)StreamingDownloadManagera Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract data, but they rather return the path or url that could be opened using the `xopen` function which extends the built-in `open` function to stream data from remote files. TN dataset_namedata_dirrR base_pathcCs.||_||_|p tjd|_|pt|_dS)Nr))Z _dataset_name _data_dirrBrCabspath _base_pathr rR)rr,r-rRr.r/r/r0rsz!StreamingDownloadManager.__init__cCs|jSr7)r/rr/r/r0 manual_dirsz#StreamingDownloadManager.manual_dircCst|j|dd}|S)aUNormalize URL(s) of files to stream data from. This is the lazy version of `DownloadManager.download` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') ``` TZ map_tuple)r _downloadr url_or_urlsr/r/r0downloadsz!StreamingDownloadManager.downloadrQrgcCs t|}t|rt|j|}|Sr7)r@rrr1)rrQr/r/r0r4s z"StreamingDownloadManager._downloadcCst|j|dd}|S)aAdd extraction protocol for given url(s) for streaming. This is the lazy version of `DownloadManager.extract` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> extracted_files = dl_manager.extract(downloaded_files) ``` Tr3)r_extract)rr6r&r/r/r0extractsz StreamingDownloadManager.extractcCst|}t||jd}|dd}t|}|dvs|dr'td|d|dur-|S|tvrUtj |dd}d|vrI|d| dn|}|d |d|S|d |S) NrSr?rrrz+Extraction protocol for TAR archives like 'z' is not implemented in streaming mode. Please use `dl_manager.iter_archive` instead. Example usage: url = dl_manager.download(url) tar_archive_iterator = dl_manager.iter_archive(url) for filename, file in tar_archive_iterator: ...r)rqz://::) r@rrRrArrNr!SINGLE_FILE_COMPRESSION_PROTOCOLSrBrCr`rindex)rrQr,rCr*Z inner_filer/r/r0r8s  z!StreamingDownloadManager._extractcCs|||S)a0Prepare given `url_or_urls` for streaming (add extraction protocol). This is the lazy version of `DownloadManager.download_and_extract` for streaming. Is equivalent to: ``` urls = dl_manager.extract(dl_manager.download(url_or_urls)) ``` Args: url_or_urls (`str` or `list` or `dict`): URL(s) to stream from data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. )r9r7r5r/r/r0download_and_extract/sz-StreamingDownloadManager.download_and_extractr cCs$t|dr t|Stj||jdS)aNIterate over files within an archive. Args: urlpath_or_buf (`str` or `io.BufferedReader`): Archive path or archive binary file object. Yields: `tuple[str, io.BufferedReader]`: 2-tuple (path_within_archive, file_object). File object is opened in binary mode. Example: ```py >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> files = dl_manager.iter_archive(archive) ``` rnrS)rrrr!rR)rr r/r/r0 iter_archiveCs  z%StreamingDownloadManager.iter_archiver&cCstj||jdS)aIterate over files. Args: urlpaths (`str` or `list` of `str`): Root paths. Yields: str: File URL path. Example: ```py >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') >>> files = dl_manager.iter_files(files) ``` rS)r%r*rR)rr&r/r/r0 iter_files\sz#StreamingDownloadManager.iter_files)NNNN)r<r=r>rZ is_streamingr r@r rrr2r7r4r9r8r<rrBufferedReaderr rr=r r>r/r/r/r0r+s2   &r+r7)r)NN)lrrrBrErer rZxml.dom.minidomr(rasynciorr itertoolsrpathlibrrtypingrrr r r r r rrZ xml.etreerrrYZaiohttp.client_exceptionsrZhuggingface_hub.utilsr packagingrrrZ filesystemsrZutils.file_utilsrrrrrrZ utils.loggingrZutils.py_utilsrrRr r<r}rrr:compilerbytesfromhexrrmaxr Exceptionr;rIrPr@r_rarcrfboolriintrprwrzrrrrrXrrmrrrrrrVrrrrrrrrrrr%r+r/r/r/r0s   ,                          > /       A