o NZh!@sddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z ddl m Z ddlmZmZmZmZddZdaGd d d e Zed gd Gd ddejZGdddeZdS)N)suppress)cached_propertywraps)parse_qs)AbstractFileSystem)"get_package_version_without_importinfer_storage_options mirror_fromtokenizecstfdd}|S)Nc s^z|i|WSty.}z|js|j^}}t|tr)d|vr)ttj||d}~ww)Nzdoes not exist)OSErrorargs isinstancestrFileNotFoundErrorerrnoENOENT)r kwargs exceptionmessagefuncS/var/www/html/lang_env/lib/python3.10/site-packages/fsspec/implementations/arrow.pywrappers z wrap_exceptions..wrapper)r)rrrrrwrap_exceptionss rcseZdZdZdZfddZeddZeddZ e d d Z d-d d Z ddZ ddZddZeddZeddZeZeddZed.ddZed/dd Zed0d!d"Zed-d#d$Zed%d&Zed'd(Zd1fd)d* Zfd+d,ZZS)2ArrowFSWrapperzwFSSpec-compatible wrapper of pyarrow.fs.FileSystem. Parameters ---------- fs : pyarrow.fs.FileSystem /c s$tda||_tjdi|dS)NZpyarrowr)rPYARROW_VERSIONfssuper__init__)selfrr __class__rrr 3szArrowFSWrapper.__init__cCs|jjSN)r type_namer!rrrprotocol9szArrowFSWrapper.protocolcCsdt|jj|jjS)NZhdfs_)r rhostportr&rrrfsid=szArrowFSWrapper.fsidcCs*t|}|d}|dr|dd}|S)Npathz//)r startswith)clsr+opsrrr_strip_protocolAs   zArrowFSWrapper._strip_protocolFc sJ|}ddlm}fddj||D}|r|Sdd|DS)Nr) FileSelectorcsg|]}|qSr) _make_entry.0entryr&rr Nsz%ArrowFSWrapper.ls..cSsg|]}|dqS)namerr3rrrr6Us)r0 pyarrow.fsr1r get_file_info)r!r+detailrr1entriesrr&rlsJs   zArrowFSWrapper.lscKs$||}|j|g\}||Sr$)r0rr9r2)r!r+rinforrrr=Ws  zArrowFSWrapper.infocCs0||}z||WdStyYdSw)NFT)r0r=rr!r+rrrexists\s   zArrowFSWrapper.existscCslddlm}|j|jurd}n|j|jurd}n|j|jur*ttjt tj|j d}|j |j ||j dS)Nr)FileType directoryfileother)r7sizetypemtime)r8r@rEZ DirectoryFileZNotFoundrrrosstrerrorr+rDrF)r!r=r@kindrrrr2es    zArrowFSWrapper._make_entryc Ks||d}||d}||d[}|dtd}z%||d}t||Wdn1s9wY|j ||Wn"t yht t |j |Wd1sbwYwWddS1stwYdS)Nrrbz.tmp.wb)r0rstrip_opensecretsZ token_hexopenshutil copyfileobjrmove BaseExceptionrr delete_file)r!path1path2rZlstreamZ tmp_fnameZrstreamrrrcp_filexs&   "zArrowFSWrapper.cp_filecKs2||d}||d}|j||dS)Nr)r0rNrrT)r!rWrXrrrrmvszArrowFSWrapper.mvcC||}|j|dSr$)r0rrVr>rrrrm_file zArrowFSWrapper.rm_fileNcCsF||d}||r|r|j|dStd|j|dS)Nrz0Can't delete directories without recursive=False)r0rNisdirr delete_dir ValueErrorrV)r!r+ recursiveZmaxdepthrrrrms  zArrowFSWrapper.rmrKTc Ks|dkr|r |jj}n|jj}n|dkr|jj}n|dkr"|jj}ntd|i}|dks1|s@ttdddkr@d|d<||fi|}t |||||fi|S) NrKrMabz'unsupported mode for Arrow filesystem: .r compression) rZopen_input_fileZopen_input_streamZopen_output_streamZopen_append_streamr`intrsplit ArrowFile) r!r+mode block_sizeseekablermethod_kwargsstreamrrrrOs     zArrowFSWrapper._opencKs4||}|r|j|dddS|jj|dddS)NT)exist_okFra)r0makedirsr create_dir)r!r+Zcreate_parentsrrrrmkdirs zArrowFSWrapper.mkdircCs||}|jj|dddS)NTrq)r0rrs)r!r+rprrrrrs zArrowFSWrapper.makedirscCr[r$)r0rr_r>rrrrmdirr]zArrowFSWrapper.rmdircCs||}|j|jSr$)r0rr9rFr>rrrmodifieds zArrowFSWrapper.modifiedc s&|dv|d<tj|fddd|S)N)Nrrl)startend)rcat_file)r!r+rwrxrr"rrrys zArrowFSWrapper.cat_filec s"d|d<tj||fi|dS)NFrl)rget_file)r!ZrpathZlpathrr"rrrzszArrowFSWrapper.get_file)F)FN)rKNT)T)NN)__name__ __module__ __qualname____doc__Z root_markerr propertyr'rr* classmethodr0r<r=r?r2rrYrZZmv_filer\rbrOrtrrrurvryrz __classcell__rrr"rr(sF                rro) readseektellwritereadablewritablecloserDrlc@s&eZdZdddZddZddZdS) riNcKs.||_||_||_||_||_|_||_dSr$)r+rjrro blocksizerkr)r!rror+rjrkrrrrr s   zArrowFile.__init__cCs|Sr$rr&rrr __enter__szArrowFile.__enter__cGs|Sr$)r)r!r rrr__exit__szArrowFile.__exit__r$)r{r|r}r rrrrrrris  rics>eZdZdZdZ      d fdd Zed d ZZS) HadoopFileSystemz]A wrapper on top of the pyarrow.fs.HadoopFileSystem to connect it's interface with fsspecZhdfsdefaultrNc  s:ddlm}|||||||d} tjdd| i|dS)a; Parameters ---------- host: str Hostname, IP or "default" to try to read from Hadoop config port: int Port to connect on, or default from Hadoop config if 0 user: str or None If given, connect as this username kerb_ticket: str or None If given, use this ticket for authentication replication: int set replication factor of file for write operations. default value is 3. extra_conf: None or dict Passed on to HadoopFileSystem r)r)r(r)user kerb_ticket replication extra_confrNr)r8rrr ) r!r(r)rrrrrrrr"rrr s zHadoopFileSystem.__init__cCst|}i}|ddr|d|d<|ddr|d|d<|ddr*|d|d<|ddrFt|d}|ddrFt|dd|d<|S)Nr(usernamerr)Z url_queryrr)rgetrrg)r+r/outZqueriesrrr_get_kwargs_from_urls$s         z&HadoopFileSystem._get_kwargs_from_urls)rrNNrN) r{r|r}r~r'r staticmethodrrrrr"rrs'r)riorHrPrR contextlibr functoolsrr urllib.parserZ fsspec.specrZ fsspec.utilsrrr r rrrIOBaserirrrrrs(   0