o
    Zh}                     @   s*  d dl m Z mZmZ d dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Zd dlmZ d dlmZmZmZ d dlmZmZmZmZmZmZmZmZmZmZmZ G dd deZejdd	 Z ejd
d Z!ejdd Z"ejdd Z#ejdd Z$ejdd Z%ejdd Z&ejdd Z'ejdd Z(dZ)ejdd Z*ejdd Z+ejdd  Z,ejd!d" Z-ejej.d	d#d$ej.dd%d$ej.dd&d$ej.dd'ej/j0d(ej.dd)ej/j1d(ej.dd*ej/j*d(ej.dd+d$ej.dd,d$ej.dd-d$ej.dd.d$ej.d d/d$ej.d"d0ej/j0d(gd1d2d3 Z2ejd4d5 Z3ejd6d7 Z4ejd8d9 Z5ejd:d; Z6d<d= Z7d>d? Z8d@dA Z9dBdC Z:ej/j0dDdE Z;dFdG Z<dHdI Z=dJdK Z>dLdM Z?dNdO Z@dPdQ ZAdRdS ZBdTdU ZCdVdW ZDdXdY ZEdZd[ ZFd\d] ZGd^d_ ZHd`da ZIdbdc ZJddde ZKdfdg ZLdhdi ZMdjdk ZNdldm ZOdndo ZPdpdq ZQdrds ZRej/jej/SdtddeRfddueRfdvdejTfdvdwejTfgdxdy ZUdzd{ ZVd|d} ZWej/jej/Sd~ddeRfddueRfdvdejXfdvdwejXfgdd ZYej/jej/SdddeReRfddueReRfdvdejTejXfdvdwejTejXfgdd ZZdd Z[dd Z\dd Z]dd Z^dd Z_ej/j1dd Z`ej/j0dd Zaej/j0dd Zbej/j0dd Zcej/j*dd Zdej/Sddedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfdedfgdd Zeej/Sdddddefgdd Zgej/j0dd Zhej/j1dd Zidd Zjdd Zkdd Zldd Zmdd Zndd Zodd Zpdd ZqddĄ ZrddƄ ZsddȄ Ztej/j0ddʄ Zuej/j0dd̄ Zvej/j0dd΄ Zwej/j0ddЄ Zxdd҄ Zyej/j0ddԄ Zzej/j0ddք Z{dS )    )datetimetimezone	timedeltaN)assert_file_not_found)_filesystem_uriProxyHandler_configure_s3_limited_user)FileTypeFileInfoFileSelector
FileSystemLocalFileSystemSubTreeFileSystem_MockFileSystemFileSystemHandlerPyFileSystemFSSpecHandler
copy_filesc                   @   s   e Zd Zd'ddZdd Zdd Zdd	 Zd
d Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&S )(DummyHandler*   c                 C   s
   || _ d S N)_value)selfvalue r   L/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/test_fs.py__init__(   s   
zDummyHandler.__init__c                 C   s   t |tr| j|jkS tS r   
isinstancer   r   NotImplementedr   otherr   r   r   __eq__+      
zDummyHandler.__eq__c                 C   s   t |tr| j|jkS tS r   r   r    r   r   r   __ne__0   r#   zDummyHandler.__ne__c                 C      dS )Ndummyr   r   r   r   r   get_type_name5      zDummyHandler.get_type_namec                 C   s   |S r   r   r   pathr   r   r   normalize_path8   r)   zDummyHandler.normalize_pathc                 C   s~   g }|D ]8}d|v r| t|tj qd|v r"| t|tj qd|v r0| t|tj qd|v r;| t  qt|S )Nfiledirnotfoundbadtype)appendr
   r	   File	DirectoryNotFoundobjectIOError)r   pathsinfor+   r   r   r   get_file_info;   s   zDummyHandler.get_file_infoc                 C   sZ   |j dkr|jr
g S t|j tdtjddtdtjg}|jr+|tdtjddg7 }|S )Nsomedirsomedir/file1{   )sizesomedir/subdir1somedir/subdir1/file2i  )base_dirallow_not_foundFileNotFoundErrorr
   r	   r2   r3   	recursive)r   selectorinfosr   r   r   get_file_info_selectorK   s   


z#DummyHandler.get_file_info_selectorc                 C   s4   |dkr|du s
J d S |dkr|du sJ d S t )NrC   Tnon-recursiveF)r6   )r   r+   rC   r   r   r   
create_dir[   s
   zDummyHandler.create_dirc                 C      |dksJ d S )N
delete_dirr   r*   r   r   r   rJ   c      zDummyHandler.delete_dirc                 C   s   | dst|dksJ d S )N/delete_dir_contents)strip
ValueError)r   r+   missing_dir_okr   r   r   rM   f   s   
z DummyHandler.delete_dir_contentsc                 C   s   d S r   r   r'   r   r   r   delete_root_dir_contentsk   r)   z%DummyHandler.delete_root_dir_contentsc                 C   rI   )Ndelete_filer   r*   r   r   r   rR   n   rK   zDummyHandler.delete_filec                 C      |dksJ |dksJ d S )N	move_frommove_tor   r   srcdestr   r   r   moveq      zDummyHandler.movec                 C   rS   )Ncopy_file_fromcopy_file_tor   rV   r   r   r   	copy_fileu   rZ   zDummyHandler.copy_filec                 C   *   d|v rt |d|d}t|S )Nr/   z{0}:input_streamutf8rB   formatencodepaZBufferReaderr   r+   datar   r   r   open_input_streamy      
zDummyHandler.open_input_streamc                 C   r^   )Nr/   z{0}:input_filer_   r`   rd   r   r   r   open_input_file   rg   zDummyHandler.open_input_filec                 C      d|v rt |t S Nr/   rB   rc   ZBufferOutputStreamr   r+   metadatar   r   r   open_output_stream      zDummyHandler.open_output_streamc                 C   ri   rj   rk   rl   r   r   r   open_append_stream   ro   zDummyHandler.open_append_streamN)r   )__name__
__module____qualname__r   r"   r$   r(   r,   r9   rF   rH   rJ   rM   rQ   rR   rY   r]   rf   rh   rn   rp   r   r   r   r   r   '   s&    
r   c                    s   t t  fdddddS )Nc                        |    S r   as_posixptempdirr   r   <lambda>       zlocalfs.<locals>.<lambda>Tfspathfnallow_move_dirallow_append_to_filedictr   requestrz   r   ry   r   localfs   s   
r   c                    s"   t ttt  fdddddS )Nc                    rt   r   ru   rw   ry   r   r   r{      r|   zpy_localfs.<locals>.<lambda>Tr}   )r   r   r   r   r   r   ry   r   
py_localfs   s   
r   c                 C   s   t t dd dddS )Nc                 S      | S r   r   rw   r   r   r   r{          zmockfs.<locals>.<lambda>Tr}   )r   r   r   r   r   r   mockfs   s   r   c                 C   s   t ttt dd dddS )Nc                 S   r   r   r   rw   r   r   r   r{      r   zpy_mockfs.<locals>.<lambda>Tr}   )r   r   r   r   r   r   r   r   	py_mockfs   s   r   c                    s   t tdd fdddddS )NTZuse_mmapc                    rt   r   ru   rw   ry   r   r   r{      r|   z#localfs_with_mmap.<locals>.<lambda>r}   r   r   r   ry   r   localfs_with_mmap   s   
r   c                 C   s"   t tt||d dd dddS )Nr~   c                 S   r   r   r   rw   r   r   r   r{      r   z!subtree_localfs.<locals>.<lambda>Tr}   )r   r   str)r   rz   r   r   r   r   subtree_localfs   s   r   c              
   c   s    | j jd ddlm} |d \}}d}|| d| ddtd	d
dd}z|| W n tyK } zt	d| d|  W Y d }~nd }~ww t
||jdddV  || d S )Ngcsr   GcsFileSystem
connectionpyarrow-filesystem/:httpT-   )secondstest-project-id)endpoint_overridescheme	anonymousZretry_time_limit
project_idzCould not create directory in z: Fr}   )configpyarrowrequires
pyarrow.fsr   r   rH   OSErrorpytestskipr   __add__rJ   )r   
gcs_serverr   hostportbucketr~   er   r   r   gcsfs   s2   "r   c           	      c   sx    | j jd ddlm} |d \}}}}d}|||d||dddd	}|| t||jd
d
dV  |	| d S )Ns3r   S3FileSystemr   r   {}:{}r   T)
access_key
secret_keyr   r   allow_bucket_creationallow_bucket_deletionFr}   )
r   r   r   r   r   ra   rH   r   r   rJ   )	r   	s3_serverr   r   r   r   r   r   r~   r   r   r   s3fs   s*   

r   c                 C   s    d}t t||d |jdddS )Nzpyarrow-filesystem/prefix/r~   Fr}   )r   r   r   )r   r   prefixr   r   r   subtree_s3fs  s   r   a  {
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:PutObject",
                "s3:GetObject",
                "s3:ListBucket",
                "s3:PutObjectTagging",
                "s3:DeleteObject",
                "s3:GetObjectVersion"
            ],
            "Resource": [
                "arn:aws:s3:::*"
            ]
        }
    ]
}c                 C   sX   | j jd t std ddlm} |\}}}||||d}t	|dd ddd	S )
NhdfsCannot locate libhdfsr   HadoopFileSystem)r   userc                 S   r   r   r   rw   r   r   r   r{   3  r   zhdfs.<locals>.<lambda>Tr}   )
r   r   r   rc   have_libhdfsr   r   r   r   r   )r   hdfs_connectionr   r   r   r   r~   r   r   r   r   &  s   

r   c                    s4   t d}|d}ttt| fdddddS )Nfsspecr-   c                    rt   r   ru   rw   ry   r   r   r{   ?  r|   z#py_fsspec_localfs.<locals>.<lambda>Tr}   )r   importorskip
filesystemr   r   r   r   rz   r   r~   r   ry   r   py_fsspec_localfs9  s   



r   c                 C   sH   t jddd}|jdkrt d |d}ttt|dd d	d	d
S )Nr   z0.7.5)Z
minversionz0.8.5z,Bug in fsspec 0.8.5 for in-memory filesystemZmemoryc                 S   r   r   r   rw   r   r   r   r{   N  r   z$py_fsspec_memoryfs.<locals>.<lambda>Tr}   )r   r   __version__r   r   r   r   r   r   r   r   r   py_fsspec_memoryfsE  s   



r   c           	      c   sv    t d}|d \}}}}d}|j||td||dd}tt|}|| t||jddd	V  |	| d S )
Nr   r   r   zhttp://{}:{})Zendpoint_url)keysecretZclient_kwargsFTr}   )
r   r   r   r   ra   r   r   rH   r   rJ   )	r   r   r   r   r   r   r   r   r~   r   r   r   py_fsspec_s3fsT  s$   

r   zLocalFileSystem())idzLocalFileSystem(use_mmap=True)z$SubTreeFileSystem(LocalFileSystem())r   )r   Zmarksr   r   z_MockFileSystem()z-PyFileSystem(ProxyHandler(LocalFileSystem()))z-PyFileSystem(ProxyHandler(_MockFileSystem()))z5PyFileSystem(FSSpecHandler(fsspec.LocalFileSystem()))z8PyFileSystem(FSSpecHandler(fsspec.filesystem("memory")))z0PyFileSystem(FSSpecHandler(s3fs.S3FileSystem())))paramsc                 C   s   |  | jS r   )Zgetfixturevalueparamr   r   r   r   filesystem_configk  s   7r   c                 C      | d S Nr~   r   r   r   r   r   r~        r~   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   r   )Nr   r   r   r   r   r   r     r   r   c                 C   sz   t | jtsJ t | jtsJ | jdksJ | jt| j d ks&J | jj}|d us0J |	d t
dks;J d S )Nr   g    eA)r   mtimer   mtime_nsintr   Zapprox	timestamptzinfo	utcoffsetr   )	file_infor   r   r   r   check_mtime  s   
r   c                 C   s    | j d u sJ | jd u sJ d S r   )r   r   r   r   r   r   check_mtime_absent  s   r   c                 C   s"   | j d u rt|  d S t|  d S r   )r   r   r   r   r   r   r   check_mtime_or_absent  s   
r   c                 C   s   | j dkrtjdd d S d S )Npy::fsspec+('s3', 's3a')zNot working with fsspec's s3fsreason)	type_namer   xfailr~   r   r   r   skip_fsspec_s3fs  s   
r   c                 C   s   ddl m} t| t | d \}}}}|ddd||dd}|d	 tjtj	d
d |d W d    n1 s<w   Y  tjtj	dd |
d W d    d S 1 sZw   Y  d S )Nr   r   r   ZlimitedZ
limited123r   r   )r   r   r   r   zexisting-bucket/testzBucket 'new-bucket' not foundmatchz
new-bucketzWould delete bucketzexisting-bucket)r   r   r   _minio_limited_policyra   rH   r   raisesrc   ArrowIOErrorrJ   )r   r   r   r   _r~   r   r   r   +test_s3fs_limited_permissions_create_bucket  s    


"r   c                  C   s,  t dtj} td}|jdksJ |jdksJ |jtj	ks!J |j
d u s(J t| tdtjddd}|jdks<J |jdksCJ |jtjksKJ |j
dksRJ |jd	ksYJ t| td
tj| d}|jd
kslJ |jd
kssJ |jtjks{J |j
d u sJ |j| ksJ |jdksJ t| d S )NR]foo/barbarzfoo/baz.txtr<   g  `A)typer=   r   zbaz.txtl    t<|+. foo)r   r   l    4 |+. )r   fromtimestampr   utcr
   r+   	base_namer   r	   Unknownr=   r   r2   r   r   r3   r   )dtr8   r   r   r   test_file_info_constructor  s0   
r   c                   C   s6   t t t  W d    d S 1 sw   Y  d S r   )r   r   	TypeErrorr   r   r   r   r   'test_cannot_instantiate_base_filesystem  s   "r   c                  C   s   t  } t  }t }| | sJ | |sJ tt | d W d    n1 s,w   Y  | |   kr<|ks?J  J | dksEJ ||ksKJ |t ksRJ td| td| ks^J td| td|ksjJ td| td| ksvJ d S )Nstring   /basez/other)r   r   equalsr   r   r   r   )Zfs0fs1fs2r   r   r   test_filesystem_equals  s   r   c                 C   sH   t jtdd | d  W d    n1 sw   Y  | d us"J d S )Nzgot NoneTyper   )r   r   r   r   r   r   r   r   test_filesystem_equals_none!  s   r   c                  C   s|   t  } td| }|jdksJ |j| ksJ t|dsJ tdt  }|jdks,J |j| ks3J t|ds<J d S )Nr   z/base/zHSubTreeFileSystem(base_path=/base/, base_fs=<pyarrow._fs.LocalFileSystemz/another/base/zPSubTreeFileSystem(base_path=/another/base/, base_fs=<pyarrow._fs.LocalFileSystem)r   r   	base_pathZbase_fsrepr
startswith)r   Zsubfsr   r   r   test_subtree_filesystem(  s   
r  c                 C   sT   | j dd dkrtjdd || }||}t|ts!J || s(J d S )N::mock"MockFileSystem is not serializabler   )	r   splitr   r   dumpsloadsr   r   r   )r~   pickle_module
serializedrestoredr   r   r   test_filesystem_pickling8  s   

r  c                 C   s  | j dd dkrtjdd t|  |d}|d}|d}| | | | W d    n1 s5w   Y  | |}|d	 W d    n1 sOw   Y  ||	| }|
|||g\}}	}
|jtjksoJ |	jtjkswJ |
jtjksJ d S )
Nr  r  r  r  r   	a/aa/aaa/a/bbc.txt   test)r   r  r   r   r   rH   rn   writer
  r	  r9   r   r	   r3   r2   )r~   r   r  aaabbcfpr  aaa_infobb_infoc_infor   r   r   ,test_filesystem_is_functional_after_picklingB  s$   
r  c                  C   s,   t  } | jdks
J t } | jdksJ d S )Nlocalr  )r   r   r   r   r   r   r   test_type_nameX  s   r  c                 C   s   |  ddks	J d S )Nr   )r,   r   r   r   r   test_normalize_path_  s   r  c              	   C   sp   G dd d}dd| t  i g dd t g}|D ]}tt | | W d    n1 s0w   Y  qd S )Nc                   @   s   e Zd ZdS )z-test_non_path_like_input_raises.<locals>.PathN)rq   rr   rs   r   r   r   r   Pathf  s    r     g?c                   S   r%   )Nr   r   r   r   r   r   r{   i  r   z1test_non_path_like_input_raises.<locals>.<lambda>)tuplepathlibr  r   r   r   rH   )r~   r  Zinvalid_pathsr+   r   r   r   test_non_path_like_input_raisese  s   r#  c                 C   s  |d}|d}|d}|d}|  | | | W d    n1 s%w   Y  | |}|d W d    n1 s?w   Y  | ||||g\}}}	}
|j|ksXJ dt|v s`J |jdksgJ | jdkru|jt	j
kstJ n|jt	jks}J d	t|v sJ |jd u sJ t| |jt|ksJ |jd
ksJ |jdksJ |jt	jksJ dt|v sJ |jdksJ | jdvrt| |	jt|ksJ |	jdksJ |	jdksJ |	jt	jksJ dt|	v sJ |	jdksJ | jdvrt|	 |
jt|ksJ |
jdksJ |
jdksJ |
jt	j
ks!J |
jd u s)J |
jd u s1J dt|
v s:J t|
 | |}|j|jksLJ |j|jksUJ d S )Nr  r  r  zzzr  r   r   zFileType.Directoryr  zFileType.Filer   )zpy::fsspec+memoryr   txtr   zFileType.NotFound)rH   rn   r  r9   r+   r  	extensionr   r   r	   r4   r3   r=   r   r   r   r2   r   r   r   )r~   r   r  r  r  r$  r  r  r  r  Zzzz_infoZ	aaa_info2r   r   r   test_get_file_infop  s`   




r(  c              	   C   s  |d}|d}|d}|d}|d}|d}z|  | | | W d    n1 s.w   Y  | | W d    n1 sCw   Y  |  | | | W d    n1 s]w   Y  |  | t|ddd	}|j|ksuJ | |}	| jd
krt|	dk nt|	dksJ |	D ]D}
|
j|s|
j|s|
j|r|
j	t
jksJ n#|
jd|s|
jd|r|
j	t
jksJ ntd|
jt|
 qt|dd}| |}	| jd
krt|	dksJ nt|	dksJ W | | d S W | | d S | | w )Nzselector-dir/zselector-dir/test_file_azselector-dir/test_file_bzselector-dir/test_dir_az#selector-dir/test_dir_a/test_file_czselector-dir/test_dir_bFT)rA   rC   r   r      rL   zunexpected path {}rC      )rH   rn   r   r@   r9   r   lenr+   endswithr   r	   r2   rstripr3   rO   ra   r   rJ   )r~   r   r@   Zfile_aZfile_bZdir_aZfile_cZdir_brD   rE   r8   r   r   r    test_get_file_info_with_selector  s\   








r/  c                 C   s~   t |  |d}ttj | | W d    n1 sw   Y  | | | | |d}| j|dd | | d S )Nztest-directory/zdeeply/nested/test-directory/Tr*  )r   r   r   rc   r   rJ   rH   )r~   r   dr   r   r   test_create_dir  s   

r1  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  ttj | | W d    d S 1 sHw   Y  d S )N
directory/directory/nested/)r   rH   rJ   r   r   rc   r   r~   r   r0  ndr   r   r   test_delete_dir  s   

"r6  c                 C   s   t |  |d}|d}| | | | | | | |}|jtjks(J |d}|d}|d}| | | | | |}|d W d    n1 sSw   Y  | | | |}|jtjksjJ d S )Nr2  r3  Z
directory2zdirectory2/nestedzdirectory2/nested/target-file   data)	r   rH   rJ   r9   r   r	   r4   rn   r  )r~   r   r0  r5  dir_infofsr   r   r   $test_delete_dir_with_explicit_subdir  s&   







r;  c                 C   s   t |  |d}|d}| | | | ttj | | W d    n1 s,w   Y  | j|dd ttj | | W d    n1 sNw   Y  | | ttj | | W d    d S 1 sow   Y  d S )Nr2  r3  T)rP   )r   rH   rM   r   r   rc   r   rJ   r4  r   r   r   test_delete_dir_contents"  s    


"r<  c                 C   s2  | d }| d }|d}|d}| | ttj |d W d    n1 s+w   Y  ttj |d W d    n1 sFw   Y  ttj |d W d    n1 saw   Y  |jddd	 |jddd	 |jddd	 ttj || W d    d S 1 sw   Y  d S )
Nr~   r   r2  r3  r%  rL   //TZaccept_root_dir)rH   r   r   rc   ArrowInvalidrM   r   rJ   )r   r~   r   r0  r5  r   r   r   _check_root_dir_contents4  s(   
"r@  c                 C   s   t |  t | d S r   )r@  )r   r   r   r   r   test_delete_root_dir_contentsJ  s   rA  c                 C   s^   |d}|d}|  | W d    n1 sw   Y  | || | | | | d S )Nztest-copy-source-fileztest-copy-target-file)rn   r]   rR   r~   r   r:  tr   r   r   test_copy_fileO  s   
rD  c                 C   s   |d}|d}|  | |r7| || ttj | | W d    n1 s+w   Y  | | d S ttj | || W d    d S 1 sOw   Y  d S )Nzsource-dir/ztarget-dir/)rH   rY   r   r   rc   r   rJ   )r~   r   r   r:  rC  r   r   r   test_move_directory[  s   
"rE  c                 C   s   t |  |d}|d}| | W d    n1 sw   Y  | || ttj | | W d    n1 s=w   Y  | | d S )Nztest-move-source-fileztest-move-target-file)r   rn   rY   r   r   rc   r   rR   rB  r   r   r   test_move_filel  s   rF  c                 C   s   |d}|  | W d    n1 sw   Y  | | ttj | | W d    n1 s4w   Y  |d}| | |d}|  |}|d W d    n1 s[w   Y  | | d S )Nztest-delete-target-fileztest-delete-nestedztest-delete-nested/target-filer7  )	rn   rR   r   r   rc   r   rH   r  rJ   )r~   r   rx   r0  r9  r:  r   r   r   test_delete_file}  s   

rG  c                 C   r   r   r   )vr   r   r   identity  r)   rI  )compressionbuffer_size
compressor@   gzip   c           	      C   s   |d}dd }|  |}||| W d    n1 sw   Y  | |||}| }W d    n1 s:w   Y  ||ksEJ d S )Nzopen-input-streams   some data for reading
   )rn   r  rf   read)	r~   r   rJ  rK  rL  rx   re   r:  resultr   r   r   test_open_input_stream  s   
rS  c                 C   s   |d}dd }|  |}|| W d    n1 sw   Y  tdd }| |}| }W d    n1 s<w   Y  ||ksGJ | |}|| | }W d    n1 s`w   Y  |||d  ksoJ d S )Nzopen-input-file	   some data   rP  )rn   r  r,  rh   rQ  seek)r~   r   rx   re   r:  Z	read_fromr9  rR  r   r   r   test_open_input_file  s   


rW  c                 C   sB   |d}t t | | W d    d S 1 sw   Y  d S )Nzopen-input-stream-not-found)r   r   rB   rf   )r~   r   rx   r   r   r    test_open_input_stream_not_found  s   "rX  )rJ  rK  decompressorc                 C   s   |d}dd }|  |||}|| W d    n1 sw   Y  | |||}|t||ks7J W d    d S 1 sBw   Y  d S )Nzopen-output-streams   some data for writingrU  )rn   r  rf   rQ  r,  )r~   r   rJ  rK  rY  rx   re   r9  r   r   r   test_open_output_stream  s   "rZ  )rJ  rK  rL  rY  c                 C   s  |d}|d}|  |}	|	| W d    n1 sw   Y  |rf| j|||d}
|
d W d    n1 s<w   Y  | |}
|
 }W d    n1 sUw   Y  ||}|dksdJ d S ttj | j|||d W d    d S 1 sw   Y  d S )Nzopen-append-streams   already existing)rJ  rK  s   
newly addeds   already existing
newly added)	rn   r  rp   rf   rQ  r   r   rc   ZArrowNotImplementedError)r~   r   rJ  rK  rL  rY  r   rx   initialr:  r9  rR  r   r   r   test_open_append_stream  s,   
"r\  c           	      C   s   |d}ddi}d}| j ||d}|| W d    n1 s!w   Y  | |}| |ks4J | }W d    n1 sBw   Y  | jdv sQd| jv rf| D ]\}}|| | kscJ qUd S |i kslJ d S )Nzopen-output-stream-metadataContent-Typezx-pyarrow/testrT  )rm   )r   r   r  )rn   r  rf   rQ  rm   r   itemsrb   )	r~   r   rx   rm   re   r9  Zgot_metadatakrH  r   r   r    test_open_output_stream_metadata  s   
r`  c                   C   sD   t dd tt t dd W d    d S 1 sw   Y  d S )NFr   )xxx)r   r   r   r   r   r   r   r   test_localfs_options  s   
"rb  c                 C   sf  | d }t   |d W d    n1 sw   Y  t   |d W d    n1 s/w   Y  t   |jddd W d    n1 sIw   Y  t   |d W d    n1 saw   Y  t   |d W d    n1 syw   Y  t   |dd W d    n1 sw   Y  t   |dd W d    d S 1 sw   Y  d S )Nr~   z/non/existent/filez/non/existent/dirFr*  z/non/existentz/xxx)r   rf   rn   rH   rJ   rR   rY   r]   )r   r~   r   r   r   test_localfs_errors  s,   "rc  c                 C   s   | d }t t}|j}|| | g\}}|j| jks#J |j	| j
ks-J t| |j	| j
ks;J t| d S r   )r"  r  __file__parentr9   rv   r=   statst_sizer   st_mtime_nsr   )r   r~   	file_pathZdir_pathr   r8  r   r   r   test_localfs_file_info+  s   


rj  c                 C   s`   t dtj}t|}|d W d    n1 sw   Y  |dg\}|j|ks.J d S )Nr   r   )r   r   r   r   r   rn   r9   r   )r   r   r~   r8   r   r   r   test_mockfs_mtime_roundtrip9  s   rk  c              	   C   s  ddl m} t }|dd|ddddd	}t||sJ |jdks#J |jdks*J | | ||ks6J | }t||s@J | | ||ksLJ |d
d}t||sXJ | | ||ksdJ |dddd}t||ssJ | | ||ksJ t	
t |dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |d
dd W d    n1 sw   Y  t	
t |t d W d    d S 1 sw   Y  d S )Nr   r   abczservice_account@apachezus-west2httpslocalhost:8999r   )access_tokentarget_service_accountcredential_token_expirationdefault_bucket_locationr   r   r   Tr   authenticated-read
text/plainZACLr]  default_metadataaccess)ro  r   )r   ro  Zacct)r   rp  )rq  )r   r   r   nowr   rr  r   r
  r	  r   r   rO   )r  r   r   r~   r   r   r   test_gcs_optionsC  sH   

"r{  c                 C   s\  ddl m}m}m}m} |ddddddd	}t||sJ |jdks$J | | ||ks0J |d
dddd}t||s?J | | ||ksKJ ||ddd}t||sZJ ||ddd}t||siJ |d
d}t||suJ | | ||ksJ ||ksJ |dd}t||sJ | | ||ksJ |dd}t||sJ | | ||ksJ |ddddd}t||sJ | | ||ksJ ||ksJ |ddd}t||sJ | | ||ksJ |ddd}t||sJ | | ||ksJ |ddd}t||sJ | | ||ks'J ||ks.J t	
t |dd W d    n	1 sDw   Y  t	
t |dd W d    n	1 s_w   Y  t	
t |ddd  W d    n	1 s{w   Y  t	
t |ddd! W d    n	1 sw   Y  t	
t |ddd"d# W d    n	1 sw   Y  t	
t |dddd$ W d    n	1 sw   Y  t	
t |d"dd% W d    n	1 sw   Y  t	
t |d&d'gd( W d    n	1 s
w   Y  t	
t || d W d    d S 1 s'w   Y  d S ))Nr   )AwsDefaultS3RetryStrategyAwsStandardS3RetryStrategyr   S3RetryStrategyry  r   token	us-east-2rm  rn  )r   r   session_tokenregionr   r   Zrolesessionr   d   )role_arnZsession_nameZexternal_idZload_frequencyr)  )Zmax_attempts)Zretry_strategy)r  Trs  )background_writesrt  ru  rv  )r  rx  )r   r   g      ?g      ?)request_timeoutconnect_timeout)r   )r   )r   r  )r   r  Zarn)r   r   r  )r   r   r   )r  r   r   r   rw  )r   r|  r}  r   r~  r   r  r
  r	  r   r   rO   )r  r|  r}  r   r~  r~   r   r   r   r   test_s3_optionsi  s   


$r  c           
      C   s  ddl m} dddd}d}ddd	d}d
}||d}t||s"J ||||ks.J ||d}t||s:J ||||ksFJ ||d}t||sRJ ||||ks^J ||d}t||sjJ ||||ksvJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks)J ||||	ks6J |||	|ksCJ ||d}||d}	||	ksTJ ||||	ksaJ |||	|ksnJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ksJ ||||	ksJ |||	|ksJ ||d}||d}	||	ks J ||||	ksJ |||	|ksJ ||d}| }	||	ks)J ||||	ks6J |||	|ksCJ ||d}| }	||	ksRJ ||||	ks_J |||	|kslJ ||d}| }	||	ks{J ||||	ksJ |||	|ksJ ||d}| }	||	ksJ ||||	ksJ |||	|ksJ tt |dd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 sw   Y  tt |dddd W d    n	1 s.w   Y  tt	j
 |dd W d    n	1 sJw   Y  tt	j
 |ddddd W d    d S 1 skw   Y  d S )Nr   r   r   	localhosti'#  )r   r   r   zhttp://localhost:8999rm  i  zhttps://localhost:8080)Zproxy_options)r   r  #  r  )r   r   )r   r   )r   r   zhttpsB://localhost:9000ZhttpA)r   r   r   r
  r	  r   r   r   KeyErrorrc   r?  )
Zmonkeypatchr  r   Zproxy_opts_1_dictZproxy_opts_1_strZproxy_opts_2_dictZproxy_opts_2_strr~   r   r   r   r   r   test_s3_proxy_options  s   



























$r  c                  C   s   ddl m}  | ddd}d}tjt|d}|d W d    n1 s%w   Y  d	t|jv r3d S | d
dd}|d d S )Nr   r   z
eu-north-1T)r  r   a  When getting information for bucket 'voltrondata-labs-datasets': AWS Error UNKNOWN \(HTTP status 301\) during HeadBucket operation: No response body. Looks like the configured region is 'eu-north-1' while the bucket is located in 'us-east-2'.|NETWORK_CONNECTIONr   voltrondata-labs-datasetsZNETWORK_CONNECTIONr  )r   r   r   r   r   r9   r   r   )r   r~   msgexcr   r   r   test_s3fs_wrong_regionJ  s   r  c                 C   s  ddl m} t std | \}}}d}d}d}d}	|||d|||d	}
||	||d|||}||	||d
|||}||	||d
|d ||}|||}|d||}|||dd}|||ddd}|||dt	dd}|||ddd}|||ddddid}|
|ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ ||ksJ t
t |  W d    n1 sw   Y  t
t |d W d    n1 sw   Y  |
||||||||||fD ]}||||ks J q| \}}}||||d}|tds8J |d|||}|tdsLJ d S )Nr   r   r      i   i   zIhdfs://{}:{}/?user={}&replication={}&buffer_size={}&default_block_size={}Zlibhdfs)r   replicationrK  default_block_sizemer   zhdfs://{}:{}Z	localuser)r   
cache_path)r   kerb_ticketZcache_path2Z
hdfs_tokenabcd)r   r  Z
extra_confr+  rL   zhdfs://{}:{}/?user={})r   r   rc   r   r   r   from_urira   r"  r  r   r   r
  r	  r9   r   )r   r  r   r   r   r   r  rK  r  uriZhdfs1Zhdfs2Zhdfs3Zhdfs4Zhdfs5Zhdfs6Zhdfs7Zhdfs8Zhdfs9Zhdfs10Zhdfs11r~   r   r   r   r   test_hdfs_optionsb  s   



r  )r  expected_klassexpected_pathzmock:r%  zmock:foo/barr   zmock:/foo/barzmock:///foo/barzmock:///some%20path/%C3%A9u   some path/ézfile:/rL   zfile:///zfile:/foo/barz/foo/barzfile:///foo/barzfile:///some%20path/%C3%A9u   /some path/éu   /some path/%20éc                 C   s,   t | \}}t||sJ ||ksJ d S r   )r   r  r   )r  r  r  r~   r+   r   r   r   test_filesystem_from_uri  s   r  r+   c                 C   sB   t | }t|\}} t|tsJ | |   ksJ d S r   )	r"  r  r   r  r   r   resolveabsoluterv   )r+   rx   r~   r   r   r    test_filesystem_from_path_object  s   
r  c           
      C   s   ddl m} | d \}}}}d||||}t|\}}t||s$J |dks*J || ||g\}	|	j|ks=J |	j	t
jksEJ d S )Nr   r   r   zZs3://{}:{}@mybucket/foo/bar?scheme=http&endpoint_override={}:{}&allow_bucket_creation=Truemybucket/foo/bar)r   r   ra   r   r  r   rH   r9   r+   r   r	   r3   )
r   r   r   r   r   r   r  r~   r+   r8   r   r   r   test_filesystem_from_uri_s3  s   
r  c                 C   s   ddl m} | d \}}dd| d| d d }t|\}}t||s'J |d	ks-J || ||g\}|j|ks@J |jt	j
ksHJ d S )
Nr   r   r   zgs://anonymous@z/mybucket/foo/bar?scheme=http&endpoint_override=r   &z0retry_limit_seconds=5&project_id=test-project-idr  )r   r   r   r  r   rH   r9   r+   r   r	   r3   )r   r   r   r   r  r~   r+   r8   r   r   r   test_filesystem_from_uri_gcs  s   
r  c                  C   sp   t  } t| }t|tsJ |jdksJ |j| u sJ tt td  W d    d S 1 s1w   Y  d S Nz	py::dummy)r   r   r   r   handlerr   r   r   )r  r~   r   r   r   test_py_filesystem  s   
"r  c                  C   s   t d} t d}t d}t| }t| }t|}t|}||us"J ||us(J ||us.J ||ks4J ||ks:J ||ks@J |t ksGJ |t ksNJ d S )Nr   r  )r   r   r   r5   )Zhandler1Zhandler2Zhandler3r   r   Zfs3Zfs4r   r   r   test_py_filesystem_equality   s   r  c                 C   s\   t  }t|}| |}| |}t|tsJ ||ksJ |j|ks%J |jdks,J d S r  )r   r   r	  r
  r   r   r  r   )r  r  r~   r  r  r   r   r   test_py_filesystem_pickling  s   

r  c                  C   s   t  } t| }t|tsJ t| }d } | d usJ d }| d u s%J t  } t| }t| }d } | |ju s;J | d usBJ d }| d u sKJ d S r   )r   r   r   weakrefrefr  )r  r~   wrr   r   r   test_py_filesystem_lifetime   s    

r  c                  C   s  t  } t| }|dg\}|jdksJ |jtjksJ |dg\}|jdks+J |jtjks3J |dg\}|jdksAJ |jtjksIJ t	
t |dg W d    n1 s_w   Y  t	
t |dg W d    d S 1 s{w   Y  d S )Nzsome/dirz	some/filer/   r0   ra  )r   r   r9   r+   r   r	   r3   r2   r4   r   r   r   r6   )r  r~   r8   r   r   r    test_py_filesystem_get_file_info5  s"   "r  c                  C   s^  t  } t| }tdd}||}t|dksJ |d jdks"J |d jtjks,J |d j	dks5J |d jdks>J |d jtj
ksHJ |d j	d u sQJ tdd	d
}||}t|dksdJ |d jdksmJ |d jdksvJ |d jdksJ tdd}tt || W d    n1 sw   Y  tdd	d}||g ksJ d S )Nr:   )r@   r  r   r;   r<   r   r>   T)r@   rC   r+  r?   r/   )r@   rA   )r   r   r   r9   r,  r+   r   r	   r2   r=   r3   r   r   rB   )r  r~   rD   rE   r   r   r   )test_py_filesystem_get_file_info_selectorL  s.   



r  c               	   C   s   t  } t| }|jddd |jddd tt |d W d    n1 s*w   Y  |d |d d	D ]#}tt || W d    n1 sRw   Y  |j|dd
 q;|	d |
dd |dd d S )NrC   Tr*  rG   FZfoobarrJ   rM   )r%  rL   r=  r>  rR   rT   rU   r[   r\   )r   r   rH   r   r   r6   rJ   rM   rO   rR   rY   r]   )r  r~   r+   r   r   r   test_py_filesystem_opsi  s"   


r  c                  C   ~   t t } | d}| dksJ W d    n1 sw   Y  tt | d W d    d S 1 s8w   Y  d S )Nsomefiles   somefile:input_streamr/   )r   r   rf   rQ  r   r   rB   r~   r9  r   r   r   test_py_open_input_stream}     
"r  c                  C   r  )Nr  s   somefile:input_filer/   )r   r   rh   rQ  r   r   rB   r  r   r   r   test_py_open_input_file  r  r  c                  C   D   t t } | d}|d W d    d S 1 sw   Y  d S Nr  r7  )r   r   rn   r  r  r   r   r   test_py_open_output_stream     
"r  c                  C   r  r  )r   r   rp   r  r  r   r   r   test_py_open_append_stream  r  r  c                  C   s   ddl m}  tjdpd}| dd}|j|ksJ | ddd}|td	}t|dks/J d
}|	|#}|
 }d|v sAJ |d dksIJ |d dksQJ W d    d S 1 s\w   Y  d S )Nr   r   ZPYARROW_TEST_S3_REGIONz	us-east-1Trs  r  )r   r  z"voltrondata-labs-datasets/nyc-taxizCvoltrondata-labs-datasets/nyc-taxi/year=2019/month=6/part-0.parquetr]  zLast-Modifieds   2022-07-12T23:32:00ZETags%   "4c6a76826a695c6ac61592bc30cda3df-16")r   r   osenvirongetr  r9   r   r,  rf   rm   )r   Zdefault_regionr~   entriesr   r9  mdr   r   r   test_s3_real_aws  s$   
"r  c                  C   s   t d\} }| jdksJ | |}d|dv sJ W d    n1 s'w   Y  t d\} }| jdks:J tjtdd t d	 W d    n1 sQw   Y  t d
\} }| jdksdJ d S )Nzs3://mf-nwp-models/README.txt	eu-west-1s&   Meteo-France Atmospheric models on AWS2   z.s3://mf-nwp-models/README.txt?region=us-east-2r  zBucket '.*' not foundr   zs3://x-arrow-nonexistent-bucketz0s3://x-arrow-nonexistent-bucket?region=us-east-3z	us-east-3)r   r  r  rf   rQ  r   r   r6   )r~   r+   r9  r   r   r   !test_s3_real_aws_region_selection  s    r  c                  C   s   ddl m}  | ddksJ | ddksJ tjtdd | d	 W d    n1 s,w   Y  tjtdd | d
 W d    d S 1 sHw   Y  d S )Nr   resolve_s3_regionr  r  zmf-nwp-modelsr  zNot a valid bucket namer   r   z	s3:bucket)r   r  r   r   rO   r  r   r   r   test_resolve_s3_region  s   

"r  c              
   C   s0  |d }|d }|d}| |}|d W d    n1 s!w   Y  | \}}}	}
d|	 d|
 d| d| d| 
}t|d	 }t|| t }||}| dksYJ W d    n1 scw   Y  t|d
 }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t|}t|| ||}| dksJ W d    n1 sw   Y  t|d }t|||d ||}| dksJ W d    n1 sw   Y  t|d }t||ddd ||}| dksJ W d    d S 1 sw   Y  d S )Nr~   r   r  r  zs3://r   @z?scheme=http&endpoint_override=zc_copied1.txtzc_copied2.txt)source_filesystemzc_copied3.txtzc_copied4.txt)destination_filesystemzc_copied5.txtr   F
chunk_sizeZuse_threads)rn   r  r   r   r   rf   rQ  r   )Zs3_connectionr   rz   r~   r   r+   r9  r   r   r   r   
source_uriZlocal_path1r   Zlocal_path2Zlocal_path3destination_uriZlocal_path4Zlocal_path5r   r   r   test_copy_files  sP   

$r  c                    s|  t   | d }|   t|d }|d W d    n1 s$w   Y   t|d }|d W d    n1 sBw   Y   fdd}| d }|  tt|t| || | d	 }|  tt|t|  d
 || | d }|  tt|}tt|}t|| || | d }	|	  t||	 ||	 | d }
|
  t||
ddd ||
 d S )Nsourcefile1   test1file2   test2c                    s     t| d }| dksJ W d    n1 sw   Y    t| d }| dks3J W d    d S 1 s>w   Y  d S )Nr  r  r  r  )rf   r   rQ  )destination_dirr9  r   r   r   check_copied_files  s   "z5test_copy_files_directory.<locals>.check_copied_filesZdestination1Zdestination2)r  r  Zdestination3Zdestination4Zdestination5r   Fr  )r   mkdirrn   r   r  r   r   )rz   
source_dirr9  r  Zdestination_dir1Zdestination_dir2Zdestination_dir3r  r  Zdestination_dir4Zdestination_dir5r   r  r   test_copy_files_directory  sD   

r  c                  C      d} t tjd| g d S )Na  if 1:
        import pytest
        from pyarrow.fs import (FileSystem, S3FileSystem,
                                ensure_s3_initialized, finalize_s3)

        fs, path = FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        assert fs.region == 'eu-west-1'
        f = fs.open_input_stream(path)
        f.read(50)

        finalize_s3()

        with pytest.raises(ValueError, match="S3 .* finalized"):
            f.read(50)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            fs.open_input_stream(path)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            S3FileSystem(anonymous=True)
        with pytest.raises(ValueError, match="S3 .* finalized"):
            FileSystem.from_uri('s3://mf-nwp-models/README.txt')
        -c
subprocess
check_callsys
executablecoder   r   r   test_s3_finalizeA  s   r  c                  C   r  )Na  if 1:
        import pytest
        from pyarrow.fs import resolve_s3_region, ensure_s3_initialized, finalize_s3

        resolve_s3_region('mf-nwp-models')

        finalize_s3()

        # Testing both cached and uncached accesses
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('mf-nwp-models')
        with pytest.raises(ValueError, match="S3 .* finalized"):
            resolve_s3_region('voltrondata-labs-datasets')
        r  r  r  r   r   r    test_s3_finalize_region_resolver]  s   r  )|r   r   r   rN  r  r"  r  r  r   r  r   rc   Zpyarrow.tests.test_ior   Zpyarrow.tests.utilr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   Zfixturer   r   r   r   r   r   r   r   r   r   r   r   r   r   r   markr   r   r   r~   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r#  r(  r/  r1  r6  r;  r<  r@  rA  rD  rE  rF  rG  rI  ZparametrizecompressrS  rW  rX  
decompressrZ  r\  r`  rb  rc  rj  rk  r{  r  r  r  r  r  rd  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r   <module>   s  4i
	
	
	
	
	
	








6





;8 

	

	

	

%
S
 

G



		



44
