o
    Zh&                     @   s  d dl Z d dlZd dlZd dlZd dlZzd dlmZ d dl	m
Z
 W n ey-   dZY nw zd dlZd dlmZ d dl	mZ W n eyM   d ZZY nw ejjZejjdd Zejjdd Zejjdd	 Zejjd
d Zejjdd Zejjdd Zdd Zdd Zejjejdg ddd Zejjejddgdd Zejjejdddgdd  Zd!d" Z ejj!ejd#d$d%d& Z"dS )'    N)_write_table)alltypes_samplec                  C   sr   t dd} tj| }t }t||ddd |d t	|}|d tj
||d}t| |   d S )N'  sizesnappy2.6)compressionversionr   )metadata)r   paTablefrom_pandasioBytesIOr   seekpqZread_metadataParquetFiletmassert_frame_equalread	to_pandas)dfa_tablebufr   Zfileh r   ^/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/parquet/test_parquet_file.pytest_pass_separate_metadata.   s   



r   c                     s   d\} }t | d}tj|}t }t||| | ddd |d t	|  j
|ks/J  fddt|D }t|}t||  d S )	Nr      r   r   r   Zrow_group_sizer	   r
   r   c                    s   g | ]}  |qS r   Zread_row_group.0ipfr   r   
<listcomp>T   s    z.test_read_single_row_group.<locals>.<listcomp>)r   r   r   r   r   r   r   r   r   r   num_row_groupsrangeconcat_tablesr   r   r   NKr   r   r   
row_groupsresultr   r%   r   test_read_single_row_groupB   s   



r0   c                     s   d\} }t | d}tj|}t }t||| | ddd |d t	|t
|jd d   fdd	t|D }t|}t|  |   fd
d	t|D }t|}t|  |  d S )Nr   r   r   r   r    r      c                    s   g | ]	}j | d qS columnsr!   r"   colsr&   r   r   r'   g   s    zAtest_read_single_row_group_with_column_subset.<locals>.<listcomp>c                    s   g | ]}j |   d qS r2   r!   r"   r5   r   r   r'   m   s    )r   r   r   r   r   r   r   r   r   r   listr4   r)   r*   r   r   r   r+   r   r5   r   -test_read_single_row_group_with_column_subsetY   s    




r8   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}|j
|ks/J |t|}t||  d S )Nr   r   r   r   r    r   )r   r   r   r   r   r   r   r   r   r   r(   read_row_groupsr)   r   r   r   )r,   r-   r   r   r   r&   r/   r   r   r   test_read_multiple_row_groupsr   s   


r:   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}t
|jd d }|jt||d}t|| |  |jt||| d}t|| |  d S )	Nr   r   r   r   r    r   r1   r3   )r   r   r   r   r   r   r   r   r   r   r7   r4   r9   r)   r   r   r   )r,   r-   r   r   r   r&   r6   r/   r   r   r   0test_read_multiple_row_groups_with_column_subset   s   


r;   c                  C   s   d\} }t | d}tj|}t }t||| | ddd |d t	|}|
 dks0J |
|jd d dks>J d S )	Nr   r   r   r   r    r   r   r   )r   r   r   r   r   r   r   r   r   r   Zscan_contentsr4   )r,   r-   r   r   r   r&   r   r   r   test_scan_contents   s   


 r<   c                 C   s   | d }t t| dt| d}tt}t| W d    n1 s(w   Y  |t	r9t
jdkr9d S || d S )N	directoryzCannot open for reading: path 'z' is a directorywin32)osmkdirstrpytestraisesIOErrorr   r   ZerrisinstancePermissionErrorsysplatformmatch)tempdirpathmsgexcr   r   r   0test_parquet_file_pass_directory_instead_of_file   s   rM   c               	   C   s   t jt ddgt ddggddgd} t  }t| | t| }|j	d
 ddgks4J |j	d	
 ddgksBJ d
D ]}tttf |j	| W d    n1 s^w   Y  qDd S )Nr      ZfoobarZintsstrs)namesr      )r1   )r   tablearrayZBufferOutputStreamr   write_tabler   getvaluereaderZread_columnZ	to_pylistrB   rC   
ValueError
IndexError)rT   Zbiofindexr   r   r   test_read_column_invalid_index   s   r]   
batch_size)i,    i  c              	   C   s   d}d}t |d}| d }tj|}t||d|d t|}|jd d |jdd  fD ]>}|j||d}	t	d	|| |}
t
|	|
D ]'\}}t||| }t| |j||d d f jd d |f jd
d qDq.d S )Ni  r_   r   pandas_roundtrip.parquetr   r
   
chunk_size
   )r^   r4   r   TZdrop)r   r   r   r   r   r   r   r4   iter_batchesr)   zipminr   r   r   iloclocreset_index)rI   r^   Z
total_sizerb   r   filenamearrow_tablefile_r4   batchesZbatch_startsbatchstartendr   r   r    test_iter_batches_columns_reader   s(   

 *rr   rb   r_   c           
      C   s   t ddd}| d }tj|}|jjd usJ t||d|d t|}dd }t	||}d	}t
|jD ]:}	t||  ||	g d
 |d7 }t||  jdd||	g jd
d  jdd |d7 }q6d S )Nr   T)r   Zcategoricalr`   r   ra   c                 s   s6    t | jD ]}| jd|gd}|D ]}|V  qqd S )N  )r^   r.   )r)   r(   re   )r[   Z	row_grouprn   ro   r   r   r   get_all_batches   s   z1test_iter_batches_reader.<locals>.get_all_batchesr   rs   rR   rd   )r   r   r   r   ZschemaZpandas_metadatar   r   r   r7   r)   r(   r   r   r   r9   headrj   rh   )
rI   rb   r   rk   rl   rm   rt   rn   Zbatch_nor$   r   r   r   test_iter_batches_reader   s2   



rv   
pre_bufferFTc                 C   sj   d\}}t |d}tj|}t }t|||| ddd |d tj	|| d}|
 j|ks3J d S )Nr   r   r   r   r    r   )rw   )r   r   r   r   r   r   r   r   r   r   r   Znum_rows)rw   r,   r-   r   r   r   r&   r   r   r   test_pre_buffer  s   

rx   c              	   C   s  |  d}tddgddgd}t|| t|d5}t|}|  |jr,J |jr1J W d   n1 s;w   Y  |jrEJ |jrJJ W d   n1 sTw   Y  |js^J |jscJ t|}|  |jrrJ W d   n1 s|w   Y  |jsJ dS )z
    Unopened files should be closed explicitly after use,
    and previously opened files should be left open.
    Applies to read_table, ParquetDataset, and ParquetFile
    zfile.parquetr   rR   )Zcol1Zcol2rbN)	joinpathr   rT   r   rV   openr   r   closed)rI   fnrT   r[   pr   r   r   #test_parquet_file_explicitly_closed  s&   




r   use_uri)TFc           
      C   s   | \}}}|r	|n|f}|ri nt |d}tdtdi}tj|||d tj|i |}| |ks6J |jr;J |	  |jsDJ tj|i |}	|	 |ksUJ |	jrZJ W d    n1 sdw   Y  |	jsnJ d S )N)
filesystemarc   )
dictr   rT   r)   r   rV   r   r   r|   close)
Zs3_example_fsr   Zs3_fsZs3_uriZs3_pathargskwargsrT   Zparquet_filer[   r   r   r   !test_parquet_file_with_filesystem<  s   


r   )#r   r?   rF   rB   Zpyarrowr   Zpyarrow.parquetZparquetr   Zpyarrow.tests.parquet.commonr   ImportErrorZpandaspdZpandas.testingZtestingr   r   markZ
pytestmarkr   r0   r8   r:   r;   r<   rM   r]   Zparametrizerr   rv   rx   r   Zs3r   r   r   r   r   <module>   s\   





+