o
    Zh1                     @   s  d dl Z d dlZd dlmZ d dlmZmZ zd dlmZ	 d dl
mZmZmZ W n ey3   dZ	Y nw zd dlZd dlmZ W n eyM   d ZZY nw e jjZe jjdd Zdd Zd	d
 Ze jjdd Ze jjdd Ze jje jdde e gdd Ze jje jjdd Ze jje jdde e gdd Z e jje jj!dd Z"e jje jj!dd Z#e jje jj!dd Z$e jjdd Z%e jjdd Z&d d! Z'dS )"    N)fs)
FileSystemLocalFileSystem)_read_table_test_dataframe_range_integersc           
      C   s   t d}d|d< tjj|dd}t }tj||jdd}g }tdD ]}||d< tjj|dd}|	| |
|  q#|  | }tt|}tj|d	d
}	t| |	 d S Nd   r   	unique_idFZpreserve_index2.6version
   TZignore_index)r   paTablefrom_pandasBufferOutputStreampqParquetWriterschemarangewrite_tableappendcopyclosegetvaluer   BufferReaderpdconcattmassert_frame_equal	to_pandas
tempdirdfarrow_tableoutwriterframesibufresultexpected r/   `/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/parquet/test_parquet_writer.py#test_parquet_incremental_file_build-   s    
r1   c              	   C   s   t dt  t dt  g}t |}t dgt dgg}t j|ddg}| d }tj	||dddd	+}t
t || W d    n1 sNw   Y  W d    d S W d    d S 1 sfw   Y  d S )
NZPOSZdesc   Zblazsimple_validate_schema.parquetr   ZsnappyZspark)r   compressionZflavor)r   fieldZuint32stringr   arrayr   from_arraysr   r   pytestraises
ValueErrorr   )r%   Zsimple_fieldsZsimple_schemaZsimple_from_arrayZsimple_tablepathwr/   r/   r0    test_validate_schema_write_tableH   s$   
"r=   c                 C   s   t t ttdt g}td | W d    n1 s"w   Y  t t t| d d  W d    d S 1 s@w   Y  d S )NxZ	some_path)	r8   r9   	TypeErrorr   r   r4   Zint32r   r   )r%   Zsome_schemar/   r/   r0   test_parquet_invalid_writer^   s   "r@   c           
      C   s   t d}d|d< tjj|dd}t }tj||jdd)}g }tdD ]}||d< tjj|dd}|	| |
|  q$W d    n1 sIw   Y  | }tt|}tj|d	d
}	t| |	 d S r   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r/   r/   r0   test_parquet_writer_context_objh   s"   

rA   c              
   C   s"  t d}d|d< tjj|dd}t }d}zDtj||jdd1}g }td	D ]"}||d< tjj|dd}|	| |
|  |d
krIt|q'W d    n1 sTw   Y  W n tyt } zt||ksjJ W Y d }~nd }~ww | }	tt|	}
tj|dd}t|
 | d S )Nr	   r   r
   Fr   zArtificial Errorr   r   r      Tr   )r   r   r   r   r   r   r   r   r   r   r   r   r:   	Exceptionstrr   r   r   r   r    r!   r"   r#   )r%   r&   r'   r(   Z
error_textr)   r*   r+   er,   r-   r.   r/   r/   r0   .test_parquet_writer_context_obj_with_exception   s>   
rF   
filesystemc           	      C   s  t d}tjj|dd}tjj|dd}t| d }t| d }tj||j|dd}|	| W d    n1 s:w   Y  t
| }t|| tj||j|dd}|| W d    n1 sew   Y  t
| }t|| tj||j|dd}|| W d    n1 sw   Y  t
| }t|| tj||j|dd}|| W d    n1 sw   Y  t
| }t|| d S )Nr	   Fr   zdata_table.parquetzdata_batch.parquetr   rG   r   )r   r   r   r   ZRecordBatchrD   r   r   r   r   r   r#   r!   r"   Zwrite_batchwrite)	r%   rG   r&   tablebatchZ
path_tableZ
path_batchr)   r-   r/   r/   r0   "test_parquet_writer_write_wrappers   sJ   



rL   c                    s   dd  fdd}|d d d |d d |d d d |d d d | d  d d |d d |d d d d S )	Ni   i   c           	         s   t jjt| dgdgd}|d u rt|d  n
tj|d |d td }|d u r1n|}|j|ks:J t| }t	|d D ]}|
|j|ksQJ qE| ||d   }|dkrl|
|d j|ksjJ d S |
|d j|ksxJ d S )Nbr>   )namesztest.parquet)Zrow_group_sizer2   r   )r   r   r7   r   r   r   read_metadataZnum_row_groupsminr   Z	row_groupZnum_rows)		data_size
chunk_sizeZexpect_num_chunksrJ   metadataZexpected_chunk_sizeZlatched_chunk_sizeZ	chunk_idx	remainderZabs_max_chunk_sizeZdefault_chunk_sizer%   r/   r0   check_chunk_size   s,   
z8test_parquet_writer_chunk_size.<locals>.check_chunk_size   r	      r2   r/   )r%   rV   r/   rU   r0   test_parquet_writer_chunk_size   s   rY   c                 C   s~   t d}tjj|dd}t| d }tj||j|dd}|| W d    n1 s,w   Y  t	|
 }t|| d S )Nr	   Fr   zdata.parquetr   rH   )r   r   r   r   rD   r   r   r   r   r   r#   r!   r"   )r%   rG   r&   rJ   r;   r)   r-   r/   r/   r0   $test_parquet_writer_filesystem_local   s   
rZ   c                 C   s|   t d}tjj|dd}| \}}}tj||j|dd}|| W d    n1 s+w   Y  t|	 }t
|| d S )Nr	   Fr   r   rH   r   r   r   r   r   r   r   r   r   r#   r!   r"   Zs3_example_fsr&   rJ   r   urir;   r)   r-   r/   r/   r0   !test_parquet_writer_filesystem_s3  s   

r^   c                 C   s~   t d}tjj|dd}| \}}}tj||jdd}|| W d    n1 s*w   Y  t||d	 }t
|| d S )Nr	   Fr   r   r   rG   r[   r\   r/   r/   r0   %test_parquet_writer_filesystem_s3_uri  s   
r`   c                 C   s   t d}tjj|dd}| \}}|d }tj||j|dd}|| W d    n1 s.w   Y  t||d	 }t
|| d S )Nr	   Fr   z/test.parquetr   rH   r_   r[   )Zs3_example_s3fsr&   rJ   r   	directoryr;   r)   r-   r/   r/   r0   #test_parquet_writer_filesystem_s3fs.  s   
rb   c                  C   sj   t d} tjj| dd}t }tjtdd t	j
t |j|d W d    d S 1 s.w   Y  d S )Nr	   Fr   zspecified path is file-like)matchr_   )r   r   r   r   r   r   r8   r9   r:   r   r   r   r   )r&   rJ   rG   r/   r/   r0   ,test_parquet_writer_filesystem_buffer_raises@  s   "rd   c                     s&  t   G  fdddt} |  }d}td}t jj|dd}tj||j|dd}|	| W d    n1 s9w   Y  |j
|ksEJ |jd	ksLJ  jsQJ   }tt |}| }t|| tt}	tjt  |j|d
 d}
t|	|
ksJ W d    d S 1 sw   Y  d S )Nc                       s"   e Zd Zdd Zd fdd	ZdS )zEtest_parquet_writer_with_caller_provided_filesystem.<locals>.CustomFSc                 S   s   d | _ d | _d S Nr;   mode)selfr/   r/   r0   __init__R  s   
zNtest_parquet_writer_with_caller_provided_filesystem.<locals>.CustomFS.__init__rbc                    s   || _ || _ S re   rf   )rh   r;   rg   r(   r/   r0   openV  s   zJtest_parquet_writer_with_caller_provided_filesystem.<locals>.CustomFS.openN)rj   )__name__
__module____qualname__ri   rl   r/   rk   r/   r0   CustomFSQ  s    rp   zexpected_fname.parquetr	   Fr   r   rH   wbr_   zVfilesystem passed but where is file-like, so there is nothing to open with filesystem.)r   r   r   r   r   r   r   r   r   r   r;   rg   closedr   r   r   r#   r!   r"   r8   r9   r:   rD   )rp   r   fnamer&   rJ   r)   r,   Z
table_readZdf_readZerr_infoZexpected_msgr/   rk   r0   3test_parquet_writer_with_caller_provided_filesystemM  s,   

"rt   c                 C   s   t dg di}| d }t||j}|| W d    n1 s$w   Y  t|}d|jv s5J |jd s<J | d }tj||jdd}|| W d    n1 sYw   Y  t|}|jd u sjJ d S )Na)r2   rW   rX   ztest_with_schema.parquets   ARROW:schemaztest_without_schema.parquetF)Zstore_schema)r   rJ   r   r   r   r   rO   rS   )r%   rJ   Zpath1r)   metaZpath2r/   r/   r0    test_parquet_writer_store_schemau  s   

rw   )(r8   Zpyarrowr   r   Zpyarrow.filesystemr   r   Zpyarrow.parquetZparquetr   Zpyarrow.tests.parquet.commonr   r   r   ImportErrorZpandasr   Zpandas.testingZtestingr!   markZ
pytestmarkr1   r=   r@   rA   rF   ZparametrizeZ_get_instancerL   Zlarge_memoryrY   rZ   Zs3r^   r`   rb   rd   rt   rw   r/   r/   r/   r0   <module>   sp   



()

'