o
    ņ©ZhyK  ć                   @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ e jj	Z
zd dlmZ d dlZW n	 ey5   Y nw e jdddd Zd	d
 Zd(ddZd)ddZe jje j dg d¢”dd Zdd Zdd Zdd Zdd Zdd Ze jjdd Zd d! Zd"d# Zd$d% Z d&d' Z!dS )*é    N)Śfs)Śutil)Śassert_frame_equalŚmodule)Śscopec                 C   s   | d S )NŚorc© )Zbase_datadirr   r   śM/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/test_orc.pyŚdatadir'   s   r
   c                 C   s&  |D ]}|| }| | }|dkr4dd |D r4||   ” }t|D ]\}}dd |D ||< q!|||< q|d j}t|tjrEt |”}nGt|tjrP|jj}n<|t	j
u rdgt| }	tt||D ]#\}\}
}t |”s|
 ” j}d|  }t	 
t|| ” |”|	|< qct |	”}|||< qdS )z_
    Fix type of expected values (as read from JSON) according to
    actual ORC datatype.
    Śmapc                 S   s&   g | ]}|D ]
}|  ” d dhkqqS ©ŚkeyŚvalue)Śkeys)Ś.0ŚmŚdr   r   r	   Ś
<listcomp>5   s   & z&fix_example_values.<locals>.<listcomp>c                 S   s   g | ]
}|d  |d fqS r   r   )r   r   r   r   r	   r   9   s    r   Né
   )ŚcopyŚ	enumerateŚ	__class__Ś
issubclassŚdatetimeŚpdŚto_datetimeŚdateŚdtŚdecimalŚDecimalŚlenŚzipZisnullŚas_tupleŚexponentŚroundŚscalebZSeries)Zactual_colsZexpected_colsŚnameŚexpectedŚactualŚcolŚir   ŚtypZconverted_decimalsr   ŚvŚexpŚfactorr   r   r	   Śfix_example_values,   s8   ’





’

ār/   c                 C   s6   |d us|d ur||| j dd}t| |dd d S )NT)ZdropF)Zcheck_dtype)Zreset_indexr   )Śorc_dfŚexpected_dfŚstartŚstopr   r   r	   Ścheck_example_valuesR   s   r4   Fc           
      C   sī   ddl m} | | ”}| ” }t|tjsJ | ”  t 	| 
” ”}t|jt|jks.J |j |j”s<|j|jd}|rCt|| t|| d}t|jD ]}| |”}	tt 	|	 
” ”|||t|	 d |t|	7 }qO||jksuJ dS )zC
    Check a ORC file against the expected columns dictionary.
    r   ©r   ©Ścolumns)r2   r3   N)Śpyarrowr   ŚORCFileŚreadŚ
isinstanceŚpaŚTableŚvalidater   Z	DataFrameZ	to_pydictŚsetr7   ŚequalsZreindexr/   r4   ŚrangeZnstripesZread_striper    Znrows)
Zorc_pathr1   Śneed_fixr   Śorc_fileŚtabler0   Zjson_posr*   Śbatchr   r   r	   Ścheck_example_fileX   s,   




żrF   Śfilename)zTestOrcFile.test1.orczTestOrcFile.testDate1900.orczdecimal.orcc                 C   s2   ||  }t jt| d”dd}t||dd dS )zŹ
    Check a ORC file example against the equivalent JSON file, as given
    in the Apache ORC repository (the JSON file has one JSON object per
    line, corresponding to one row in the ORC file).
    z.jsn.gzT)Ślines)rB   N)r   Ś	read_jsonŚstrŚwith_suffixrF   )rG   r
   ŚpathrD   r   r   r	   Śtest_example_using_json}   s   rM   c                 C   s&  ddl m} | | d ” ” }|jdksJ t dt ” fdt ” fdt 	” fdt 
” fdt ” fd	t ” fd
t ” fdt ” fdt ” fdt dt t dt 
” fdt ” fg””fg”fdt t dt 
” fdt ” fg””fdt t ” t dt 
” fdt ” fg””fg”}|j|ksJ d S )Nr   r5   zTestOrcFile.emptyFile.orcZboolean1Zbyte1Zshort1Zint1Zlong1Zfloat1Zdouble1Zbytes1Zstring1ŚmiddleŚlistr   )r8   r   r9   r:   Śnum_rowsr<   ŚschemaZbool_Zint8Zint16Śint32Śint64Zfloat32Zfloat64ŚbinaryŚstringŚstructŚlist_Zmap_)r
   r   rD   Zexpected_schemar   r   r	   Śtest_orcfile_empty   s@   









’’’
’’
’’ķrX   c                 C   sŖ   ddl m} t dg d¢i”}| d }| ”  |d }| |t|” |j|t 	” d}| 
|”s3J |jdt | ”d}| 
|”sDJ | t |””}| 
|”sSJ d S )	Nr   r5   Śa©é   é   é   Śdata_dirzdata.orc)Ś
filesystemzdata_dir/data.orc)r8   r   r<   rD   ŚmkdirŚwrite_tablerJ   Ś
read_tabler   ZLocalFileSystemr@   r   Z_filesystem_uri)Śtmpdirr   rD   Ś	directoryrL   Śresultr   r   r	   Śtest_filesystem_uri°   s    
’’rf   c                 C   s¾   ddl m} t g d¢”}t g d¢”}t ||d”}|  d”}| ||” | |”}| |”s3J | |g ”}d|j	ks@J d|j
ksGJ |j|dgd	}d|j	ksVJ d
|j
ks]J d S )Nr   r5   ©r[   Nr]   N©NŚArrowNZORC©rS   Śutf8śtest.orcé   rS   r6   r[   )r8   r   r<   ŚarrayrD   Śjoinra   rb   r@   rP   Znum_columns)rc   r   rY   ŚbrD   ŚfileŚoutput_tabler   r   r	   Śtest_orcfile_readwriteČ   s   

rs   c                  C   s   ddl m}  ddlm} | }t g d¢”}t g d¢”}t ||d”}|  ||” | d” |  	|”}| 
” }| |”s@J d S )Nr   r5   )ŚBytesIOrg   rh   rj   )r8   r   Śiort   r<   rn   rD   ra   Śseekr9   r:   r@   )r   rt   ŚbufrY   rp   rD   rC   rr   r   r   r	   Śtest_bytesio_readwriteŪ   s   

rx   c                  C   sZ  ddl m}  t ” }t g d¢”}t g d¢”}t ||d”}|  ||” t | ” ”}|  	|”}| 
” }| |”s=J |jdksDJ |jdksKJ |jdksRJ |jd	ksYJ t ” }t t” |  ||” W d    n1 ssw   Y  t | ” ”}|  	|”}| 
” }| |”sJ |jdksJ |jdksJ |jdks¤J |jd	ks«J d S )
Nr   r5   rg   rh   rj   ŚUNCOMPRESSEDz0.12i'  i   ©r8   r   r<   ŚBufferOutputStreamrn   rD   ra   ZBufferReaderŚgetvaluer9   r:   r@   ŚcompressionŚfile_versionŚrow_index_strideZcompression_sizeŚpytestZwarnsŚFutureWarning©r   Śbuffer_output_streamrY   rp   rD   Zbuffer_readerrC   rr   r   r   r	   Śtest_buffer_readwriteź   s4   
’
r   c               	   C   sn  ddl m}  t ” }t g d¢”}t g d¢”}t ||d”}| j||dddd	d
 t | ” ”}|  	|”}| 
” }| |”sBJ |jdksIJ |jdksPJ |jdksWJ |jd	ks^J t ” }t t” | j||ddddd
 W d    n1 s}w   Y  t | ” ”}|  	|”}| 
” }| |”sJ |jdks J |jdks§J |jdks®J |jdksµJ d S )Nr   r5   rg   rh   rj   Śsnappyz0.11i  i   )r}   r~   r   Ścompression_block_sizeZSNAPPYZuncompressedi N  i @  ry   rz   r   r   r   r	   Ś'test_buffer_readwrite_with_writeoptions  sP   ś
ś’	
r   c                  C   s6  ddl m}  t ” }t g d¢”}t d|i”}t t” | j	||dd W d    n1 s0w   Y  t t” | j	||dd W d    n1 sMw   Y  t t” | j	||dd W d    n1 sjw   Y  t t” | j	||dd	 W d    n1 sw   Y  t t” | j	||d
d	 W d    n1 s¤w   Y  t t” | j	||dd W d    n1 sĮw   Y  t t” | j	||dd W d    n1 sŽw   Y  t t” | j	||dd W d    n1 sūw   Y  t t
” | j	||dd W d    n	1 sw   Y  t t” | j	||dd W d    n	1 s7w   Y  t t” | j	||dd W d    n	1 sUw   Y  t t” | j	||dd W d    n	1 ssw   Y  t t” | j	||dd W d    n	1 sw   Y  t t” | j	||dd W d    n	1 sÆw   Y  t t
” | j	||dd W d    n	1 sĶw   Y  t t” | j	||dd W d    n	1 sėw   Y  t t” | j	||dd W d    n	1 s	w   Y  t t” | j	||dd W d    n	1 s'w   Y  t t” | j	||dd W d    n	1 sEw   Y  t t” | j	||dd W d    n	1 scw   Y  t t” | j	||dd W d    n	1 sw   Y  t t” | j	||dd W d    n	1 sw   Y  t t” | j	||dd W d    n	1 s½w   Y  t t” | j	||dd W d    n	1 sŪw   Y  t t” | j	||d d! W d    n	1 słw   Y  t t” | j	||dd"gd! W d    n	1 sw   Y  t t” | j	||h d#£d! W d    n	1 s9w   Y  t t” | j	||dd$ W d    n	1 sWw   Y  t t” | j	||d%d$ W d    n	1 suw   Y  t t” | j	||d&d$ W d    d S 1 sw   Y  d S )'Nr   r5   rg   rS   )Z
batch_sizei’’’gRøė @g¤p=
×£Ą?)r~   z1.1)Zstripe_sizeipž’’g®Gįŗ °@)r}   ŚnoneZzlid)r   i8’’’gRøė"@)Zcompression_strategyŚnoZlarge)r   iąü’’g®Gįz0Ø@Ścat)Zpadding_toleranceŚarrow)Zdictionary_key_size_thresholdg333333ó?g	ĄrU   )Zbloom_filter_columnsgffffffö?>   r   r\   é’’’’)Zbloom_filter_fppgń?g¹æ)r8   r   r<   r{   rn   rD   r   ŚraisesŚ
ValueErrorra   Ś	TypeError)r   r   rY   rD   r   r   r	   Ś+test_buffer_readwrite_with_bad_writeoptions;  s¬  ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’ż’	ż’ż’ż’	ż’ż’ż’ż’ż’ż$’r   c                 C   s.  ddl m} t dt ” ”}t dt |g””}t dt ” ”t dt t dt ” ”””t dt |t d	t ” ”g””t d
t t dt t dt ” ”t d	t ” ”g””””t dt ” ”g}dgddggddiddgddddddggdgg}tj|t 	|”d}t
| d }| ||” | |”}| ” }	|	 |”s¤J |jddgd}
|
 | ddg””søJ |jg d¢d}| | g d¢””sĢJ |jdgd}t ddddiigi”}| |”sęJ |jdgd}t dd	digi”}| |”sžJ |jg d¢d}| | ddg””sJ |jdgd}t d
ddiddiggi”}| |”s0J |jddgd}
|
 | ddg””sEJ |jg d ¢d}| | g d!¢””sZJ t t” |jd"gd W d    n	1 srw   Y  t t” |jdgd W d    d S 1 sw   Y  d S )#Nr   r5   ŚinnerrN   ŚbasicrO   ŚitemrV   Śinner2ślist-structŚinner1Śbasic2r[   r\   r]   rm   )rN   r   é   é   )r   r   é   é   é	   )rQ   rl   r6   )rO   rV   r   śstruct.middle.innerśstruct.inner2)rO   r   r   zlist-struct.inner1rZ   )rO   rV   r   Zwrong)r8   r   r<   ŚfieldrS   rV   rR   rW   rD   rQ   rJ   ra   r9   r:   r@   Śselectr   r   ŚIOErrorr   )Śtempdirr   r   rN   ŚfieldsZarrsrD   rL   rC   Zresult1Zresult2Zresult3Zresult4Z	expected4Zresult5Z	expected5Zresult6Zresult7Z	expected7r   r   r	   Śtest_column_selection  sp   ’’
ž’’šž
’ż$’r¤   c              	   C   s   ddl m} t| d }| |”*}t t” | ”  W d    n1 s&w   Y  W d    d S W d    d S 1 s>w   Y  d S )Nr   r5   rl   )r8   r   rJ   Z	ORCWriterr   r   ŚAttributeErrorŚtest)r¢   r   rL   Śwriterr   r   r	   Śtest_wrong_usage_orc_writerl  s   
’’"’rØ   c                 C   s   ddl m} t| d }t g d¢”}t g d¢”}t ||d”}t tj” | 	||” W d    d S 1 s:w   Y  d S )Nr   r5   rl   rg   )NNNNrj   )
r8   r   rJ   r<   rn   rD   r   r   ZArrowNotImplementedErrorra   )r¢   r   rL   rY   rp   rD   r   r   r	   Ś test_orc_writer_with_null_arraysu  s   "’r©   )NN)F)"r   r   r   r8   r<   r   Zpyarrow.testsr   Śmarkr   Z
pytestmarkZpandas.testingr   Zpandasr   ŚImportErrorZfixturer
   r/   r4   rF   ZparametrizerM   rX   rf   rs   rx   r   r   r   r   r¤   rØ   r©   r   r   r   r	   Ś<module>   sB   ’


&
%!!
/ cO	