o
    ò©Zh4™  ã                   @   s–  d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZ	d dl
Zd dlmZmZ zd dlmZ d dlZW n	 eyC   Y nw G dd„ dƒZG dd„ deƒZG d	d
„ d
eƒZG dd„ deƒZejdd„ ƒZejdd„ ƒZejdd„ ƒZejejdddejdddgddd„ ƒZdd„ Zdd„ Zdd„ Z ej! "dd d!„ d"d!„ g¡d#d$„ ƒZ#d%d&„ Z$ej!jd'd(„ ƒZ%d)d*„ Z&d+d,„ Z'ej!jd-d.„ ƒZ(ej!jd/d0„ ƒZ)d1d2„ Z*ej! "d3ej+ ,¡ ej+j,d4d5g¡d6d7„ ƒZ-d8d9„ Z.ej! "d3ej+ ,¡ ej+j,d4d5g¡d:d;„ ƒZ/d<d=„ Z0ej!jd>d?„ ƒZ1ej!jd@dA„ ƒZ2ej! "dBd4dCg¡dDdE„ ƒZ3ej!j4dFdG„ ƒZ5dHdI„ Z6dJdK„ Z7ej! "d3ej+ 8¡ ej+j8dCdLej+j8dCdMej+j8ej+j9j:dNej+j8dCej+j9j:dOg¡dPdQ„ ƒZ;dRdS„ Z<dTdU„ Z=dVdW„ Z>dXdY„ Z?dZd[„ Z@ej!jd\d]„ ƒZAejd^d_„ ƒZBd`da„ ZCdbdc„ ZDddde„ ZEej!jFdfdg„ ƒZGdhdi„ ZHdjdk„ ZIdldm„ ZJG dndo„ doejKƒZLG dpdq„ dqeƒZMejdrds„ ƒZNdtdu„ ZOdvdw„ ZPej!jdxdy„ ƒZQej!jdzd{„ ƒZRej!jej! "d|d}d~g¡dd€„ ƒƒZSdd‚„ ZTej!jdƒd„„ ƒZUej!jd¡d…d†„ƒZVej!jd‡dˆ„ ƒZWej!jd‰dŠ„ ƒZXej!jd‹dŒ„ ƒZYej!jddŽ„ ƒZZej!jdd„ ƒZ[ej!jd‘d’„ ƒZ\ej!jd“d”„ ƒZ]d•d–„ Z^d—d˜„ Z_d™dš„ Z`d›dœ„ Zaddž„ ZbdŸd „ ZcdS )¢é    )ÚUserListN)Úchanged_environÚinvoke_script)Úassert_frame_equalc                   @   s:   e Zd ZdZdd„ fdd„Zdd„ Zdd	„ Zddd„ZdS )Ú
IpcFixtureNc                   C   ó   t  ¡ S ©N©ÚioÚBytesIO© r   r   úM/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/test_ipc.pyÚ<lambda>*   ó    zIpcFixture.<lambda>c                 C   s   || _ |  ¡ | _d S r   )Ú_sink_factoryÚget_sinkÚsink)ÚselfÚsink_factoryr   r   r   Ú__init__*   s   zIpcFixture.__init__c                 C   s   |   ¡ S r   )r   ©r   r   r   r   r   .   s   zIpcFixture.get_sinkc                 C   s
   | j  ¡ S r   )r   Úgetvaluer   r   r   r   Ú
get_source1   ó   
zIpcFixture.get_sourceé   Fc           
      C   s®   d}t  dt  ¡ fdt  ¡ fg¡}|  | j|¡}g }t|ƒD ]}t jtj	 
|¡g d¢g|d}| |¡ q|rCt j |¡}	| |	¡ n
|D ]}| |¡ qE|j| _| ¡  |S )Nr   ÚoneÚtwo)ÚfooNÚbarÚbazbazÚqux©Úschema)Úpar"   Zfloat64Úutf8Ú_get_writerr   ÚrangeÚrecord_batchÚnpÚrandomÚrandnÚappendÚTableÚfrom_batchesÚwrite_tableÚwrite_batchÚstatsÚwrite_statsÚclose)
r   Znum_batchesÚas_tableÚnrowsr"   ÚwriterÚbatchesÚiÚbatchÚtabler   r   r   Úwrite_batches4   s(   
ÿýzIpcFixture.write_batches)r   F)Ú__name__Ú
__module__Ú__qualname__r1   r   r   r   r:   r   r   r   r   r   '   s    r   c                   @   s&   e Zd ZdZdZdd„ Zddd„ZdS )	ÚFileFormatFixtureTNc                 C   s   t jj||| jdS )N©Úoptions)r#   ÚipcÚnew_filer@   ©r   r   r"   r   r   r   r%   S   ó   zFileFormatFixture._get_writerFc                 C   sÀ   | j |d}t |  ¡ ¡}tj |¡}|jt|ƒksJ ‚t|ƒD ]\}}| 	|¡}||  
|¡s2J ‚|j 
|d j¡s=J ‚q t|jtjjƒsHJ ‚t| jtjjƒsRJ ‚t|jƒt| jƒks^J ‚d S )N©r3   r   )r:   r#   ÚBufferReaderr   rA   Ú	open_fileÚnum_record_batchesÚlenÚ	enumerateÚ	get_batchÚequalsr"   Ú
isinstancer0   Z	ReadStatsr1   Z
WriteStatsÚtuple)r   r3   r6   Úfile_contentsÚreaderr7   r8   r   r   r   Ú_check_roundtripV   s   
z"FileFormatFixture._check_roundtrip©F)r;   r<   r=   Úis_filer@   r%   rQ   r   r   r   r   r>   N   s
    r>   c                   @   s    e Zd ZdZdZdZdd„ ZdS )ÚStreamFormatFixtureFNc                 C   s   t jj||| j| jdS )N)Úuse_legacy_formatr@   )r#   rA   Ú
new_streamÚuse_legacy_ipc_formatr@   rC   r   r   r   r%   q   s   üzStreamFormatFixture._get_writer)r;   r<   r=   rW   r@   rS   r%   r   r   r   r   rT   i   s
    rT   c                   @   s   e Zd Zdd„ ZdS )ÚMessageFixturec                 C   ó   t  ||¡S r   ©r#   ZRecordBatchStreamWriterrC   r   r   r   r%   |   ó   zMessageFixture._get_writerN)r;   r<   r=   r%   r   r   r   r   rX   z   s    rX   c                   C   ó   t ƒ S r   )r   r   r   r   r   Úipc_fixture€   ó   r]   c                   C   r\   r   )r>   r   r   r   r   Úfile_fixture…   r^   r_   c                   C   r\   r   )rT   r   r   r   r   Ústream_fixtureŠ   r^   r`   zFile Format)ÚidzStream Format)Úparamsc                 C   s   |   | j¡S r   )ZgetfixturevalueÚparam)Úrequestr   r   r   Úformat_fixture   s   re   c                  C   sH   d} t  tj¡ tj t | ¡¡ W d   ƒ d S 1 sw   Y  d S ©Nó    )ÚpytestÚraisesr#   ÚArrowInvalidrA   rG   rF   ©Úbufr   r   r   Útest_empty_file   s   "ÿrm   c                 C   ó   | j dd d S )NFrE   ©rQ   ©r_   r   r   r   Útest_file_simple_roundtrip£   ó   rq   c                 C   rn   )NTrE   ro   rp   r   r   r   Útest_file_write_table§   rr   rs   r   c                   C   r   r   r	   r   r   r   r   r   ¬   r   r   c                   C   r   r   )r#   ÚBufferOutputStreamr   r   r   r   r   ­   r   c                 C   sP   t | ƒ}| ¡ }t | ¡ ¡}tj |¡}| ¡ }tj 	|¡}| 
|¡s&J ‚d S r   )r>   r:   r#   rF   r   rA   rG   Úread_allr,   r-   rL   )r   Úfixturer6   rO   rP   ÚresultÚexpectedr   r   r   Útest_file_read_all«   s   ry   c           	      C   s®   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s/J ‚| |¡s6J ‚|j	}|j
dks@J ‚|jdksGJ ‚|j	|ksNJ ‚|j	|ksUJ ‚d S ©Né   r   )r:   r   r#   rA   rG   rF   ÚRecordBatchFileReaderru   rL   r0   Únum_messagesrH   )	r_   ÚsourceÚreader1Úreader2Úreader3Úresult1Úresult2Úresult3Úst1r   r   r   Útest_open_file_from_buffer¼   s   
r†   c                 C   sT   dd„ |   ¡ D ƒ}t |  ¡ ¡}tj |¡}| ¡ }t |¡j	dd}t
||ƒ d S )Nc                 S   ó   g | ]}|  ¡ ‘qS r   ©Ú	to_pandas©Ú.0r8   r   r   r   Ú
<listcomp>Õ   ó    z)test_file_read_pandas.<locals>.<listcomp>T©Zdrop)r:   r#   rF   r   rA   rG   Úread_pandasÚpdÚconcatÚreset_indexr   )r_   ÚframesrO   rP   rw   rx   r   r   r   Útest_file_read_pandasÓ   s   r”   c                 C   sŽ   |   ¡  |  ¡ }| d¡j}t|dƒ}| |¡ W d   ƒ n1 s#w   Y  tj t	 
|¡¡ ¡ }tj t |¡¡ ¡ }| |¡sEJ ‚d S )Nú
file.arrowÚwb)r:   r   ÚjoinÚstrpathÚopenÚwriter#   rA   rG   ÚpathlibÚPathru   ÚOSFilerL   )r_   Útmpdirr~   ÚpathÚfÚt1Út2r   r   r   Útest_file_pathlibß   s   ÿr£   c                  C   sH   t  d¡} t tj¡ tj | ¡ W d   ƒ d S 1 sw   Y  d S rf   )r
   r   rh   ri   r#   rj   rA   Úopen_streamrk   r   r   r   Útest_empty_streamí   s   
"ÿr¥   c              	   C   s¸   t  ¡ }t  t  d|¡g¡}|  d¡j}t  |d¡3}t j ||¡}t j	dg|d}t  
|g|¡}| |¡ W d  ƒ n1 sAw   Y  W d  ƒ n1 sPw   Y  td|ƒ dS )zËARROW-15783: Verify to_pandas works for interval types.

    Interval types require static structures to be enabled. This test verifies
    that they are when no other library functions are invoked.
    Znumsr•   r–   ©é   é   é   ©ÚtypeNzread_record_batch.py)r#   Zmonth_day_nano_intervalr"   Úfieldr—   r˜   r   rA   rB   Úarrayr'   rš   r   )rž   Zmdn_interval_typer"   rŸ   r   r5   Zinterval_arrayr8   r   r   r   Ú"test_read_year_month_nano_intervaló   s   ý€ÿr®   c              	   C   s¦   t  tj d¡t jdtjdddgddgdddœ¡}tj 	|¡}|  
| j|j¡}| |¡ W d   ƒ n1 s8w   Y  tj t |  ¡ ¡¡ ¡ }t| ¡ |ƒ d S )Nr   r   r   T©Ú
categoriesÚordered©r   r   )r   Ú	DataFramer(   r)   r*   ÚCategoricalÚnanr#   ÚRecordBatchÚfrom_pandasr%   r   r"   r/   rA   r¤   rF   r   ru   r   r‰   )r`   Údfr8   Úwrr9   r   r   r   Ú!test_stream_categorical_roundtrip  s   
þþÿÿrº   c           	      C   sÄ   |   ¡  |  ¡ }tj |¡}tj t |¡¡}t |¡}| ¡ }| ¡ }| ¡ }| |¡s/J ‚| |¡s6J ‚|j	}|j
dks@J ‚|jdksGJ ‚|j	|ksNJ ‚|j	|ksUJ ‚t|ƒt| jƒks`J ‚d S rz   )r:   r   r#   rA   r¤   rF   ZRecordBatchStreamReaderru   rL   r0   r}   rH   rN   r1   )	r`   r~   r   r€   r   r‚   rƒ   r„   r…   r   r   r   Útest_open_stream_from_buffer  s    
r»   r@   F©Úuse_threadsc                 C   sd   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdksJ ‚|jdks%J ‚t	|ƒt	| j
ƒks0J ‚d S ©Nr?   r{   r   )r:   r   r#   rA   r¤   ru   r0   r}   rH   rN   r1   )r`   r@   r~   rP   Ústr   r   r   Útest_open_stream_options0  s   rÀ   c                 C   óP   |   ¡  |  ¡ }t t¡ tjj|dd W d   ƒ d S 1 s!w   Y  d S ©NTr?   )r:   r   rh   ri   Ú	TypeErrorr#   rA   r¤   )r`   r~   r   r   r   Ú#test_open_stream_with_wrong_optionsB  ó
   "ÿrÄ   c                 C   sN   |   ¡  |  ¡ }tjj||d}| ¡  |j}|jdksJ ‚|jdks%J ‚d S r¾   )	r:   r   r#   rA   rG   ru   r0   r}   rH   )r_   r@   r~   rP   r¿   r   r   r   Útest_open_file_optionsJ  s   rÆ   c                 C   rÁ   rÂ   )r:   r   rh   ri   rÃ   r#   rA   rG   )r_   r~   r   r   r   Ú!test_open_file_with_wrong_options[  rÅ   rÇ   c              	   C   sÒ   t  tj d¡t jdtjdddgddgdddœ¡}tjj	|dd}tj
j	|dd}|  | j|j¡}| |¡ | |¡ W d   ƒ n1 sGw   Y  tj t |  ¡ ¡¡ ¡ }t| ¡ t j||gdd	ƒ d S )
Nr   r   r   Tr¯   r²   F©Zpreserve_index©Zignore_index)r   r³   r(   r)   r*   r´   rµ   r#   r,   r·   r¶   r%   r   r"   rš   rA   r¤   rF   r   ru   r   r‰   r‘   )r`   r¸   r9   r8   r¹   r   r   r   Útest_stream_write_dispatchc  s&   
þþ
þÿÿrÊ   c                 C   sú   t  dtj d¡i¡}tjj|d d… dd}tjj|dd}tj 	|||g¡}|  
| j|j¡}|j|dd W d   ƒ n1 sBw   Y  ttj |  ¡ ¡ƒ}ttt|ƒƒg d¢ks^J ‚tj 	|¡}t| ¡ t j|d d… ||d d… gd	d
ƒ d S )Nr   é   é
   FrÈ   é   )Zmax_chunksize)rÌ   rÍ   r   rÌ   TrÉ   )r   r³   r(   r)   r*   r#   r¶   r·   r,   r-   r%   r   r"   r.   ÚlistrA   r¤   r   ÚmaprI   r   r‰   r‘   )r`   r¸   Úb1Úb2r9   r¹   r6   Zresult_tabler   r   r   Útest_stream_write_table_batchesx  s"   ÿÿÿÿrÒ   rW   Tc                 C   s¶   || _ |  ¡ }t |  ¡ ¡}tj |¡}|j |d j¡sJ ‚d}t	|ƒD ]\}}| || ¡s2J ‚|d7 }q%|t
|ƒks?J ‚t t¡ | ¡  W d   ƒ d S 1 sTw   Y  d S ©Nr   r§   )rW   r:   r#   rF   r   rA   r¤   r"   rL   rJ   rI   rh   ri   ÚStopIterationÚread_next_batch)r`   rW   r6   rO   rP   Útotalr7   Ú
next_batchr   r   r   Útest_stream_simple_roundtrip  s   

"ÿrØ   c                  C   s.  t  ¡ } tj ddd¡}tjj|gdgd}tjj	dd}tjj
| |j|d}| |¡ W d   ƒ n1 s7w   Y  t|  ¡ ƒ}t  ¡ }tjdd	d
}tjj	|d}tjj
||j|d}| |¡ W d   ƒ n1 snw   Y  t| ¡ ƒ}||ksJ ‚tj | ¡ ¡ }	tj |¡ ¡ }
|	|
ks•J ‚d S )Nr   r©   i'  Úvalues©ÚnamesÚzstd©Úcompressionr?   r   )Zcompression_level)r
   r   r(   r)   Úrandintr#   r,   Úfrom_arraysrA   ÚIpcWriteOptionsZRecordBatchFileWriterr"   r.   rI   r   ÚCodecrG   ru   )r   rÙ   r9   r@   r5   Zlen1Zsink2ÚcodecZlen2r¡   r¢   r   r   r   Útest_compression_roundtrip¤  s2   ÿþÿþrä   c               	   C   sò  t j ¡ } | jdu sJ ‚| jdu sJ ‚| jt jjjksJ ‚d| _| jdu s'J ‚d| _| jdu s1J ‚t jjj| _| jt jjjksAJ ‚dD ]}t	 
ttf¡ || _W d   ƒ n1 sZw   Y  qC| jd u sgJ ‚dD ]}t j |¡r‡|| _| j|ks{J ‚| ¡ | _| j|ks‡J ‚qid | _| jd u s’J ‚t	 
t¡ d| _W d   ƒ n1 s¥w   Y  | jdu s±J ‚d| _| jdu s»J ‚t j d¡rõt jjt jjjddddd} | jt jjjksÙJ ‚| jdu sàJ ‚| jdu sçJ ‚| jdksîJ ‚| jdu s÷J ‚d S d S )NFT)ÚV5é*   )Úlz4rÜ   r   rç   )Úmetadata_versionÚallow_64bitrU   rÞ   r½   )r#   rA   rá   ré   rU   rè   ÚMetadataVersionrå   ÚV4rh   ri   rÃ   Ú
ValueErrorrÞ   râ   Zis_availableÚupperr½   )r@   Úvaluer   r   r   Útest_write_optionsÃ  s\   
ÿ€
€ÿûõrï   c                 C   sN   t jtdd d| _tj ¡ | _|  ¡  W d   ƒ d S 1 s w   Y  d S )Nz4provide at most one of options and use_legacy_format©ÚmatchT)	rh   ri   rì   rW   r#   rA   rá   r@   r:   )r`   r   r   r   Ú#test_write_options_legacy_exclusiveô  s   þ
"ûrò   )ré   )rU   )rè   )rU   rè   c           	      C   sÜ   d | _ || _|  ¡ }t |  ¡ ¡}tj |  ¡ ¡}|j|jks!J ‚tj 	|¡}|j
 |d j
¡s2J ‚d}t|ƒD ]\}}| || ¡sEJ ‚|d7 }q8|t|ƒksRJ ‚t t¡ | ¡  W d   ƒ d S 1 sgw   Y  d S rÓ   )rW   r@   r:   r#   rF   r   rA   Úread_messagerè   r¤   r"   rL   rJ   rI   rh   ri   rÔ   rÕ   )	r`   r@   r6   rO   ÚmessagerP   rÖ   r7   r×   r   r   r   Útest_stream_options_roundtripý  s    	

"ÿrõ   c                  C   sì   t j ¡ } | jdu sJ ‚| jdu sJ ‚| jg ksJ ‚d| _| jdu s$J ‚d| _| jdu s.J ‚ddg| _| jddgks<J ‚t t¡ d | _W d   ƒ n1 sOw   Y  t jjdddgd} | jdu seJ ‚| jdu slJ ‚| jdgkstJ ‚d S )NTFr   r§   )r½   Úensure_native_endianÚincluded_fields)	r#   rA   ÚIpcReadOptionsr½   rö   r÷   rh   ri   rÃ   r?   r   r   r   Útest_read_options  s(   

ÿþrù   c                 C   s  t j ¡ }t jjdgd}t jjt  g d¢¡t  g d¢¡gddgd}|  | j|j¡}| 	|¡ W d   ƒ n1 s;w   Y  |  
¡ }t jj||d}t jj||t  ¡ d	}| ¡ }| ¡ }	|jd
ksfJ ‚|	jdksmJ ‚t jjt  g d¢¡gdgd}
|	|
ks‚J ‚||ksˆJ ‚d S )Nr§   )r÷   ©r   r   Zbazr    ©r§   r¨   r©   é   ÚaÚbrÚ   r?   )r@   Zmemory_poolr¨   )r#   rA   rø   r,   rà   r­   r%   r   r"   r.   r   r¤   Zsystem_memory_poolru   Znum_columns)r`   Zoptions1Zoptions2r9   r¹   r~   r   r€   r‚   rƒ   rx   r   r   r   Ú!test_read_options_included_fields8  s,   
ÿþÿ
ÿrÿ   c                    sŒ  t  t  ¡ t  ¡ ¡‰g d¢g d¢ddgg d¢ddgg}‡fdd„|D ƒ}|d d	… }|d
 j‰d‡ ‡fdd„	}ˆ jrot t j¡ ||ƒ W d   ƒ n1 sOw   Y  t t j¡ ||ƒ W d   ƒ n1 siw   Y  n ||ƒ}|j	dkszJ ‚|j
d	ksJ ‚|jdksˆJ ‚|jd
ksJ ‚d ˆ _t jjddˆ _ˆ jr¸t t j¡ ||ƒ W d   ƒ n1 s²w   Y  n ||ƒ}|j	dksÃJ ‚|j
d	ksÊJ ‚|jdksÑJ ‚|jdksØJ ‚||ƒ}|j	d	ksãJ ‚|j
dksêJ ‚|jd
ksñJ ‚|jdksøJ ‚t jjddˆ _||dd}|j	dksJ ‚ˆ jr,|j
dksJ ‚|jd
ks"J ‚|jd
ks*J ‚d S |j
d	ks4J ‚|jdks<J ‚|jd
ksDJ ‚d S )N)r   r   N)r   r   r   r   r   )r   Nr   Úquuxr   c                    s*   g | ]}t jjt j|ˆ d gdgd‘qS )rª   ZdictsrÚ   )r#   r¶   rà   r­   )r‹   Úv)Útyr   r   rŒ   Y  s    ÿÿz)test_dictionary_delta.<locals>.<listcomp>rü   r   Fc                    sj   ˆ   t ¡ ˆ¡#}|rtj | ¡}| |¡ n
| D ]}| |¡ q|jW  d   ƒ S 1 s.w   Y  d S r   )r%   r#   ZMockOutputStreamr,   r-   r.   r/   r0   )r6   r3   r5   r9   r8   )re   r"   r   r   r:   _  s   
ÿ$øz,test_dictionary_delta.<locals>.write_batchesr   r©   T)Zemit_dictionary_deltasr§   r¨   )Zunify_dictionariesrE   rR   )r#   Ú
dictionaryÚint8r$   r"   rS   rh   ri   rj   rH   Znum_dictionary_batchesZnum_replaced_dictionariesZnum_dictionary_deltasrW   rA   rá   r@   )re   Údatar6   Zbatches_delta_onlyr:   r¿   r   )re   r"   r  r   Útest_dictionary_deltaQ  sr   ü
þ

ÿ
ÿ€ÿ
ÿ€ÿr  c               	   C   s\  t  t  dt  ¡ ¡g¡} t j t  ¡ | ¡}|jrJ ‚|jt jj	j
ks$J ‚t j t  ¡ | ¡}|jr2J ‚|jt jj	j
ks<J ‚tddƒ8 t j t  ¡ | ¡}|jsPJ ‚|jt jj	j
ksZJ ‚t j t  ¡ | ¡}|jshJ ‚|jt jj	j
ksrJ ‚W d   ƒ n1 s|w   Y  tddƒ8 t j t  ¡ | ¡}|jr•J ‚|jt jj	jksŸJ ‚t j t  ¡ | ¡}|jr­J ‚|jt jj	jks·J ‚W d   ƒ n1 sÁw   Y  tddƒX tddƒ9 t j t  ¡ | ¡}|jsàJ ‚|jt jj	jksêJ ‚t j t  ¡ | ¡}|jsøJ ‚|jt jj	jksJ ‚W d   ƒ n1 sw   Y  W d   ƒ d S W d   ƒ d S 1 s'w   Y  d S )Nr   ZARROW_PRE_0_15_IPC_FORMATÚ1ZARROW_PRE_1_0_METADATA_VERSION)r#   r"   r¬   Úint32rA   rV   rt   Z_use_legacy_formatZ_metadata_versionrê   rå   rB   r   rë   )r"   r5   r   r   r   Ú!test_envvar_set_legacy_ipc_formatœ  sF   



ú

ú

 úÿ$ÿr	  c                 C   sH   |   ¡ }t |  ¡ ¡}tj |¡}| ¡ }tj |¡}| 	|¡s"J ‚d S r   )
r:   r#   rF   r   rA   r¤   ru   r,   r-   rL   )r`   r6   rO   rP   rw   rx   r   r   r   Útest_stream_read_allÀ  s   r
  c                 C   sN   dd„ |   ¡ D ƒ}|  ¡ }tj |¡}| ¡ }t |¡jdd}t	||ƒ d S )Nc                 S   r‡   r   rˆ   rŠ   r   r   r   rŒ   Ì  r   z+test_stream_read_pandas.<locals>.<listcomp>TrŽ   )
r:   r   r#   rA   r¤   r   r   r‘   r’   r   )r`   r“   rO   rP   rw   rx   r   r   r   Útest_stream_read_pandasÊ  s   r  c                 C   s2   |   ¡ }|  ¡ }t |¡}tj |¡}|t|ƒfS r   )r:   r   r#   rF   ÚMessageReaderr¤   rÎ   )r`   r6   rO   Z
buf_readerrP   r   r   r   Úexample_messagesÕ  s
   
r  c                   C   sr   t  t¡ tt ¡ ƒ W d   ƒ n1 sw   Y  t  t¡ tt ¡ ƒ W d   ƒ d S 1 s2w   Y  d S r   )rh   ri   rÃ   Úreprr#   ÚMessager  r   r   r   r   Útest_message_ctors_no_segfaultÞ  s   ÿ"ÿr  c                 C   sÆ   | \}}t |ƒdksJ ‚|d jdksJ ‚t|d jtjƒs J ‚t|d jtjƒs+J ‚|d jtjj	ks6J ‚|dd … D ]$}|jdksEJ ‚t|jtjƒsNJ ‚t|jtjƒsWJ ‚|jtjj	ks`J ‚q<d S )Nr{   r   r"   r§   zrecord batch)
rI   r«   rM   Úmetadatar#   ÚBufferÚbodyrè   rê   rå   )r  Ú_ÚmessagesÚmsgr   r   r   Útest_message_readeræ  s   ür  c           
      C   s  | \}}|d }|  ¡ }t | ¡ d ¡}tj |¡}tj |¡}tj | ¡ ¡}tj |¡}	| |¡s6J ‚| |¡s=J ‚| |¡sDJ ‚| |	¡sKJ ‚tjtj	dd tj t d¡¡ W d   ƒ n1 sgw   Y  t t
¡ tj |¡ W d   ƒ d S 1 sƒw   Y  d S )Nr   r¨   zCorrupted messagerð   s   ab)Ú	serializer#   rF   Z
to_pybytesrA   ró   rL   rh   ri   rj   ÚEOFError)
r  r  r  r  rl   rP   ZrestoredZ	restored2Z	restored3Z	restored4r   r   r   Ú#test_message_serialize_read_messageö  s$   ÿ"ÿr  c              	   C   s‚   | \}}|D ]8}t  ¡ }t j|dd}| |¡ W d   ƒ n1 s#w   Y  | ¡ }t j t j|dd¡}| |¡s>J ‚qd S )NÚgziprÝ   )	r#   rt   Zoutput_streamZserialize_tor   rA   ró   Zinput_streamrL   )r  r  r  rô   Zraw_outZcompressed_outZcompressed_bufrw   r   r   r   Ú!test_message_read_from_compressed  s   ÿÿ÷r  c                 C   s0   | \}}t j |d ¡}| |d j¡sJ ‚d S rÓ   )r#   rA   Úread_schemarL   r"   )r  r6   r  r"   r   r   r   Útest_message_read_schema  s   r  c                 C   sF   | \}}t ||dd … ƒD ]\}}tj ||j¡}| |¡s J ‚qd S )Nr§   )Úzipr#   rA   Úread_record_batchr"   rL   )r  r6   r  r8   rô   Z
read_batchr   r   r   Útest_message_read_record_batch$  s
   þr!  c                  C   s°   t jt jdgt  ¡ dgdgd} t  ¡ }t j || j¡}| | ¡ W d   ƒ n1 s-w   Y  | 	¡ }t
jtdd t j || j¡ W d   ƒ d S 1 sQw   Y  d S )Nó   foorª   ÚstrsrÚ   z type record batch but got schemarð   )r#   r'   r­   r$   rt   rA   rV   r"   r/   r   rh   ri   ÚIOErrorr   )r8   Ústreamr5   rl   r   r   r   Ú.test_read_record_batch_on_stream_error_message,  s   ÿÿÿ"þr&  c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚStreamReaderServerc                 C   sT   t   t jt j¡| _| j d¡ | j d¡ | j ¡ \}}|| _d | _g | _	d | _
|S )N)ú	127.0.0.1r   r§   )ÚsocketÚAF_INETÚSOCK_STREAMÚ_sockÚbindÚlistenÚgetsocknameÚ_do_read_allÚ_schemaÚ_batchesÚ_table)r   Údo_read_allÚhostÚportr   r   r   Úinit?  s   zStreamReaderServer.initc                 C   s¤   | j  ¡ \}}z@|jdd}tj |¡}|j| _| jr!| 	¡ | _
nt|ƒD ]\}}| j |¡ q%W | ¡  | j  ¡  d S W | ¡  | j  ¡  d S | ¡  | j  ¡  w )NÚrb©Úmode)r,  ÚacceptÚmakefiler#   rA   r¤   r"   r1  r0  ru   r3  rJ   r2  r+   r2   )r   Ú
connectionÚclient_addressr~   rP   r7   r8   r   r   r   ÚrunJ  s"   ýüÿzStreamReaderServer.runc                 C   s   | j | jr	| jfS | jfS r   )r1  r0  r3  r2  r   r   r   r   Ú
get_resultY  s   ÿzStreamReaderServer.get_resultN)r;   r<   r=   r7  r?  r@  r   r   r   r   r'  =  s    r'  c                   @   s4   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ ZdS )ÚSocketStreamFixturec                 C   s   d S r   r   r   r   r   r   r   `  s   zSocketStreamFixture.__init__c                 C   sN   t ƒ | _| j |¡}| j ¡  t tjtj¡| _| j d|f¡ |  	¡ | _
d S )Nr(  )r'  Ú_serverr7  Ústartr)  r*  r+  r,  Úconnectr   r   )r   r4  r6  r   r   r   Ústart_servere  s   
z SocketStreamFixture.start_serverc                 C   sD   dd l }| j | dd¡¡ | j ¡  | j ¡  | j ¡  | j 	¡ S )Nr   ÚQ)
Ústructr   rš   ÚpackÚflushr,  r2   rB  r—   r@  )r   rG  r   r   r   Ústop_and_get_resultm  s   



z'SocketStreamFixture.stop_and_get_resultc                 C   s   | j jddS )Nr–   r9  )r,  r<  r   r   r   r   r   u  s   zSocketStreamFixture.get_sinkc                 C   rY   r   rZ   rC   r   r   r   r%   x  r[   zSocketStreamFixture._get_writerN)r;   r<   r=   r   rE  rJ  r   r%   r   r   r   r   rA  ^  s    rA  c                   C   r\   r   )rA  r   r   r   r   Úsocket_fixture|  r^   rK  c                 C   sp   | j dd |  ¡ }|  ¡ \}}| |d j¡sJ ‚t|ƒt|ƒks$J ‚t|ƒD ]\}}||  |¡s5J ‚q(d S )NF©r4  r   )rE  r:   rJ  rL   r"   rI   rJ   )rK  Úwriter_batchesZreader_schemaZreader_batchesr7   r8   r   r   r   Útest_socket_simple_roundtrip  s   ÿrN  c                 C   s>   | j dd |  ¡ }|  ¡ \}}tj |¡}| |¡sJ ‚d S )NTrL  )rE  r:   rJ  r#   r,   r-   rL   )rK  rM  r  rw   rx   r   r   r   Útest_socket_read_allŒ  s
   rO  c                  C   s`   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }tj 	|dd … ¡}| 
¡ }t| |ƒ d S )Nr   ç      ø?é   )r   r³   r#   r¶   r·   rt   Ú
write_filer   rA   r¤   r   r   )r¸   r8   r   ÚbufferrP   Úrdfr   r   r   Útest_ipc_file_stream_has_eos˜  s   
rU  c                  C   sl   t  ddgi¡} tj | ¡}t ¡ }t||ƒ | ¡ }t |¡}t	|ƒ}|d  
¡ }t  |¡}t| |ƒ d S )Nr   rP  r   )r   r³   r#   r¶   r·   rt   rR  r   rF   Ú	read_filer‰   r   )r¸   r8   r   rS  rP   r6   r  rT  r   r   r   Útest_ipc_zero_copy_numpyª  s   


rW  Úipc_typer%  Úfilec           
         s¢  t  ddgi¡}tj |¡}t ¡ }d}tjjtjjdœ|  }|||j	ƒ}t
|ƒD ]}|j|dt|ƒid q*| |¡ W d   ƒ n1 sGw   Y  | ¡ }| dkrqtj |¡‰ tˆ  ¡ ƒ}	W d   ƒ n1 skw   Y  n"tj |¡‰ ‡ fdd	„t
ˆ jƒD ƒ}	W d   ƒ n1 sŽw   Y  t
|ƒD ]$}|	| jjd
ks£J ‚t|	| jtjƒs®J ‚|	| jdt|ƒiks»J ‚q—|	| jjd
ksÆJ ‚|	| jd u sÏJ ‚d S )Nr   rP  r¨   )r%  rY  Zbatch_id)Úcustom_metadatar%  c                    ó   g | ]}ˆ   |¡‘qS r   )Zget_batch_with_custom_metadata©r‹   r7   ©rP   r   r   rŒ   Ô  s    ÿz?test_batches_with_custom_metadata_roundtrip.<locals>.<listcomp>r§   )r   r³   r#   r¶   r·   rt   rA   rV   rB   r"   r&   r/   Ústrr   r¤   rÎ   Z!iter_batches_with_custom_metadatarG   rH   r8   Znum_rowsrM   rZ  ZKeyValueMetadata)
rX  r¸   r8   r   Zbatch_countZfile_factoryr5   r7   rS  Zbatch_with_metasr   r]  r   Ú+test_batches_with_custom_metadata_roundtrip»  sB   ÿÿüÿ€
ÿÿÿr_  c                  C   sÆ   t jjt  g d¢¡t  g d¢¡gddgd} t  ¡ }t j || j¡ W d   ƒ n1 s-w   Y  | ¡ }t j 	|¡}| 
¡ }W d   ƒ n1 sKw   Y  |j | j¡sYJ ‚t|ƒdksaJ ‚d S )Nrû   rú   rý   rþ   rÚ   r   )r#   r,   rà   r­   rt   rA   rV   r"   r   r¤   ru   rL   rI   )r9   r   r~   rP   rw   r   r   r   Útest_ipc_stream_no_batchesâ  s   ÿþÿ
ÿr`  c                  C   sF   d} d}t  dtj | ¡i¡}tj |¡}tj 	|¡| | ks!J ‚d S )NrÌ   rQ  r   )
r   r³   r(   r)   r*   r#   r¶   r·   rA   Zget_record_batch_size)ÚNÚitemsizer¸   r8   r   r   r   Útest_get_record_batch_sizeô  s
   rc  c                 C   s2   t j| |rdndd}t j||d}t|| ƒ d S )Nr¨   r§   )Znthreadsr¼   )r#   Úserialize_pandasÚdeserialize_pandasr   )r¸   r½   rl   rw   r   r   r   Ú"_check_serialize_pandas_round_tripþ  s   rf  c                  C   sD   t jg d¢dd} ddg}t jg d¢tdƒdœ| |d	}t|ƒ d S )
Nr¦   Úmy_index©Únamer   r   ©rP  gš™™™™™ù?g333333û?Úabc©r   r   ©ÚindexÚcolumns©r   ÚIndexr³   rÎ   rf  ©rn  ro  r¸   r   r   r   Ú test_pandas_serialize_round_trip  s   þrs  c                  C   sH   t jg d¢dd} ddg}t jg d¢tdƒdœ| |d	}t|d
d d S )Nr¦   rg  rh  r   r   rj  rk  rl  rm  Tr¼   rp  rr  r   r   r   Ú)test_pandas_serialize_round_trip_nthreads  s   þrt  c                  C   sf   t jg d¢dd} t jtdƒd d}t j | |g¡}ddg}t jg d¢tdƒd	œ||d
}t|ƒ d S )Nr¦   Zlevel_1rh  Údefr   r   rj  rk  rl  rm  )r   rq  rÎ   Z
MultiIndexrà   r³   rf  )Zindex1Úindex2rn  ro  r¸   r   r   r   Ú,test_pandas_serialize_round_trip_multi_index  s   ýrw  c                  C   s   t  ¡ } t| ƒ d S r   )r   r³   rf  )r¸   r   r   r   Ú%test_serialize_pandas_empty_dataframe*  s   rx  c                  C   s:   t  ttg d¢dƒƒ¡} t | ¡}t |¡}t|| ƒ d S )Nrj  rk  )r   r³   rÎ   r  r#   rd  re  r   )r¸   rl   rw   r   r   r   Ú3test_pandas_serialize_round_trip_not_string_columns0  s   

ry  c                  C   st   t jdg d¢ig d¢d} t  dg d¢i¡}tj| dd}t |¡}t||ƒ tj| dd}t |¡}t|| ƒ d S )Nrý   r¦   )rn  FrÈ   T)r   r³   r#   rd  re  r   )r¸   rx   rl   rw   r   r   r   Ú'test_serialize_pandas_no_preserve_index8  s   


rz  c                  C   sr   d} t  tj | ¡dtjdddgdœ¡}tj |¡}|j	 
¡ }| 
¡ }tj |¡}tj ||¡}| |¡s7J ‚d S )Nr   r   r   r   r    r²   )r   r³   r(   r)   r*   rµ   r#   r¶   r·   r"   r  rA   r  r   rL   )r4   r¸   r8   Ús_schemaZs_batchÚrecons_schemaZrecons_batchr   r   r   Ú#test_schema_batch_serialize_methodsF  s   
þ
r}  c                  C   s¢   dddœ} dddœ}t  dt  ¡ ¡}t jdt  ¡ | d}t j||g|d}| ¡ }t j |¡}| |¡s6J ‚|j	|ks=J ‚|d j	d u sFJ ‚|d	 j	| ksOJ ‚d S )
Ns   bars   field)r"  s   kinds   schemarý   rþ   )r  r   r§   )
r#   r¬   r  Ústringr"   r  rA   r  rL   r  )Zfield_metadataZschema_metadataZf0Úf1r"   r{  r|  r   r   r   Ú'test_schema_serialization_with_metadataV  s   

r€  c                 C   s@   t j || j¡}| | ¡ W d   ƒ d S 1 sw   Y  d S r   )r#   rA   rB   r"   r/   )r8   r   r5   r   r   r   rR  h  s   "ÿrR  c                    sH   t j | ¡‰ ‡ fdd„tˆ jƒD ƒW  d   ƒ S 1 sw   Y  d S )Nc                    r[  r   )rK   r\  r]  r   r   rŒ   o  s    zread_file.<locals>.<listcomp>)r#   rA   rG   r&   rH   )r~   r   r]  r   rV  m  s   $ÿrV  c                  C   s®   t  dt  ¡ fg¡} t  ¡ }t j || ¡ W d   ƒ n1 s w   Y  | ¡ }t  t  |¡¡}| 	¡ }W d   ƒ n1 s@w   Y  t
|ƒdksMJ ‚|j | ¡sUJ ‚d S )Nr¬   r   )r#   r"   Úint64rt   rA   rB   r   r|   rF   ru   rI   rL   )r"   r   rl   rP   r9   r   r   r   Útest_write_empty_ipc_filer  s   ÿ
ÿr‚  c                     sŠ  dd„ ‰ ‡ fdd„} t | ƒ ƒ}t |¡}tj ˆ ƒ |¡!}d }|ƒ d us&J ‚t|ƒ| ƒ ks/J ‚|ƒ d u s6J ‚W d   ƒ n1 s@w   Y  tt | ƒ ƒƒ}t |¡}tj ˆ ƒ |¡!}d }|ƒ d uscJ ‚t|ƒ| ƒ kslJ ‚|ƒ d u ssJ ‚W d   ƒ n1 s}w   Y  | ƒ }t 	t
¡ tj dt ¡ fg|¡}W d   ƒ n1 s¡w   Y  t 	t
¡ tj d |¡}W d   ƒ d S 1 s¾w   Y  d S )Nc                   S   s   t  dt  ¡ fg¡S )Nr¬   )r#   r"   r  r   r   r   r   Úmake_schemaƒ  rD   z0test_py_record_batch_reader.<locals>.make_schemac                     s6   ˆ ƒ } t jg d¢g| d}t jddgg| d}||gS )Nr¦   r!   rü   r   )r#   r'   )r"   Zbatch1Zbatch2©rƒ  r   r   Úmake_batches†  s   z1test_py_record_batch_reader.<locals>.make_batchesr¬   )r   ÚweakrefÚrefr#   ÚRecordBatchReaderr-   rÎ   Úiterrh   ri   rÃ   r  )r…  r6   r¹   rP   r   r„  r   Útest_py_record_batch_reader‚  sF   


ÿû

ÿû	ÿý"þrŠ  c               
   C   s˜  G dd„ dƒ} t jt jg d¢t  ¡ dgdgdt jt jg d¢t  ¡ dgdgdg}| |ƒ}t j |¡}t j |¡}| ¡ |ksCJ ‚t j |¡}| ¡ |ksQJ ‚t jj||d j	d	}| ¡ |ksdJ ‚t
 t¡ t jj|t  	t  dt  ¡ ¡g¡d	 W d   ƒ n1 s†w   Y  t
 t¡ t j |d d ¡ W d   ƒ n1 s¥w   Y  t
 t¡ t jj||d d	 W d   ƒ d S 1 sÅw   Y  d S )
Nc                   @   s   e Zd Zdd„ Zddd„ZdS )zAtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapperc                 S   s
   || _ d S r   )r6   )r   r6   r   r   r   r   ²  r   zJtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapper.__init__Nc                 S   s"   t j | jd j| j¡}| |¡S )Nr   )r#   rˆ  r-   r6   r"   Ú__arrow_c_stream__)r   Zrequested_schemarP   r   r   r   r‹  µ  s   ÿ
zTtest_record_batch_reader_from_arrow_stream.<locals>.StreamWrapper.__arrow_c_stream__r   )r;   r<   r=   r   r‹  r   r   r   r   ÚStreamWrapper±  s    rŒ  r¦   rª   rý   rÚ   )rü   r   r{   r   r!   )r#   r'   r­   r  r,   r-   rˆ  Úfrom_streamru   r"   rh   ri   ÚNotImplementedErrorr¬   r  rÃ   )rŒ  r  Úwrapperrx   rP   r   r   r   Ú*test_record_batch_reader_from_arrow_stream¯  s.   "
"þÿÿÿ"ÿr  rR   )dÚcollectionsr   r
   r›   rh   r)  Ú	threadingr†  Únumpyr(   Zpyarrowr#   Zpyarrow.tests.utilr   r   Zpandas.testingr   Zpandasr   ÚImportErrorr   r>   rT   rX   rv   r]   r_   r`   rc   re   rm   rq   rs   ÚmarkZparametrizery   r†   r”   r£   r¥   r®   rº   r»   rA   rø   rÀ   rÄ   rÆ   rÇ   rÊ   rÒ   rØ   rÜ   rä   rï   rò   rá   rê   rë   rõ   rù   rÿ   r  r	  r
  r  r  r  r  r  r  r  r  r!  r&  ÚThreadr'  rA  rK  rN  rO  rU  rW  r_  r`  rc  rf  rs  rt  rw  rx  ry  rz  r}  r€  rR  rV  r‚  rŠ  r  r   r   r   r   Ú<module>   s  ÿ'


þþû

þ



þ
þ




1	ÿû
K$




!


%
	








-