o
    Zh                    @   s  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
m	Z	mZmZmZ d dlZd dlmZ d dlZd dlmZ d dlZd dlmZmZ d dlmZmZmZ d dl m!  mZ" d dlm!  m#Z$ d dl%m&Z& d dl'Z(zd dl'm)Z* W n	 e+y   Y nw zd dl,Z-d dl.mZ/ dd	l0m1Z1m2Z2 W n	 e+y   Y nw zej3Z4W n e5y   d d
l6m3Z4 Y nw ej7j,Z8dddZ9					dddZ:dddZ;		dddZ<dddZ=G dd dZ>G dd dZ?ej7@dg ddd ZAej7@dg dd d! ZBG d"d# d#ZCG d$d% d%ZDG d&d' d'ZEG d(d) d)ZFG d*d+ d+ZGG d,d- d-ZHd.d/ ZId0d1 ZJG d2d3 d3ZKd4d5 ZLd6d7 ZMej7@d8d9gd:gfd;d< ZNd=d> ZOd?d@ ZPdAdB ZQeRe"Se"jTdCdD ZUdEdF ZVdGdH ZWdIdJ ZXdKdL ZYdMdN ZZdOdP Z[dQdR Z\dSdT Z]dUdV Z^dWdX Z_dYdZ Z`d[d\ Zad]d^ Zbd_d` Zcdadb Zddcdd Zededf Zfdgdh Zgdidj Zhej7@dke(i dlfe(j dmfe(k dnfe(l dofe(m dpfe(n dqfe(o drfe(p dsfe(q dtfe(r dufe(s dvfe(t dwfe(u dxfe(v dyfe(w dyfe(x dzfe(jxd{d|dzfe(y d}fe(ze(ze(l d~fe({dddfe(|ddfe(|dddfe(}ddfe(~ddfgdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd	ddZdd Zdd Zdd Zdd Zdd Zdd ZG dd de(jZdddZdd Zdd Zdd ZG dd de(jZddĄ ZddƄ ZddȄ Zddʄ Zej7@ddddggddτ Zddф Zej7j,ddӄ ZddՄ Zddׄ Zddل Zddۄ Zdd݄ Zej7j)ej7dޡdd Zdd Zej7@dg dej7@dddgej7@de	ddde	dddgdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zej7jd d Zdd ZdS (
      N)OrderedDict)datedatetimetime	timedeltatimezone)get_logical_type_pandas_api)invoke_scriptrandom_asciirands)Version)parquet   )dataframe_with_arraysdataframe_with_lists)VisibleDeprecationWarningd   c                 C   s  t i dtj| tjddtj| tjddtj| tjddtj| tjddtj| tjddtj| tjddtj| tj	dd	tj| tj
dd
tj| tjddtj| tjddtj| dkdtjd| dddtjd| dddtjd| dddtjd| dddtjd| dddtjd| ddtjd| ddtjd| dddd t| D d gdd t| d D  d g dg|  d S )!Nuint8dtypeuint16uint32uint64int8int16int32int64float32float64boolr   zdatetime[s]2016-01-01T00:00:00.001datetime64[s]zdatetime[ms]datetime64[ms]zdatetime[us]datetime64[us]zdatetime[ns]datetime64[ns]ztimedelta64[s]ztimedelta64[ms]timedelta64[us]timedelta64[ns]c                 S      g | ]}t |qS  str.0xr)   r)   P/var/www/html/lang_env/lib/python3.10/site-packages/pyarrow/tests/test_pandas.py
<listcomp>[       z%_alltypes_example.<locals>.<listcomp>c                 S   r(   r)   r*   r,   r)   r)   r/   r0   \   r1       )r&   r'   r+   Zstr_with_nullsZ	empty_str)pd	DataFramenparanger   r   r   r   r   r   r   r   r   randomrandnrangesizer)   r)   r/   _alltypes_exampleB   sb   	




 
r=   FTc                 C   s   |rt jnt j}|j| |||rdndd}	|	j|d}
|r%|	j|s%J |d u r+| }t  t	ddt
 tj|
|||r@dndd	 W d    d S 1 sOw   Y  d S )
Nr2   r   )schemapreserve_indexZnthreadsuse_threadsignoreelementwise comparison failedequivF)check_dtypeZcheck_index_type)paRecordBatchTablefrom_pandas	to_pandasr>   equalswarningscatch_warningsfilterwarningsDeprecationWarningtmassert_frame_equal)dfexpectedrA   expected_schemarE   r>   r?   as_batchklasstableresultr)   r)   r/   _check_pandas_roundtripa   s&   


"rY   c                 C   s^   t j| d|d}|d ur|d u r|}|d ur|j|ksJ tj| | jd}t| | d S )NTrI   typename)	rF   arrayr[   r4   SeriesrJ   r]   rP   assert_series_equal)stype_expected_pa_typearrrX   r)   r)   r/   _check_series_roundtrip|   s   re   c                 C   s   t j| d||d}| }t| }|d u r |j| ksJ n|j||B  ks+J |d u rF|d u r9t| }nt|  }d || < t	j
t||dd d S )NT)rI   maskr[   FZcheck_names)rF   r^   rJ   r4   isnull
null_countsumr_   copyrP   r`   )valuesrS   rf   r[   rd   rX   Zvalues_nullsr)   r)   r/   _check_array_roundtrip   s   
rm   c                 C   s(   t j| d|d}| }t||  d S )NTrZ   rF   r^   rJ   nptassert_array_equal)Znp_arrayr[   rd   rX   r)   r)   r/   "_check_array_from_pandas_roundtrip   s   rq   c                   @   s0  e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.d/ Zd0d1 Zd2d3 Zd4d5 Zd6d7 Zd8d9 Zd:d; Z d<d= Z!d>d? Z"d@dA Z#dBdC Z$dDdE Z%dFdG Z&dHdI Z'dJS )KTestConvertMetadataz9
    Conversion tests for Pandas metadata & indices.
    c                 C   s6   t dg di}tj|}|djdksJ d S )Nr   r   r2      0)r4   r5   rF   rH   rI   fieldr]   selfrR   rW   r)   r)   r/   test_non_string_columns   s   z+TestConvertMetadata.test_non_string_columnsc                 C   s   t g dg dd}|d}tt tj|}|	dj
dks'J W d    n1 s1w   Y  | }t|jj
|j_
tt t||dd W d    d S 1 sZw   Y  d S )N      ?       @      @)      @      @      @)r   r   r   1TrS   r?   )r4   r5   	set_indexpytestwarnsUserWarningrF   rH   rI   rv   r]   rk   r+   indexrY   )rx   rR   rW   rS   r)   r)   r/   "test_non_string_columns_with_index   s   
"z6TestConvertMetadata.test_non_string_columns_with_indexc                 C   s   t jg dg dg ddddgd}tjj|ddgd}tj|ddg }||s.J tjj|ddgd}tj|ddg }||sIJ d S )Nrs   )r   rt   rt   )r2         r   r   r2   r   r   columns)r4   r5   rF   rH   rI   rK   rG   )rx   rR   rW   rS   Zrecord_batch_tableZrecord_batch_expectedr)   r)   r/   test_from_pandas_with_columns   s   z1TestConvertMetadata.test_from_pandas_with_columnsc                 C   s,   t dg di}dg|j_t|dd d S )Ndatars   aTr?   )r4   r5   r   namesrY   rx   rR   r)   r)   r/   %test_column_index_names_are_preserved   s   
z9TestConvertMetadata.test_column_index_names_are_preservedc                 C   s6   t jtjddt jdddddd}t|dd	 d S )
Nr   rt   z
2021-01-01Z50DZCET)periodsfreqtzr   Tr   )r4   r5   r6   r8   r9   
date_rangerY   r   r)   r)   r/   test_column_index_names_with_tz   s
   z3TestConvertMetadata.test_column_index_names_with_tzc                 C   s  d}t jdg dit jddd|dd}t jdg d	it dd
d}tj|}tj|}t|jdks8J | }t	
|| t|jt jsKJ t|jddksVJ |jj|ks^J | }t	
|| t|jt jsqJ t|jddks|J |jjd u sJ d S )Nfoor   r   r2   rt   r   r      r2   stepr]   r   r   r         r   r   r   )r4   r5   
RangeIndexrF   rH   rI   lenr>   rJ   rP   rQ   
isinstancer   r	   Zget_rangeindex_attributer]   )rx   Z
index_namerR   df2rW   Ztable_no_index_namerX   result2r)   r)   r/   test_range_index_shortcut   s(   
z-TestConvertMetadata.test_range_index_shortcutc                 C   sh   t jdg dit jdddddd}tjj|d	d
}|jdks!J d|jv s(J | }t	
|| d S )Nr   r   r   r   r2   r   r   r   Tr   )r4   r5   r   rF   rH   rI   Znum_columnsZcolumn_namesrJ   rP   rQ   )rx   rR   rW   restoredr)   r)   r/   $test_range_index_force_serialization   s   z8TestConvertMetadata.test_range_index_force_serializationc                 C   sp   t jtjddddgd}t  tjdd tdd	t	 t
|d
d W d    d S 1 s1w   Y  d S )Nr   r2   r   br   erroractionrB   make_block is deprecatedTr   )r4   r5   r6   r8   r9   rL   rM   simplefilterrN   rO   rY   r   r)   r)   r/   test_rangeindex_doesnt_warn  s   
"z/TestConvertMetadata.test_rangeindex_doesnt_warnc                 C   :   t jddgddgg}t jg d|d}t|dd d S )	NonetwoXYr   r   r2   r   )rt   cr   Tr   r4   
MultiIndexfrom_arraysr5   rY   rx   r   rR   r)   r)   r/   test_multiindex_columns  s
   z+TestConvertMetadata.test_multiindex_columnsc                 C   sH   t jjddgt ddggddgd}t jg d|d	}t|d
d d S )Nr   r   z
2017-08-01z
2017-08-02Zlevel_1Zlevel_2r   r   r   Tr   )r4   r   r   DatetimeIndexr5   rY   r   r)   r)   r/   #test_multiindex_columns_with_dtypes  s   z7TestConvertMetadata.test_multiindex_columns_with_dtypesc                 C   s   t jdgt jdgtdd}t|dd t jdgt jdgtdd}t|dd t jdgt jtdddgddd}t|dd d S )	Nr   r   r   Tr   皙?  object)r4   r5   Indexr   rY   r   r   r)   r)   r/   (test_multiindex_with_column_dtype_object#  s   
z<TestConvertMetadata.test_multiindex_with_column_dtype_objectc                 C   r   )	N   あ   いr   r   r   r   Tr   r   r   r)   r)   r/   test_multiindex_columns_unicode5  s   z3TestConvertMetadata.test_multiindex_columns_unicodec                 C   s~   t jddgddgg}t jg d|d}t  tjdd td	d
t t	|dd W d    d S 1 s8w   Y  d S )Nr   r   r   r   r   r   r   r   rB   r   Tr   )
r4   r   r   r5   rL   rM   r   rN   rO   rY   r   r)   r)   r/   test_multiindex_doesnt_warn:  s   
"z/TestConvertMetadata.test_multiindex_doesnt_warnc                 C   s   t g d}t|dd d S )Nr   Tr   r4   r5   rY   r   r)   r)   r/   test_integer_index_columnI  s   z-TestConvertMetadata.test_integer_index_columnc                 C   sX  t jg dt jjg dg dgd dgdg dd}tt tjj	|dd	}W d    n1 s2w   Y  |j
j}|d
 \}}}}}|d dksLJ |d |d ksVJ |d d u s^J |d dksfJ |d dksnJ |d |d ksxJ |d \}	}
|	dksJ |d |	ksJ |d d u sJ |
dksJ |d |
ksJ |d |
ksJ d S )N))r   r   @)r2   r   g@)rt   r   g?)r   r   r   )rt   r2   r   r   r   )r   N__index_level_0__)r   r   Tr   r   r]   r   
field_nameNoner   index_columns)r4   r5   r   r   r   r   r   rF   rH   rI   r>   pandas_metadata)rx   rR   tjscol1Zcol2Zcol3Zidx0r   Z
idx0_descrZ	foo_descrr)   r)   r/   test_index_metadata_field_nameM  s4   z2TestConvertMetadata.test_index_metadata_field_namec                 C   s   t jg dt jtdddd}tjj|dd}|jj}|d \}|d	 d u s)J |d
 dks1J |d dks9J |d }|d dksEJ |d du sMJ d S )N)r   r   r|   )r2   r   r}   )rt   r   r~   defcategoryr   r   Tr   column_indexesr]   pandas_typeZcategorical
numpy_typer   metadataZnum_categoriesrt   orderedF)	r4   r5   r   listrF   rH   rI   r>   r   rx   rR   r   r   r   mdr)   r)   r/   test_categorical_column_indexo  s   
z1TestConvertMetadata.test_categorical_column_indexc                 C   s   t jg dt jtdddd}tjj|dd}|jj}|d \}|d	 dks)J |d	 |d
 ks3J |d dks;J |d dksCJ |d }t	|dksOJ |d dksWJ d S )Nr   r   Zstringzr\   r   Tr   r   r]   r   r   r   r   unicoder   r   encodingUTF-8)
r4   r5   r   r   rF   rH   rI   r>   r   r   r   r)   r)   r/   test_string_column_index  s   
z,TestConvertMetadata.test_string_column_indexc                 C   s   t jg dt jddddd}tjj|dd}|jj}|d	 \}|d
 d u s(J |d dks0J |d dks8J |d }|d dksDJ d S )Nr   
2017-01-01rt   America/New_Yorkstartr   r   r   Tr   r   r]   r   
datetimetzr   r%   r   r   )r4   r5   r   rF   rH   rI   r>   r   r   r)   r)   r/   test_datetimetz_column_index  s   
z0TestConvertMetadata.test_datetimetz_column_indexc                 C   s4   t dt jddddi}|d}t|dd d S )Nr   r   rt   r   r   Tr   )r4   r5   r   r   rY   r   r)   r)   r/   test_datetimetz_row_index  s   
z-TestConvertMetadata.test_datetimetz_row_indexc                 C   sB   t g dg dd}|jd|d< |d}t|dd d S )Nrs   r   r   r   r   Tr   )r4   r5   r   astyper   rY   r   r)   r)   r/   test_categorical_row_index  s   
z.TestConvertMetadata.test_categorical_row_indexc                 C   sR   t jddgtdd}tt tj| W d    d S 1 s"w   Y  d S )Nr   r   aar   )	r4   r5   r   r   raises
ValueErrorrF   rH   rI   r   r)   r)   r/   *test_duplicate_column_names_does_not_crash  s   "z>TestConvertMetadata.test_duplicate_column_names_does_not_crashc              	   C   s   ddgddgg}|D ]P}t jj|dgdd}t j|gdg}t j|||g}tt |	  W d    n1 s<w   Y  tt |	  W d    n1 sUw   Y  q
d S )Nr   r   r   Fsafer   )
rF   DictionaryArrayr   rG   rH   from_batchesr   r   
IndexErrorrJ   )rx   indicesZindsrd   batchrW   r)   r)   r/   #test_dictionary_indices_boundscheck  s   

z7TestConvertMetadata.test_dictionary_indices_boundscheckc                 C   s&   t jddgidgd}t|dd d S )Nr   r   u   うr   Tr   r   r   r)   r)   r/   *test_unicode_with_unicode_column_and_index  s   z>TestConvertMetadata.test_unicode_with_unicode_column_and_indexc              	   C   s   t ddgddgd}ddgddgdd	gfD ]M}t j|td
|_tt tj	
| W d    n1 s7w   Y  | }|jjt|_tt t||dd W d    n1 s_w   Y  qd S )Nr   r2   rt   r   r   r      a2      ?r   Tr   )r4   r5   r   r   r   r   r   r   rF   rH   rI   rk   rl   r   r+   rY   )rx   rR   colsrS   r)   r)   r/   test_mixed_column_names  s   z+TestConvertMetadata.test_mixed_column_namesc                 C   s   t dt tj  krt dk rn ntd dg}d }||i}t|}tjj	|dd}|
 }|jd |jd ks@J |jjd |jjd ksNJ |jd |ksWJ d S )	N2.0.02.3.0Regression in pandas 2.0.0r   r   Tr   r   )r   r4   __version__r   skipencoder5   rF   rH   rI   rJ   rl   r   r   )rx   Zcolumn_datakeyr   rR   r   r   r)   r)   r/   test_binary_column_name  s   &

z+TestConvertMetadata.test_binary_column_namec                 C   sN   d}t t|}tjjg d|gddgd}tjd|i|d}t|dd	 d S )
Nrt   )r   r   barZfoobarZsome_numbersr   numbersr   Tr   )r   r:   r4   r   r   r5   rY   )rx   Znum_rowsr  r   rR   r)   r)   r/    test_multiindex_duplicate_values  s   
z4TestConvertMetadata.test_multiindex_duplicate_valuesc                 C   sb   t dddgi}tj|}|jj}d|vsJ |d d }|d dks'J |d	 d
ks/J d S )Nr   s
   some_bytesZsome_unicodemixedr   r   r   bytesr   r   r4   r5   rF   rH   rI   r>   r   )rx   rR   rW   r   data_columnr)   r)   r/   test_metadata_with_mixed_types  s   z2TestConvertMetadata.test_metadata_with_mixed_typesc                 C   s\   t jg dg ddg dd}tj|}|jdd}||j  }t	
|| d S )Nrs   r   r  bazr   )r   r   threer   T)Zignore_metadata)r4   r5   rF   rH   rI   rJ   castr>   remove_metadatarP   rQ   )rx   rR   rW   rX   rS   r)   r)   r/   test_ignore_metadata  s   z(TestConvertMetadata.test_ignore_metadatac                 C   s   t ddgg ddgd gi}ttjdtt dg}tjj||d}|jj	}d|vs2J |d	 d
 }|d dks@J |d dksHJ d S )Nr   r   r2   rt   r   r   r   r[   r>   r  r   r   r   zlist[int64]r   r   )
r4   r5   rF   r>   rv   list_r   rH   rI   r   )rx   rR   r>   rW   r   r  r)   r)   r/   test_list_metadata  s     z&TestConvertMetadata.test_list_metadatac                 C   sN   t dddddddgi}tj|}|jj}|d d d	 d
ks%J d S )Ndictsr   r2   r   rt   r   r   r   r   r   r  )rx   rR   rW   r   r)   r)   r/   test_struct_metadata  s   z(TestConvertMetadata.test_struct_metadatac                 C   s   t dtdtd gi}tj|}|jj}d|vs J |d d }|d dks.J |d	 d
ks6J |d dddksAJ d S )Ndecimals394092382910493.12341234678z314292388910493.12343437128r  r   r   r   decimalr   r   r         )	precisionscale)	r4   r5   r$  DecimalrF   rH   rI   r>   r   )rx   rS   rW   r   r  r)   r)   r/   test_decimal_metadata#  s   
z)TestConvertMetadata.test_decimal_metadatac                 C   s   t jg dddt jddddfD ]K}t jg dg d	d
|d}tj|}|d}| }|dg }t	|j
t jr@d |j
_t|| |d}| }t||dg jdd qd S )Nr   r   r   r   r\   r   rt   Europe/Brusselsr   r   rs   皙?皙?g333333?r   r   r   r   TZdrop)r4   r   r   r5   rF   rH   rI   Zremove_columnrJ   r   r   r   r   rP   rQ   reset_index)rx   r   rR   rW   Ztable_subsetrX   rS   Ztable_subset2r)   r)   r/   !test_table_column_subset_metadata2  s&   



z5TestConvertMetadata.test_table_column_subset_metadatac                 C   sv   t ttdttdtdd}tj|	ddg}|
ddg}| }|ddg 	d}t|| d S )Nr   )firstsecondvaluer4  r5  r6  )r4   r5   r   r:   r6   r7   rF   rH   rI   r   selectrJ   rP   rQ   )rx   rR   rW   ZsubsetrX   rS   r)   r)   r/   'test_to_pandas_column_subset_multiindexG  s   

z;TestConvertMetadata.test_to_pandas_column_subset_multiindexc              	   C   s   dgddgd g}g g g g}t dtj|tt dfdtj|tt dfg}tjt| t|	 }tj
|g}| }tj
|}|jj}| }	tt d|fd|fg}
t|	|
 |d ddd dd	d
ddd ddd
gks|J d S )Ntestr   r   c1r  c2r   r   zlist[unicode])r]   r   r   r   r   zlist[empty])r   rF   r^   r  stringrG   r   r   rl   keysrH   r   rJ   rI   r>   r   r4   r5   rP   rQ   )rx   r:  r;  arraysrbtblrR   Ztbl2Zmd2r   rS   r)   r)   r/   test_empty_list_metadataU  s>   


z,TestConvertMetadata.test_empty_list_metadatac                 C   s<   t g dg dd}tj|}|jjd d usJ d S )Nrs   r   pandas_versionr  rw   r)   r)   r/   test_metadata_pandas_version  s   z0TestConvertMetadata.test_metadata_pandas_versionc           
      C   s   t dt jdddi}tj|}|d tjddd}|dt	d|j
|}tdtjdddfg}tjj||d	}| }|d jd|d< ||fD ]}|jjd us[J | }	t|	| qQd S )
Nr   z
2020-01-01rt   r   nsUTCr   r   r  )r4   r5   r   rF   rH   rI   r  	timestampZ
set_columnrv   r[   r>   rk   dttz_localizer   rJ   rP   rQ   )
rx   rR   rW   Znew_colZ
new_table1r>   Z
new_table2rS   Z	new_tablerX   r)   r)   r/   test_mismatch_metadata_schema  s   z1TestConvertMetadata.test_mismatch_metadata_schemaN)(__name__
__module____qualname____doc__ry   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r  r  r!  r*  r3  r8  rA  rC  rK  r)   r)   r)   r/   rr      sL    
"
	
,rr   c                   @   s   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zd,d- Zd.S )/TestConvertPrimitiveTypesz>
    Conversion tests for primitive (e.g. numeric) types.
    c           
      C   s   i }g }dt  fdt  fdt  fg}d}|D ]\}}tj|}||||< |t 	|| qt
|}t |}	t||	d d S )Nf2f4f8r   rT   )rF   float16r   r   r6   r8   r9   r   appendrv   r4   r5   r>   rY   )
rx   r   fieldsdtypes
num_valuesZnumpy_dtypearrow_dtyperl   rR   r>   r)   r)   r/   test_float_no_nulls  s   




z-TestConvertPrimitiveTypes.test_float_no_nullsc                 C   s  d}t jjdd|ddk }dt fdt fdt fg}g d	}g }g }g }|D ]-\}}	t j||}
tj	|
d
|d}|
| |
t||	 t j|
|< |
|
 q*tjtt|||d}tj||}|jt|svJ | }t|| d S )Nr   r   
   r;   rt   rQ  rR  rS  )rQ  rR  rS  T)rI   rf   r   )r6   r8   randintrF   rU  r   r   r9   r   r^   rV  rv   nanr4   r5   dictziprH   r   r>   rK   rJ   rP   rQ   )rx   rY  	null_maskrX  r   expected_colsr>  rW  r]   rZ  rl   rd   ex_framerW   rX   r)   r)   r/   test_float_nulls  s0   




z*TestConvertPrimitiveTypes.test_float_nullsc                 C   sp   t dddtjgi}ttjdt ddg}tjj	||dd}|d 
 g d	ks.J t||  d S )
Nr   r{   r|   TnullableFr>   r   r   r   r2   N)r4   r5   r6   r^  rF   r>   rv   r   rH   rI   	to_pylistrP   rQ   rJ   rx   rR   r>   rW   r)   r)   r/   test_float_nulls_to_ints  s
   z2TestConvertPrimitiveTypes.test_float_nulls_to_intsc                 C   s2   t g d}t g d}t||t d d S )N)        r{   r|   Ng      )FTTNTrS   r[   )r4   r_   rm   rF   bool_rx   ra   rS   r)   r)   r/   test_float_nulls_to_boolean  s   z5TestConvertPrimitiveTypes.test_float_nulls_to_booleanc                 C   sF   t dtjg}tj|dd}|jdksJ t|d  s!J d S )Nrl  FrI   r   r   )	r4   r_   r6   r^  rF   r^   ri   isnanas_py)rx   ra   rd   r)   r)   r/   'test_series_from_pandas_false_respected  s   zATestConvertPrimitiveTypes.test_series_from_pandas_false_respectedc                 C   s  t  }g }dt fdt fdt fdt fdt fdt fdt fdt	 fd	t fd
t	 fg
}d}|D ]4\}}t
|}t
jjt|jt
t
jjt|jt
t
jj|d}||||< |t|| q=t|}	t|}
t|	|
d d S )Ni1i2i4i8u1u2u4u8longlong	ulonglongr   r;   rT  )r   rF   r   r   r   r   r   r   r   r   r6   Ziinfor8   r]  maxminint_r   rV  rv   r4   r5   r>   rY   )rx   r   rW  numpy_dtypesrY  r   rZ  inforl   rR   r>   r)   r)   r/   test_integer_no_nulls  s(   


z/TestConvertPrimitiveTypes.test_integer_no_nullsc                 C   sl   t  }g d}|D ]}tjd|d||< q	t|}t| | D ]}t|}|	 |
 ks3J q"d S )N)ru  rv  rw  rx  ry  rz  r{  r|  byteZubyteZshortZushortZintcZuintcr  Zuintr}  r~     r   )r   r6   r7   r4   r5   rY   rl   rF   r^   ri  tolist)rx   r   r  r   rR   np_arrrd   r)   r)   r/   test_all_integer_types  s   

z0TestConvertPrimitiveTypes.test_all_integer_typesc                 C   s   g d}|D ]L}dD ]G}t jg d|| d}||d d d fD ]0}|jjr6t|}| | ks5J q tt	 t|}W d    n1 sKw   Y  q q
qd S )Nru  rv  rw  rx  ry  rz  r{  r|  z=<>)r   r2   *   r   r2   )
r6   r^   r   ZisnativerF   ri  r  r   r   NotImplementedError)rx   
int_dtypesrI  orderr   r  rd   r)   r)   r/   test_integer_byteorder  s   
z0TestConvertPrimitiveTypes.test_integer_byteorderc                 C   s   g d}d}t jjdd|ddk }g }g }|D ]&}t jjdd|d}tj||d}|| |d}	t j|	|< ||	 qtj	t
t|||d	}
tj||}| }t||
 d S )
Nr  r   r   r\  r;   rt   rf   rS  r   )r6   r8   r]  rF   r^   rV  r   r^  r4   r5   r_  r`  rH   r   rJ   rP   rQ   )rx   r  rY  ra  rb  r>  r]   rl   rd   rS   rc  rW   rX   r)   r)   r/   test_integer_with_nulls$  s$   


z1TestConvertPrimitiveTypes.test_integer_with_nullsc                 C   sF   t jddd}t }tj||d}t|d}||s!J d S )Nr\  r   r   r  r   )r6   r7   rF   r   r^   r   rK   )rx   rd   Ztarget_typerX   rS   r)   r)   r/    test_array_from_pandas_type_castA  s
   z:TestConvertPrimitiveTypes.test_array_from_pandas_type_castc                 C   sV   d}t jd tdt j|dki}tdt }t	|g}t
||d d S )Nr   r   boolsrT  )r6   r8   seedr4   r5   r9   rF   rv   rn  r>   rY   )rx   rY  rR   rv   r>   r)   r)   r/   test_boolean_no_nullsJ  s   z/TestConvertPrimitiveTypes.test_boolean_no_nullsc                 C   s   d}t jd t jjdd|ddk }t jjdd|ddk }tj||d}|t}d ||< tdt	 }t
|g}td|i}tj|gdg}	|	j
|sTJ |	 }
t|
| d S )	Nr   r   r\  r;   rt   r   r  r  )r6   r8   r  r]  rF   r^   r   r   rv   rn  r>   r4   r5   rH   r   rK   rJ   rP   rQ   )rx   rY  rf   rl   rd   rS   rv   r>   rc  rW   rX   r)   r)   r/   test_boolean_nullsT  s   
z,TestConvertPrimitiveTypes.test_boolean_nullsc                 C   :   t g dd }t g dd }t||t d d S )NTTFTTr2   r   r   r   r   r   rm  )r4   r_   rm   rF   r   ro  r)   r)   r/   test_boolean_to_intk     z-TestConvertPrimitiveTypes.test_boolean_to_intc                 C   st   t jg dd td}t g dd }d}tjtj|d t||t d W d    d S 1 s3w   Y  d S )Nr  r2   r   r  zExpected integer, got boolmatchrm  )	r4   r_   r   r   r   rF   ArrowTypeErrorrm   r   )rx   ra   rS   Zexpected_msgr)   r)   r/   test_boolean_objects_to_intq  s   "z5TestConvertPrimitiveTypes.test_boolean_objects_to_intc                 C   r  )N)TTFNTr2   )r{   r{   rl  Nr{   rm  )r4   r_   rm   rF   r   ro  r)   r)   r/   test_boolean_nulls_to_floaty  r  z5TestConvertPrimitiveTypes.test_boolean_nulls_to_floatc                 C   s(   t jtjdddddgd}t| d S )N)rt   r2   r    r   r   r   r   )r4   r5   r6   onesrY   r   r)   r)   r/   test_boolean_multiple_columns  s   z7TestConvertPrimitiveTypes.test_boolean_multiple_columnsc                 C   n   t jd dt dgd td}td|i}tdt|i}tdt }t	|g}t
|||d d S )Nr  g      @r   r   floatsrS   rT   )r6   r^   r   r   r4   r5   
to_numericrF   rv   r>   rY   rx   rd   rR   rS   rv   r>   r)   r)   r/   test_float_object_nulls     
z1TestConvertPrimitiveTypes.test_float_object_nullsc              	   C   s   t tjddtjg}t t t t t	 t
 t t g}|D ]?}tj||d}tjg d|d}||s@J t d|i}ttd|g}tjj||dd}|d d|sfJ q'd S )	Nr{   r|   r  )Nr   r2   NZ	has_nullsFr>   r?   r   )r4   r_   r6   r^  rF   r   r   r   r   r   r   r   r   r^   rK   r5   r>   rv   rH   rI   chunk)rx   ra   typestyrX   rS   rR   r>   r)   r)   r/   test_float_with_null_as_integer  s   
z9TestConvertPrimitiveTypes.test_float_with_null_as_integerc                 C   r  )Nr   rt   r   r   intsr  )r6   r^   r   r   r4   r5   r  rF   rv   r>   rY   r  r)   r)   r/   test_int_object_nulls  r  z/TestConvertPrimitiveTypes.test_int_object_nullsc                 C   sP   t jg dd td}td|i}tdt }t|g}t	||d d S )N)FNTr   r   r  rT  )
r6   r^   r   r4   r5   rF   rv   rn  r>   rY   )rx   rd   rR   rv   r>   r)   r)   r/   test_boolean_object_nulls  s
   z3TestConvertPrimitiveTypes.test_boolean_object_nullsc                    s8   t jd gtd  fdd}|t  |t  d S )Nr   c                    s4   t j | d}|j| ksJ |d  d u sJ d S )Nr  r   )rF   r^   r[   rs  )r   a2rd   r)   r/   _check_type  s   zJTestConvertPrimitiveTypes.test_all_nulls_cast_numeric.<locals>._check_type)r6   r^   r   rF   r   r   )rx   r  r)   r  r/   test_all_nulls_cast_numeric  s   z5TestConvertPrimitiveTypes.test_all_nulls_cast_numericc                 C   s   t jdt jgt jd}tj|t d}| \}}t|t js"J |dks(J t|t js0J t |s7J tj|t dd}| \}}t|t jsOJ |dksUJ |d u s[J d S )Nr  r   r  Tr[   rI   )r6   r^   r^  rU  rF   ri  r   rr  )rx   rd   r   r.   yr)   r)   r/   test_half_floats_from_numpy  s   z5TestConvertPrimitiveTypes.test_half_floats_from_numpyN)rL  rM  rN  rO  r[  rd  rk  rp  rt  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r)   r)   r)   r/   rP    s0    	
		rP  r   r  c                 C   sz   d}t jjdd|ddk }t jjdd|| d}tj||d}| r,|d}d ||< n|}|jd	d
}t j	|| d S Nr   r   r\  r;   rt   r<   r   r  OTinteger_object_nulls)
r6   r8   r]  rF   r^   anyr   rJ   testingZassert_equal)r   rY  ra  rl   r^   rS   rX   r)   r)   r/   &test_array_integer_object_nulls_option  s   

r  c                 C   s   d}t jjdd|ddk }t jjdd|| d}tj||d}| r,|d}d ||< n|}t| |i}tj	
|g| g}|jd	d
}t|| d S r  )r6   r8   r]  rF   r^   r  r   r4   r5   rH   r   rJ   rP   rQ   )r   rY  ra  rl   r^   rS   rW   rX   r)   r)   r/   &test_table_integer_object_nulls_option  s   

r  c                   @   s  e Zd ZdZdd Zdd Zejdg ddd	 Z	d
d Z
dd Zee ejB ejdddd Zdd Zdd Zdd Zdd Zejddeg dgdd Zejddd gd!d" Zejd#dd gd$d% Zejd&e e e d'e d(e d)e d*e d'd+e d(d+e d)d+e d*d+g
d,d- Z!ejd&e e e d'e d(e d)e d*e d'd+e d(d+e d)d+e d*d+g
d.d/ Z"d0d1 Z#d2d3 Z$d4d5 Z%d6d7 Z&d8d9 Z'd:d; Z(d<d= Z)d>d? Z*d@dA Z+dBdC Z,dDdE Z-ejdFe e gdGdH Z.dIdJ Z/ejj0e1dKe1ej2  ko2e1dLk n  dMdNdOdP Z3ejdg ddQdR Z4ejdg ddSdT Z5dUdV Z6dS )WTestConvertDateTimeLikeTypeszQ
    Conversion tests for datetime- and timestamp-like types (date64, etc.).
    c                 C   J   t dtjg dddi}tdtd}t|g}t||d d S )N
datetime64)2007-07-13T01:23:34.1234567892006-01-13T12:34:56.4325397842010-08-13T05:46:57.437699912r%   r   rE  rT  	r4   r5   r6   r^   rF   rv   rH  r>   rY   rx   rR   rv   r>   r)   r)   r/   #test_timestamps_notimezone_no_nulls     
z@TestConvertDateTimeLikeTypes.test_timestamps_notimezone_no_nullsc                 C   r  )Nr  )r  Nr  r%   r   rE  rT  r  r  r)   r)   r/    test_timestamps_notimezone_nulls
  r  z=TestConvertDateTimeLikeTypes.test_timestamps_notimezone_nullsunit)ra   msusrE  c                 C   s   t tjt dk r|dkrtd tdtjg dd| ddi}|d j	d	|d< t
| t|d  tdtjg d
d| ddi}|d j	d	|d< t
| d S )Nr  rE  z0pandas < 2.0 only supports nanosecond datetime64r  )2007-07-13T01:23:34.1232006-01-13T12:34:56.4322010-08-13T05:46:57.437zdatetime64[]r   
US/Easternr  Nr  r  )r   r4   r  r   r	  r5   r6   r^   rI  rJ  rY   re   )rx   r  rR   r)   r)   r/   test_timestamps_with_timezone  s"   


z:TestConvertDateTimeLikeTypes.test_timestamps_with_timezonec                 C   s   dd t dD }tdtj|tdi}tj|}t|d 	dtj
s(J | }tdtj|ddi}t|| d S )Nc                 S   s   g | ]}t  t|d  qS ))days)r   todayr   r,   r)   r)   r/   r0   8  s    zETestConvertDateTimeLikeTypes.test_python_datetime.<locals>.<listcomp>r\  r   r   r   r$   )r:   r4   r5   r_   r   rF   rH   rI   r   r  TimestampArrayrJ   rP   rQ   )rx   
date_arrayrR   rW   rX   expected_dfr)   r)   r/   test_python_datetime6  s   z1TestConvertDateTimeLikeTypes.test_python_datetimec              
   C   sZ   t d}|j|d|dfD ]}tdddddd|dg}td	|i}t| qd S )
Npytzr  r   r   r     -   tzinfor   )	r   importorskiputcr   FixedOffsetr   r4   r5   rY   )rx   r  r   rl   rR   r)   r)   r/   %test_python_datetime_with_pytz_tzinfoI  s   

zBTestConvertDateTimeLikeTypes.test_python_datetime_with_pytz_tzinfoN)deadlinec              	   C   sL   t |dv rtd tdddddd|dg}td	|i}t|d
d d S )N)zbuild/etc/localtimeFactoryz Localtime timezone not supportedr   r   r  r  r  r  r   FrE   )r+   r   r	  r   r4   r5   rY   )rx   r   rl   rR   r)   r)   r/   'test_python_datetime_with_pytz_timezoneQ  s
   
zDTestConvertDateTimeLikeTypes.test_python_datetime_with_pytz_timezonec           
   	   C   s   t d}ddlm} tdddddd|jd	g}tjd
|i|d}t|dd d}|t|d}|	|d }tdddddd|d	g}tdddddd|d	g}tjd
|i|d}tjd
|i|d}	t||	dd d S )Nr  r   )r   r   r   r  r  r  r  r   r   Tr   )hours<   r   )
r   r  r   r   r  r4   r5   rY   r   r  )
rx   r  r   rl   rR   r  Ztz_timezoneZtz_pytzZ
values_expZdf_expr)   r)   r/   )test_python_datetime_with_timezone_tzinfoZ  s   
zFTestConvertDateTimeLikeTypes.test_python_datetime_with_timezone_tzinfoc                 C   s   G dd dt }|ddddddg}tdtj|tdi}tj|}t|d 	dtj
s1J | }tdtj|ddi}t|d |d< t|| d S )	Nc                   @   s   e Zd ZdZdS )zNTestConvertDateTimeLikeTypes.test_python_datetime_subclass.<locals>.MyDatetimerl  N)rL  rM  rN  Z
nanosecondr)   r)   r)   r/   
MyDatetimeo  s    r    r   r   r   r   r$   )r   r4   r5   r_   r   rF   rH   rI   r   r  r  rJ   to_datetimerP   rQ   )rx   r  r  rR   rW   rX   r  r)   r)   r/   test_python_datetime_subclassm  s   z:TestConvertDateTimeLikeTypes.test_python_datetime_subclassc                 C   s   G dd dt }|dddg}tdtj|tdi}tj|}t|d 	dtj
s.J | }tdtjt dddgtdi}t|| d S )Nc                   @      e Zd ZdS )zFTestConvertDateTimeLikeTypes.test_python_date_subclass.<locals>.MyDateNrL  rM  rN  r)   r)   r)   r/   MyDate      r  r  r   r   r   r   )r   r4   r5   r_   r   rF   rH   rI   r   r  ZDate32ArrayrJ   r6   r^   rP   rQ   )rx   r  r  rR   rW   rX   r  r)   r)   r/   test_python_date_subclass  s   z6TestConvertDateTimeLikeTypes.test_python_date_subclassc                 C   sV   t tdddd g}t |g}| }t jj|t  d}||	ds)J d S )N  r\  r  r  date32)
rF   r^   r   chunked_arrayrJ   ArrayrI   r  rK   r  )rx   rd   r   ra   arr2r)   r)   r/   test_datetime64_to_date32  s
   z6TestConvertDateTimeLikeTypes.test_datetime64_to_date32rf   )TFFTFFc                 C   s   t g d}tjj|t |d}ttdddtdddtdddtdddtdddtdddg}tj||t d}|	|sFJ d S )	N)z2018-05-10T00:00:00z2018-05-11T00:00:00z2018-05-12T00:00:00z2018-05-10T10:24:01z2018-05-11T10:24:01z2018-05-12T10:24:01r[   rf   r   r   r\  r&  r  rf   r[   )
r4   r  rF   r  rI   date64r6   r^   r   rK   )rx   rf   ra   rd   r   rS   r)   r)   r/   test_pandas_datetime_to_date64  s   





z;TestConvertDateTimeLikeTypes.test_pandas_datetime_to_date64zcoerce_to_ns,expected_dtype)Fr#   )Tr%   c           
      C   s   t dddd t dddt dddg}tjg ddd	}ttjtd
k r&d}tjg d|d	}t|t|gg}|D ]0}|j|d}|	t
}	|j|	jksQJ t||	 |jd|d}|j|jksfJ t|| q<d S )Nr  r       r2   r%  )
2000-01-01Nz
1970-01-01z
2040-02-26datetime64[D]r   r  r%   coerce_temporal_nanosecondsFdate_as_objectr  )r   r6   r^   r   r4   r  rF   r  rJ   r   r   r   ro   rp   )
rx   coerce_to_nsZexpected_dtyper   Zexpected_daysrS   objectsobjrX   Zexpected_objr)   r)   r/   test_array_types_date_as_object  s6   






z<TestConvertDateTimeLikeTypes.test_array_types_date_as_objectzcoerce_to_ns,expected_typec              
   C   s~   t dtdddd tdddtdddgi}tjj|dd	}|jd|d
}| }tj|	||dd tj||dd d S )Nr   r  r   r   r  r2   r%  Fr   r  Tr  )
r4   r5   r   rF   rH   rI   rJ   rP   rQ   r   )rx   r  expected_typerR   rW   Zdf_datetimeZ	df_objectr)   r)   r/   !test_table_convert_date_as_object  s    

z>TestConvertDateTimeLikeTypes.test_table_convert_date_as_object
arrow_typera   r  r  rE  rF  c                 C   s   t dddtdddg}t|}t||}|jddd}d }t|dr.|j	d ur.d}t
d	| }t||| d S )
Nr  r     TFr  r  r   rF  rE  )r   r   r4   r_   rF   r^   r  rJ   hasattrr   rH  to_pandas_dtyperP   r`   r   )rx   r  r   rS   rd   rX   expected_tzr  r)   r)   r/   &test_array_coerce_temporal_nanoseconds  s   
zCTestConvertDateTimeLikeTypes.test_array_coerce_temporal_nanosecondsc           	      C   s   t dddtdddg}ttd|g}td|i}tjt|g|d}|j	ddd}d }t
|d	r=|jd ur=d
}td| }t||| d S )Nr  r   r  r   r  TFr  r   rF  rE  )r   r   rF   r>   rv   r4   r5   rW   r^   rJ   r  r   rH  r  rP   rQ   r   )	rx   r  r   r>   r  rW   Z	result_dfr  r  r)   r)   r/   &test_table_coerce_temporal_nanoseconds  s   zCTestConvertDateTimeLikeTypes.test_table_coerce_temporal_nanosecondsc              
   C   s   t dtdddd tdddtdddgi}tjj|dd	}tdt }tj|g|jj	d
}|j
|s9J | }t|| d S )Nr   r  r   r   r  r2   r%  Fr   r   )r4   r5   r   rF   rH   rI   rv   r  r>   r   rK   rJ   rP   rQ   )rx   rR   rW   rv   rT   rX   r)   r)   r/   test_date_infer  s   

z,TestConvertDateTimeLikeTypes.test_date_inferc                 C   sx   t jtdddtdddgdd}ddg}tj|t |d}t jd tdddgdd}tj|dd	}||s:J d S )
Nr  r   rt   r  r   TFr  rq  )r6   r^   r   rF   rK   )rx   rd   rf   rX   rS   r)   r)   r/   test_date_mask%  s   z+TestConvertDateTimeLikeTypes.test_date_maskc                 C   sh  t jtdddd tdddtdddgtd}t jg ddd}|dd	 }t g d
}t }t }tj||d}tj||d}tj|||d}	tj|||d}
||	s[J ||
sbJ ddg}tj	
||g|}t jg ddd}tjj|d< |d}tj||d|d}|jdd}t|| | }|d}tj||d|d}t|| d S )Nr  r   rt   r   r   )kC  r   lC  mC  r   r    \&)FTFFr  r  r  r  )z
2017-04-03
2017-04-04r  z
2017-04-05r  r   r#   )r  r  r   F)r  r   )r6   r^   r   r   r   rF   r  r  rK   rH   r   r4   NaTr6  r5   rJ   rP   rQ   )rx   rd   Zarr_i4Zarr_i8rf   Zt32Zt64Za32Za64Za32_expectedZa64_expectedZcolnamesrW   	ex_valuesZex_datetime64msZexpected_pandasZtable_pandasZtable_pandas_objectsZ
ex_objectsZexpected_pandas_objectsr)   r)   r/   test_date_objects_typed.  sV   





z4TestConvertDateTimeLikeTypes.test_date_objects_typedc                 C   s   t tdd }tjtdddtj|gtd}tjtdddd d gtd}tj|dd}tj|dd}||s8J |j	dks?J tjtjgddj
t ksOJ tj|gddj
t ks^J d S )NNAr  r   r   Trq  r2   )getattrr4   r6   r^   r   r  r   rF   rK   ri   r[   null)rx   Zpd_NArl   Zvalues_with_nonerX   rS   r)   r)   r/   test_pandas_null_values_  s    "z4TestConvertDateTimeLikeTypes.test_pandas_null_valuesc                 C   s   t  }t  }tjg ddd}|dd }t j||d}t j||d}tddd	}|d
  |ks6J |d
  |ks@J d S )N)r  r  r  r   r   r   r  r  r  r   rt   r   )rF   r  r  r6   r^   r   r   rs  )rx   t1t2rd   r  a1r  rS   r)   r)   r/   test_dates_from_integersn  s   z5TestConvertDateTimeLikeTypes.test_dates_from_integersc                 C   s  t ddddt ddddg}td}tj|d g td	}t|}|j|ks)J |d
  |d
 ks5J |d  |d ksAJ |d  d u sKJ t	d|i}tj
|}|d
 |saJ tjdd |D dd	}tj|tdd}|d
  |d
 ksJ tj|d tdd}	|	d
  |d
 ksJ tj|d dtdd}
|
d
  |d
 jddksJ tj|d dtdd}|d
  |d
 jd
dksJ d S )Nr   r2   rt   L  r   r   r   r  r   r   timesc                 S   r(   r)   _pytime_to_microsr-   vr)   r)   r/   r0     r1   zHTestConvertDateTimeLikeTypes.test_pytime_from_pandas.<locals>.<listcomp>r   r    rE  rw  r  microsecond@B ra   )r   rF   time64r6   r^   r   r[   rs  r4   r5   rG   rI   rK   r   time32replace)rx   pytimesr%  ZaobjsparrrR   r   rd   r'  r  a3a4r)   r)   r/   test_pytime_from_pandas|  s8   

 $z4TestConvertDateTimeLikeTypes.test_pytime_from_pandasc                 C   s  t ddddt ddddt dddg}t|d d d g }td	d
 |d d D d g }tdd
 |d d D d g }tjdd
 |D dd}tjdd
 |D dd}tjg dtd}tj||tdd}tj|d |tdd}tj|d d|tdd}	tj|d d|tdd}
g d}tj	|||	|
g|}||f||f|	|f|
|ffD ]\}}|
 }|j|k sJ q|
 }tj||||d|d}t|| d S )Nr   r2   rt   r)  r   r   r   r   c                 S      g | ]}|j d dqS )r/  r0  r5  r,   r)   r)   r/   r0         zJTestConvertDateTimeLikeTypes.test_arrow_time_to_pandas.<locals>.<listcomp>c                 S   r;  )r   r0  r<  r,   r)   r)   r/   r0     r=  c                 S   r(   r)   r+  r-  r)   r)   r/   r0     r1   r   r   c                 S   r(   r)   r+  r-  r)   r)   r/   r0     r1   )FFTr  r  r/  rE  rw  r  r2  ra   )
time64[us]z
time64[ns]
time32[ms]z	time32[s]r   )r   r6   r^   r    rF   r3  r   r4  rG   r   rJ   rl   allr4   r5   rP   rQ   )rx   r6  rS   Zexpected_msZ
expected_srd   ra  r'  r  r8  r9  r   r   Zexpected_valuesZresult_pandasrR   r  r)   r)   r/   test_arrow_time_to_pandas  sd   




z6TestConvertDateTimeLikeTypes.test_arrow_time_to_pandasc                 C   sl   t jg ddd}t| t jg ddd}t| t jg ddd}t| t jg dd	d}t| d S )
Nr  r%   r   )z2007-07-13T01:23:34.123456Nz2006-01-13T12:34:56.432539z2010-08-13T05:46:57.437699r$   )r  Nr  r  r#   )z2007-07-13T01:23:34Nz2006-01-13T12:34:56z2010-08-13T05:46:57r"   r6   r^   rq   )rx   Zdatetime64_nsZdatetime64_usZdatetime64_msZdatetime64_sr)   r)   r/   test_numpy_datetime64_columns  s    



z:TestConvertDateTimeLikeTypes.test_numpy_datetime64_columnsc                 C   s   t tjt dkrtd tg dtd}ttj	g ddd}|
 }t|| t|g}|
 }t|| d S )Nr  z0pandas >= 2.0 supports non-nanosecond datetime64rs   r  )r  )r   r4   r  r   r	  rF   r^   rH  r_   r  rJ   rP   r`   r  )rx   rd   rS   ra   r)   r)   r/   &test_timestamp_to_pandas_coerces_to_ns  s   
zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_coerces_to_nsc              
   C   s.  dD ]}dD ]}t tdddgt j||d}t d|i}d}tjt|d |jdd	 W d    n1 s9w   Y  tjt|d |jdd	 W d    n1 sVw   Y  tjt|d |	djdd	 W d    n1 svw   Y  |jd
dd |jd
dd |	djd
dd qqd S )Nra   r  r  )Nr   r   rG  r   z'would result in out of bounds timestampr  Tr  F)r   r  )
rF   r^   r   rH  rW   r   r   r   rJ   column)rx   r  r   rd   rW   msgr)   r)   r/   &test_timestamp_to_pandas_out_of_bounds  s*    
zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_out_of_boundsc                 C   sN   t dt jg t ddi}| }tdtjg ddi}t	|| d S )Nr   r  r  r$   r   )
rF   rW   r  rH  rJ   r4   r5   r_   rP   rQ   )rx   rW   rX   rS   r)   r)   r/   &test_timestamp_to_pandas_empty_chunked  s   zCTestConvertDateTimeLikeTypes.test_timestamp_to_pandas_empty_chunkedr   c                 C   s"   t jg ddd}t||d d S )N)
2007-07-13Nz
2006-01-15z
2010-08-19r  r   r  rB  )rx   r   Zdatetime64_dr)   r)   r/   test_numpy_datetime64_day_unit!  s   
z;TestConvertDateTimeLikeTypes.test_numpy_datetime64_day_unitc                 C   sx   t g d}ttdddtdddtdddg}tjj||d}td tdddd g}tj||s:J d S )NTFTi  r   i  i  r  )	r6   r^   r4   r_   r   rF   r  rI   rK   )rx   mr   rX   rS   r)   r)   r/   %test_array_from_pandas_date_with_mask+  s   


zBTestConvertDateTimeLikeTypes.test_array_from_pandas_date_with_maskz1.16.0z1.16.1z#Until numpy/numpy#12745 is resolved)reasonc                 C   s*   t dt dt jgi}t|dd d S )Nr   z2012-11-11 00:00:00+01:00Fr  )r4   r5   	Timestampr  rY   r   r)   r)   r/   test_fixed_offset_timezone8  s   
z7TestConvertDateTimeLikeTypes.test_fixed_offset_timezonec                 C   h   t tjt dk rd}tdtjg dd| ddi}tdt|}t	|g}t
||d d S )	Nr  rE  timedelta64)r   l     qa    @B1timedelta64[r  r   rT  r   r4   r  r5   r6   r^   rF   rv   durationr>   rY   rx   r  rR   rv   r>   r)   r)   r/   test_timedeltas_no_nullsH     

z5TestConvertDateTimeLikeTypes.test_timedeltas_no_nullsc                 C   rR  )	Nr  rE  rS  )r   NrT  rU  r  r   rT  rV  rX  r)   r)   r/   test_timedeltas_nullsW  rZ  z2TestConvertDateTimeLikeTypes.test_timedeltas_nullsc              
   C   sN   ddl m} tdd |dddddgi}tdt fg}t||d d S )	Nr   )
DateOffsetZdate_offseti  rt   iX  )r  monthsmicrosecondsnanosecondsrT  )Zpandas.tseries.offsetsr\  r4   r5   rF   r>   Zmonth_day_nano_intervalrY   )rx   r\  rR   r>   r)   r)   r/   test_month_day_nano_intervalf  s   
z9TestConvertDateTimeLikeTypes.test_month_day_nano_interval)7rL  rM  rN  rO  r  r  r   markparametrizer  r  r  hgivenstnonepastZ	timezonessettingsr  r  r  r  r  r6   r^   r  r  r  rF   r  r  rH  r  r  r  r  r   r$  r(  r:  rA  rC  rD  rH  rI  rK  rN  Zskipifr   r  rQ  rY  r[  r`  r)   r)   r)   r/   r    s    









	1%0!
	&


r  c                   @   s   e Zd Zdd Zdd Zejjdd Zejjej	ddd	gd
d Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*S )+TestConvertStringLikeTypesc                 C   sn   d}dd ddt jg}td|| i}tdt }t|g}g d}td|| i}t|||d d S )Nr/  r   r     mañanastrings)r   Nr  rj  Nr  )	r6   r^  r4   r5   rF   rv   r<  r>   rY   )rx   repeatsrl   rR   rv   r>   r  rS   r)   r)   r/   test_pandas_unicodey  s   z.TestConvertStringLikeTypes.test_pandas_unicodec                 C   sj   ddd t ddtjg}td|i}tj|}|d jt	 ks#J g d}td|i}t
|| d S )Nqux   foo   barzrk  r   )   quxro  Nrp  rq  N)	bytearrayr6   r^  r4   r5   rF   rH   rI   r[   binaryrY   )rx   rl   rR   rW   Zvalues2rS   r)   r)   r/   test_bytes_to_binary  s   z/TestConvertStringLikeTypes.test_bytes_to_binaryc                 C   s   dd }dd }t d|gd |g dgd  i}t|d }t|tjs)J |jdks0J d }tj|}|d jdksAJ d S )	N   x krk     r.   r2   r   )	r4   r5   rF   r^   r   ChunkedArray
num_chunksrH   rI   )rx   v1v2rR   rd   rW   r)   r)   r/   test_bytes_exceed_2gb  s   z0TestConvertStringLikeTypes.test_bytes_exceed_2gbcharr.   ru  c                 C   s   |d }|d }t d|ggd |gg dgg i}tj|d dd}|jdd t|tjs2J |jd	ks9J t|	d
dksDJ t|	ddksOJ d S )Nrv  rw  rk  rx  ru  Trq  fullr2   r      r   )
r4   r5   rF   r^   validater   ry  rz  r   r  )rx   r~  r{  r|  rR   rd   r)   r)   r/   +test_auto_chunking_pandas_series_of_strings  s   zFTestConvertStringLikeTypes.test_auto_chunking_pandas_series_of_stringsc                 C   s   dd t dd d dg}td|i}ttdtdg}tjj||d}|jd j	|d j	ks3J |jd j
|d j
ks@J | }t|| d S )Nro     bar   heyrk  rt   r  r   )rr  r4   r5   rF   r>   rv   rs  rH   rI   r[   r]   rJ   rP   rQ   )rx   rl   rR   r>   rW   rX   r)   r)   r/   test_fixed_size_bytes  s   z0TestConvertStringLikeTypes.test_fixed_size_bytesc                 C   sr   g d}t d|i}ttdtdg}ttj tj	j
||d W d    d S 1 s2w   Y  d S )N)ro  Ns   baNNr  rk  rt   r  )r4   r5   rF   r>   rv   rs  r   r   ArrowInvalidrH   rI   )rx   rl   rR   r>   r)   r)   r/   5test_fixed_size_bytes_does_not_accept_varying_lengths  s   "zPTestConvertStringLikeTypes.test_fixed_size_bytes_does_not_accept_varying_lengthsc                 C   s"   t g d}t|t d d S )N   123    r   Nrb   )r4   r_   re   rF   rs  rx   ra   r)   r)   r/   test_variable_size_bytes  s   z3TestConvertStringLikeTypes.test_variable_size_bytesc                 C   sB   t tdtdtdd g}t|t d t|t d d S )Nr  r  r   r  rc   )r4   r_   rr  re   rF   rs  r  r)   r)   r/   test_binary_from_bytearray  s
   z5TestConvertStringLikeTypes.test_binary_from_bytearrayc                 C   L   t g d}t|t d t d|i}t|tdt fgd d S )Nr  r  r   r  )r4   r_   re   rF   large_binaryr5   rY   r>   rx   ra   rR   r)   r)   r/   test_large_binary     
z,TestConvertStringLikeTypes.test_large_binaryc                 C   r  )N)Z123r3   r   Nr  r   r  )r4   r_   re   rF   large_stringr5   rY   r>   r  r)   r)   r/   test_large_string  r  z,TestConvertStringLikeTypes.test_large_stringc           
      C   s   g d}t d|i}tdt }t|g}tjj||d}|jdd}t d|i}t	j
||dd |jdd}t dt |i}	t	j
||	dd d S )Nr3   r3   r3   r3   r3   rk  r  Fstrings_to_categoricalTr  )r4   r5   rF   rv   r<  r>   rH   rI   rJ   rP   rQ   Categorical)
rx   rl   rR   rv   r>   rW   result1	expected1r   	expected2r)   r)   r/   test_table_empty_str  s   z/TestConvertStringLikeTypes.test_table_empty_strc                 C   s   g d}t d|i}tdt }t|g}tjj||d}t d|i}t dt |i}|j	dgd}t
j||dd |j	g d}	t
j|	|dd |j	dd}
t
j|
|dd |j	t d}t
j||dd d S )Nr  rk  r  
categoriesTr  )rk  )r4   r5   rF   rv   r<  r>   rH   rI   r  rJ   rP   rQ   tuple)rx   rl   rR   rv   r>   rW   Zexpected_strZexpected_catr  r   result3Zresult4r)   r)   r/   test_selective_categoricals  s   z6TestConvertStringLikeTypes.test_selective_categoricalsc                 C   s8   t jg t  d}t jj|gdgd}|jdgd d S )Nr  colr>  r   r  )rF   r^   r   rH   r   rJ   )rx   r^   rW   r)   r)   r/   &test_to_pandas_categorical_zero_length  s   zATestConvertStringLikeTypes.test_to_pandas_categorical_zero_lengthc                 C   sJ   t g d }t jj|gdgd}|jdgd}| |s#J d S )N)r   r   r   r  r  r  r  )rF   r^   dictionary_encoderH   r   rJ   rK   )rx   r^   rW   rX   r)   r)   r/   ,test_to_pandas_categories_already_dictionary  s   zGTestConvertStringLikeTypes.test_to_pandas_categories_already_dictionaryc                 C   s   g d}t d|i}tdt }t|g}tjj||d}|jdd}t dt 	|i}t
j||dd ttj |jddd W d    d S 1 sRw   Y  d S )N)r   r   r   r   r   rk  r  Tr  r  r  zero_copy_only)r4   r5   rF   rv   r<  r>   rH   rI   rJ   r  rP   rQ   r   r   r  rx   rl   rR   rv   r>   rW   rX   rS   r)   r)   r/   (test_table_str_to_categorical_without_na  s   "zCTestConvertStringLikeTypes.test_table_str_to_categorical_without_nac                 C   s   d ddt jg}td|i}tdt }t|g}tjj	||d}|j
dd}tdt|i}tj||dd ttj |j
ddd W d    d S 1 sUw   Y  d S )	Nr   r   rk  r  Tr  r  r  )r6   r^  r4   r5   rF   rv   r<  r>   rH   rI   rJ   r  rP   rQ   r   r   r  r  r)   r)   r/   %test_table_str_to_categorical_with_na  s   "z@TestConvertStringLikeTypes.test_table_str_to_categorical_with_nac                 C   s2   t tjdgtdt  }|jt  ksJ d S )Nru  r   )rF   r^   r6   r   r<  r[   )rx   	convertedr)   r)   r/   test_array_of_bytes_to_strings*  s   z9TestConvertStringLikeTypes.test_array_of_bytes_to_stringsc                 C   sT   t jtjjdd ttjdgtdt  W d    d S 1 s#w   Y  d S )Nzwas not a utf8 stringr  s   r   )	r   r   rF   libr  r^   r6   r   r<  rx   r)   r)   r/   'test_array_of_bytes_to_strings_bad_data1  s   "zBTestConvertStringLikeTypes.test_array_of_bytes_to_strings_bad_datac                 C   sL  t jg ddd}tj|tdd}tjt|tdd}||s&J t g d}tj|td|d}tjg dtdd}||sKJ tjtjj	d	d
 t jg ddd}tj|tdd W d    n1 srw   Y  tjtjj	dd
 t jg ddd}tj|tdd W d    d S 1 sw   Y  d S )N)ro  r     bazz|S3r   rt   r  FTFr  )ro  Nr  z)Got bytestring of length 3 \(expected 4\)r  r   z*Got bytestring of length 12 \(expected 3\)z|U3)
r6   r^   rF   rs  r   rK   r   r   r  r  )rx   rd   r  rS   rf   r)   r)   r/   ,test_numpy_string_array_to_fixed_size_binary7  s*   
"zGTestConvertStringLikeTypes.test_numpy_string_array_to_fixed_size_binaryN)rL  rM  rN  rm  rt  r   ra  large_memoryr}  rb  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r)   r)   r)   r/   ri  w  s.    

ri  c                   @   s  e Zd ZdZededgZededgZededgZe	j
de	jeed	d
dde	jeedddde	jeeddddgdd Ze	j
de	jedde	jedde	jeddgdd Zdd Zdd Zdd Zdd Zd d! Zd"S )#TestConvertDecimalTypesz,
    Conversion test for decimal types.
    z	-1234.123z1234.439z-129934.123331z129534.123731r#  z-314292388910493.12343437128)rl   r  r   rt   	decimal32idr  r   	decimal64r%  r&  
decimal128c                 C   sR   t d|i}tjj|dd}td|}tj|g|jjd}|j|s'J d S )Nr"  Fr   r  )	r4   r5   rF   rH   rI   rv   r>   r   rK   )rx   rl   r  rS   rW   rv   rT   r)   r)   r/   test_decimal_from_pandas`  s
   z0TestConvertDecimalTypes.test_decimal_from_pandasrl   c                 C   s2   t d|i}tj|}| }t|| d S )Nr"  )r4   r5   rF   rH   rI   rJ   rP   rQ   )rx   rl   rS   r  rR   r)   r)   r/   test_decimal_to_pandasn  s   z.TestConvertDecimalTypes.test_decimal_to_pandasc                 C   s   t dg}tdd}ttj tj||d W d    n1 s$w   Y  t dg}tdd}ttj tj||d W d    d S 1 sNw   Y  d S )Nz1.234r\  r2   r  z1.2345rt   )r$  r)  rF   r  r   r   r  r^   )rx   data1Ztype1data2Ztype2r)   r)   r/   "test_decimal_fails_with_truncationy  s   "z:TestConvertDecimalTypes.test_decimal_fails_with_truncationc                 C   s   t dt dg}t|}t|}| |ksJ |jtddks'J tj|tddd}t dt dg}| |ksDJ d S )	Nz0.01z0.001rt   r  r   r  z0.01000z0.00100)	r$  r)  r4   r_   rF   r^   ri  r[   r  )rx   r   seriesr^   rS   r)   r)   r/   &test_decimal_with_different_precisions  s   

z>TestConvertDecimalTypes.test_decimal_with_different_precisionsc                 C   sP   t tdd g}t|tddd t d gd }t|tddd d S )N3.14r  r   r  r2   r4   r_   r$  r)  re   rF   r  rx   r  r)   r)   r/   $test_decimal_with_None_explicit_type  s   z<TestConvertDecimalTypes.test_decimal_with_None_explicit_typec                 C   s,   t tdd g}t|tddd d S )Nr  rt   r2   r  r  r  r)   r)   r/   !test_decimal_with_None_infer_type  s   z9TestConvertDecimalTypes.test_decimal_with_None_infer_typec                 C   s0   ddidt did}tj|}t| d S )Nr   r   z0.0r   )r$  r)  r4   r5   	from_dictrY   )rx   Ztmpdirr   rR   r)   r)   r/   test_strided_objects  s
   z,TestConvertDecimalTypes.test_strided_objectsN)rL  rM  rN  rO  r$  r)  r  r  r  r   ra  rb  paramrF   r  r  r  r  r  r  r  r)   r)   r)   r/   r  O  s8    
	
r  c                
   @   s  e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zd d! Zd"d# Zd$d% Zd&d' Zd(d) Zd*d+ Zejd,ejd-d.gd/gd0gd0d/gd0gfejd1d2gd3gd0gd0d3gd0gfejd0d0gd0gd0gd0d0gd0gfgd4d5 Z d6d7 Z!d8d9 Z"d:d; Z#d<d= Z$d>d? Z%ejj&d@dA Z'dBdC Z(dDdE Z)dFdG Z*dHdI Z+dJdK Z,dLdM Z-d0S )NTestConvertListTypesz,
    Conversion tests for list<> types.
    c                 C   t   t  \}}t|||d tjj||dd}||jj}|j|s$J |j	D ]}|
|}t|| |jd q'd S Nr>   rT   Fr  r  )r   rY   rF   rH   rI   with_metadatar>   r   rK   r   rv   rm   r[   rx   rR   r>   rW   rT   rF  rv   r)   r)   r/   test_column_of_arrays     


z*TestConvertListTypes.test_column_of_arraysc                 C   s   d}t jt jd|dt jd|dd t jd|dgtd}tt }tj||d}|d  tt	dks8J |d  tt	dksFJ |d  d u sPJ |d	  dgks[J d S )
Nru  r\  r   r   r   r  r   r2   rt   )
r6   r^   r7   r   rF   r  r   rs  r   r:   )rx   r   rd   rb   r7  r)   r)   r/   test_column_of_arrays_to_py  s   z0TestConvertListTypes.test_column_of_arrays_to_pyc                 C   s   t jddgdggt t  d}t jj|gdgd}| }tdddgdggi}t	
|| |d  }t	jt||d dd d S )NTFr  r   r   r   rg   )rF   r^   r  rn  rH   r   rJ   r4   r5   rP   rQ   r`   r_   )rx   r^   rW   rR   r  ra   r)   r)   r/   test_column_of_boolean_list  s   "z0TestConvertListTypes.test_column_of_boolean_listc                 C   s   t jtdtdgtdggt t ddd}t jj|gdgd}| }t	
dtdtdgtdggi}t|| d S )	Nr   r  z3.3r2   r   r  r   r   )rF   r^   r$  r)  r  r  rH   r   rJ   r4   r5   rP   rQ   )rx   r^   rW   rR   r  r)   r)   r/   test_column_of_decimal_list  s   

z0TestConvertListTypes.test_column_of_decimal_listc              	   C   s   t tjtjtjgtd}tt t	t t
tddgfD ]6}tj||d}tjd d g|d}||s>J tt tj|j|d W d    n1 sVw   Y  q%d S )Nr   f0r   r  )r4   r_   r6   r^   r^  r   rF   r  r   
large_liststructrv   rK   r   r   	TypeErrorrl   )rx   ra   r  rX   rS   r)   r)   r/   +test_nested_types_from_ndarray_null_entries  s   z@TestConvertListTypes.test_nested_types_from_ndarray_null_entriesc                 C   r  r  )r   rY   rF   rH   rI   r  r>   r   rK   r   rv   rm   r[   r  r)   r)   r/   test_column_of_lists  r  z)TestConvertListTypes.test_column_of_listsc                 C   sT   g g dg dg dgg}t dd |D }t|}t | }t|| d S )Nr  )rt   r   r   r   r2   c                 S   s   g | ]	}t j|td qS )r   )r6   r^   float)r-   ra   r)   r)   r/   r0         zITestConvertListTypes.test_column_of_lists_first_empty.<locals>.<listcomp>)r4   r_   rF   r^   rJ   rP   r`   )rx   Z	num_listsr  rd   rX   r)   r)   r/    test_column_of_lists_first_empty   s
   
z5TestConvertListTypes.test_column_of_lists_first_emptyc                 C   s   t dtjddgd ddgddgddgd	d
ggtdi}ttdtt	 g}tj
j|d d |d}tj
j|dd  |d}t||g}| }t|| d S )Nlistsr   r2   rt   r   r   r   r   r   	   r   r  )r4   r5   r6   r^   r   rF   r>   rv   r  r   rH   rI   concat_tablesrJ   rP   rQ   )rx   rR   r>   r%  r&  rW   rX   r)   r)   r/   test_column_of_lists_chunked  s(   z1TestConvertListTypes.test_column_of_lists_chunkedc              	   C   s^   t dtjg tdi}ttdtt	 g}tj
j||d}| }t|| d S )Nr  r   r  )r4   r5   r6   r^   r   rF   r>   rv   r  r   rH   rI   rJ   rP   rQ   )rx   rR   r>   rW   rX   r)   r)   r/   "test_empty_column_of_lists_chunked!  s   z7TestConvertListTypes.test_empty_column_of_lists_chunkedc           
   	   C   s   ddgddgddgddgd	d
gddgddgddgg}ddgddgg}t |}t |}t jj|gdgd}t jj|gdgd}t ||g}| }td|| i}	t	||	 d S )Nr   r   r2   rt   r   r   r   r   r\  r&  r                 r   r        r   r   )
rF   r^   rH   r   r  rJ   r4   r5   rP   rQ   )
rx   r  r  r'  r  r%  r&  ZconcatenatedrX   rS   r)   r)   r/   test_column_of_lists_chunked2/  s   

z2TestConvertListTypes.test_column_of_lists_chunked2c                 C   sP   t  \}}tj|gd dd}|d jd d d }|jd dks"J t| d S )Nr   T)Zignore_indexr   rt   r   r   )r   r4   concatrl   stridesrm   )rx   rR   r>   rd   r)   r)   r/   test_column_of_lists_stridedA  s
   
z1TestConvertListTypes.test_column_of_lists_stridedc                 C   s   t jd d gd gtd}t|}tt|}||sJ |jtt ks*J t jd d d d gt jd d gtdgtd}t|}td d d d gd d gg}||sWJ d S )Nr   )	r6   r^   r   rF   r   rK   r[   r  r#  )rx   r   rd   rS   r  r)   r)   r/   test_nested_lists_all_noneJ  s   

z/TestConvertListTypes.test_nested_lists_all_nonec                 C   sR   t g g g g}t|}tt|}||sJ |jtt ks'J d S N)	r4   r_   rF   r^   r   rK   r[   r  r#  rx   r   rd   rS   r)   r)   r/   test_nested_lists_all_emptyY  
   
z0TestConvertListTypes.test_nested_lists_all_emptyc                 C   sR   t g dgg}t|}tt|}||sJ |jtt ks'J d S )Nr   )	r4   r_   rF   r^   r   rK   r[   r  r<  r  r)   r)   r/   test_nested_list_first_emptya  r  z1TestConvertListTypes.test_nested_list_first_emptyc                 C   s   t tjg dddd g}t|}t|j}tjg dd gtt d}||s0J ||s7J t tjg dddd g}t|}tjg dd gtt	 d}||saJ d S )Nrs   ru  r   r  rR  )
r4   r_   r6   r^   rF   rl   r  r   rK   r   )rx   r   rX   r   rS   Zdata3r  Z	expected3r)   r)   r/   test_nested_smaller_intsi  s   
 
 z-TestConvertListTypes.test_nested_smaller_intsc                 C   s   t dtjdgddggfdddgddggfdd dgd	d
ggfdd dgd	d
ggd gfg}t|}ttdtt	 tdtt	 tdtt
 tdttt
 g}t||d d S )NZnan_intsr   r2   rt   r  r   strsr   r   dZnested_strsrT  )r   r6   r^  r4   r5   rF   r>   rv   r  r   r<  rY   rx   r   rR   rT   r)   r)   r/   test_infer_listsx  s   
z%TestConvertListTypes.test_infer_listsc           
      C   s   t jt  dd}t t  }g dd g dg dg}t j||d}t j||d}| }| }t||D ]\}}	|d u rE|	d u sEJ t||	 q7d S )Nr   )Z	list_size)r   r   r2   rt   r   )r   r  r\  r&  r  )rF   r  r   r^   rJ   r`  ro   rp   )
rx   Zfixed_tyZvariable_tyr   Z	fixed_arrZvariable_arrrX   rS   leftrightr)   r)   r/   test_fixed_size_list  s   z)TestConvertListTypes.test_fixed_size_listc              	   C   sj   t dtjddgtjdtjddgtjdgfg}t|}ttdt	t g}t
||d d S )Nr  r   r   r   r2   rt   rT  )r   r6   r^   r   r4   r5   rF   r>   rv   r  rY   r  r)   r)   r/   test_infer_numpy_array  s   
z+TestConvertListTypes.test_infer_numpy_arrayc                 C   s   t g dt  }t ddgddgddggt t  }t j||gdd	g}t jg d
|}t	dddgdgdddgddddgdgg}t	|
 }t  tddt t|| W d    d S 1 sow   Y  d S )Nrs   r   r   r   r  eff1rQ  )r   r   rt   r   )r  rQ  r2   rt   rB   rC   )rF   r^   r   r  r<  StructArrayr   	ListArrayr4   r_   rJ   rL   rM   rN   rO   rP   r`   )rx   r  rk  structsr   rS   r  r)   r)   r/   test_to_list_of_structs_pandas  s&   
"z3TestConvertListTypes.test_to_list_of_structs_pandasc                 C   s   t tjt dkrt tjt dk rtd dddgfddd	gfggd
g fddd dgfgdddgfggg}t|tt	t
 tt
 }| }t|}t  tddt t|| W d    d S 1 spw   Y  d S )N1.25.0.dev0r  $Regression in pandas with numpy 1.25r   r   r   r  r   r  r  rn  Nquuxr   quzr  grB   rC   )r   r6   r  r4   r   r	  rF   r^   r  map_utf8rJ   r_   rL   rM   rN   rO   rP   r`   )rx   r   rd   r  rS   r)   r)   r/   test_to_list_of_maps_pandas  s   
"(

"z0TestConvertListTypes.test_to_list_of_maps_pandasc           	   
   C   sV  t tjt dkrt tjt dk rtd tg ddd}tdgdgdd	gd
dgg dddggt	t
 dd}tjg d||}tjg d|}| }tddd	gfdd
dgfggdg fdggg}|dd }tdg fdggg}t  tddt t|| t|| W d   dS 1 sw   Y  dS )zL
        A slightly more rigorous test for chunk/slice combinations
        r  r  r  )rB   r   r  r  rn  r
  rB   r   r   rB   r   r   r   r  Nr   r2   )r   r2   r   r   r   r  r  r	  rC   )r   r6   r  r4   r   r	  rF   r^   slicer  r<  MapArrayr   r  rJ   r_   rL   rM   rN   rO   rP   r`   )	rx   r=  itemsmaprd   r  rS   Zseries_slicedZexpected_slicedr)   r)   r/   "test_to_list_of_maps_pandas_sliced  s<   
 
"z7TestConvertListTypes.test_to_list_of_maps_pandas_slicedzt,data,expectedr   r2   rt   NZaaabbr   c                 C   sX   t g d}t|}tjj||t| d}tjj|t| d|s*J d S )NrL  r  r  )	r6   r^   r4   r_   rF   r  rI   r  rK   )rx   r   r   rS   rM  ra   rX   r)   r)   r/   ,test_array_from_pandas_typed_array_with_mask  s   


zATestConvertListTypes.test_array_from_pandas_typed_array_with_maskc                 C   sV   t jdtd}|g  tt g d|d}tj	|}|
 }t|| d S )N)rt   r   )r   r  3r   )r6   emptyr   fillr4   r5   r^   rF   rH   rI   rJ   rP   rQ   )rx   Zempty_list_arrayrR   r@  rX   r)   r)   r/   test_empty_list_roundtrip	  s   
z.TestConvertListTypes.test_empty_list_roundtripc                 C   s`   t  \}}|D ]&}||j j}tjt||jd}t|}|j|jks&J ||s-J qd S )Nr  )r   r]   rl   rF   r^   r   r[   rK   )rx   rR   r>   rv   rd   rS   rX   r)   r)   r/   test_array_from_nested_arrays!	  s   

z2TestConvertListTypes.test_array_from_nested_arraysc                 C   s   t jg ddggd gt t t  d }t + tddt tddt	 t
j|tjg ddggd gtddd	 W d    d S 1 sJw   Y  d S )
Nrs   r   r  rB   z&Creating an ndarray from ragged nestedrC   r   Frg   )rF   r^   r  r   rJ   rL   rM   rN   _np_VisibleDeprecationWarningrO   rP   r`   r4   r_   r   r  r)   r)   r/   test_nested_large_list*	  s$   
"z+TestConvertListTypes.test_nested_large_listc              	   C   s   t jt jfD ]N}t jddgd dgg g|t  d }tj|t	ddgd dgg gdd	 t jddgd dgg g|t 
 d }tj|t	ddgd dgg gdd	 qd S )
Nr   r  ccr  s   aas   bbs   ccFrg   )rF   r  r  r^   r  rJ   rP   r`   r4   r_   r  )rx   Zlist_type_factoryra   r)   r)   r/   test_large_binary_list9	  s(   

z+TestConvertListTypes.test_large_binary_listc                 C   sl   t g d }t jg d|}t| }t	|
 | |g d}d |d< t	|
 | d S )N)r   r  Nr   )r   r   rt   rt   r   )r   r   Nrt   r2   )rF   r^   r  r  r   r4   r_   ri  rP   r`   rJ   take)rx   childrd   rS   r)   r)   r/   test_list_of_dictionaryH	  s   z,TestConvertListTypes.test_list_of_dictionaryc                 C   s   d}t jttj|dfddt|d}tj	|}|j
dd |d }|jd	ks-J t|dd
ks8J t|ddksCJ d S )Ni    i   r   r   r   Tr  r   r2   i r   )r4   r5   r  r   r6   zerosr:   rF   rH   rI   r  rz  r   r  )rx   nrR   rW   Zcolumn_ar)   r)   r/   #test_auto_chunking_on_list_overflowU	  s   z8TestConvertListTypes.test_auto_chunking_on_list_overflowc                 C   sV   ddgdgg ddgg}t d|i}tdtt t fg}t||d d S )Nr   r      br2      crt      dr      er   )   fr      gr   r  r  )r4   r5   rF   r>   r  rs  r   rY   )rx   r   rR   r>   r)   r)   r/   test_map_array_roundtripe	  s    z-TestConvertListTypes.test_map_array_roundtripc                 C   s   ddgdgg ddgg}dd |D }t j|t t  t  d}t j|t t  t  d}t ||g}t|| }| }t	j
||d	d
 d S )Nr&  r'  r)  r+  r1  c                 S   s   g | ]	}d d |D qS )c                 S   s   g | ]
\}}||d  fqS )r2   r)   r-   kr.  r)   r)   r/   r0   u	  s    zJTestConvertListTypes.test_map_array_chunked.<locals>.<listcomp>.<listcomp>r)   r-   rowr)   r)   r/   r0   u	  r  z?TestConvertListTypes.test_map_array_chunked.<locals>.<listcomp>r  Frg   )rF   r^   r  rs  r   r  r4   r_   rJ   rP   r`   )rx   r  r  arr1r  rd   rS   actualr)   r)   r/   test_map_array_chunkedp	  s   z+TestConvertListTypes.test_map_array_chunkedc                 C   sh   ddgd g ddgg}dd |D }t |}tj|tt t d}| }tj	||dd	 d S )
Nr&  r'  )r,  r.  )r0  Nr1  c                 S   s&   g | ]}|d urdd |D nd qS )Nc                 S   s(   g | ]\}}||d urt |nd fqS r  )r  r4  r)   r)   r/   r0   	  s   ( zMTestConvertListTypes.test_map_array_with_nulls.<locals>.<listcomp>.<listcomp>r)   r6  r)   r)   r/   r0   	  s
    
zBTestConvertListTypes.test_map_array_with_nulls.<locals>.<listcomp>r  Frg   )
r4   r_   rF   r^   r  rs  r   rJ   rP   r`   )rx   r   rS   rd   r9  r)   r)   r/   test_map_array_with_nulls	  s   
z.TestConvertListTypes.test_map_array_with_nullsc                 C   sx   t g d}t g d }t ttt|}t j|||}t	g dddgg}|
 }tj||dd d S )Nr   rt   r   )r   r   r   r   r  ))r   r   )r   r   )r2   r   )rt   r   )r   r  Frg   )rF   r^   r  r   r:   r   r  r   r4   r_   rJ   rP   r`   )rx   offsetsr  r=  rd   rS   r9  r)   r)   r/   !test_map_array_dictionary_encoded	  s   z6TestConvertListTypes.test_map_array_dictionary_encodedc           	   	   C   sd  t ddgg dd dd gddgg}t |dd|ddg}| }tjd	d
gg dd dtjggdd}t||D ]\}}|d u rK||ksJJ q<t	|| q<td	d
ddddtjgg}t	|d j
| |dd }tjg dd dtjggdd}t||D ]\}}|d u r||ksJ qt	|| qtddddtjgg}t	|d j
| d S )Nr   r2   )rt   r   r   r   r   r   r   rt   r{   r|   )r}   r~   r   r   r   r   r}   r~   r   )rF   r^   r  r  to_numpyr6   r^  r`  ro   rp   base)	rx   rd   Zchunked_arrr  rS   r  r  Zexpected_baseZnp_arr_slicedr)   r)   r/   test_list_no_duplicate_base	  s,   $z0TestConvertListTypes.test_list_no_duplicate_basec                 C   s   t jjt g dt g dt g dd}|jdd}tjddgd d	tjggd
d}t||D ]\}}|d u rA||ks@J q2t	|| q2d S )Nr   r2   r   r   )r   r2   c   rC  rt   Nr  )r=  rl   rf   Fr  r{   r|   r}   r   r   )
rF   r  r   r^   r?  r6   r^  r`  ro   rp   )rx   rd   r  rS   r  r  r)   r)   r/   test_list_values_behind_null	  s   z1TestConvertListTypes.test_list_values_behind_null).rL  rM  rN  rO  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   ra  rb  rF   r   r<  r#  r  r  r  r  r  r"  r  r%  r3  r:  r;  r>  rA  rE  r)   r)   r)   r/   r    sf    	'



		
r  c                   @   sh   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Ze	j
je	j
jdd Zdd Zdd Zdd ZdS )TestConvertStructTypesz,
    Conversion tests for struct types.
    c              	   C   sf   t dddddddgi}tdtdt fdt fgfg}t||d	 t|||d
 d S )Nr   r   r2   r   rt   r   r   r   rT  r  )r4   r5   rF   r>   r  r   rY   )rx   rR   rT   r)   r)   r/   test_pandas_roundtrip	  s    
z,TestConvertStructTypes.test_pandas_roundtripc                 C   s   t jg dt  d}t jg dt  d}t jg dt  d}t j|||gg d}td ddddd d	dd
dd dg}t|	 }t
|| d S )N)Nr2   rt   r  )r   Nr   )TFN)r  r  r  r   Tr2   Frt   r   )rF   r^   r   r<  rn  r  r   r4   r_   rJ   rP   r`   )rx   r  r  r  rd   rS   r  r)   r)   r/   test_to_pandas	  s   


z%TestConvertStructTypes.test_to_pandasc           
      C   s   t   t }tjdgt d}tjdgt d}tj|gdg}tj|gdg}t||g}t	
ddiddig}t	
| }t|| ~~~~~~t }	|	|ks]J d S )Nr   r  r2   r  )gccollectrF   total_allocated_bytesr^   r   r  r   r  r4   r_   rJ   rP   r`   )
rx   Zbytes_startZints1ints2r8  r  rd   rS   r  Z	bytes_endr)   r)   r/   test_to_pandas_multiple_chunks	  s*   z5TestConvertStructTypes.test_to_pandas_multiple_chunksc                 C   sX  t dt jfdt jfg}ttdt tdt g}t jg |d}tj||d}| g ks6J t jddg|d}tj||d}| dd	d
ddd
gksVJ tj|t dd	g|d}| dd	d
d gkspJ t g }tg }t jg |d}tj||d}| g ksJ t jddg|d}tj||d}| i i gksJ d S )Nr.   )Zy_titler  r  r   r  )r  T)+   Fr  T)r.   r  rN  Fr  r)   )	r6   r   r   rn  rF   r  rv   r^   ri  rx   rI  r  r   rd   r)   r)   r/   test_from_numpy
  s0   


z&TestConvertStructTypes.test_from_numpyc                 C   s  t dt dt jfdt jfgfdt jfdt jfg}|jdks"J tt	dtt	dt t	dt gt	dt t	dt
 g}t jg |d}tj||d}| g ksaJ t jd	d
g|d}tj||d}| ddddddddddddgksJ d S )Nr.   xxyyr  zr  r   r  ))r   Tr2   r   ))rt   Fr   r  r   T)rQ  rR  r2   r   )r.   r  rS  rt   Fr   r  )r6   r   r   rn  r   object_itemsizerF   r  rv   r<  r^   ri  rO  r)   r)   r/   test_from_numpy_nested&
  s6   z-TestConvertStructTypes.test_from_numpy_nestedc                    s4  d}t dt jfdg}d|j }d| }|||j  }t j||d}t j||d< ||d< t j|d |d dk < t	t
dt t
dt g}tj||d	d
}|jd	d |jdksbJ dd  d fdd	}	|	|| ~t j|dk }
tj|||
d	d}|jd	d |jdksJ |	|||
 ~d S )Nl        r.   )r  r   i      .r   r  r0  Tr  r  r2   c                 s   s    |   D ]}|E d H  qd S r  )Z
iterchunks)rd   r  r)   r)   r/   iter_chunked_arrayT
  s   zHTestConvertStructTypes.test_from_numpy_large.<locals>.iter_chunked_arrayc           	   	      s   t | t |ks
J |d }|d }t | D ]J\}}z9| }|d ur0|| r0|d u s/J n$|| }t|rB|d d u sAJ n|d |ksJJ |d || ksTJ W q tyb   td|  w d S )Nr.   r  zFailed at index)r   	enumeraters  r6   rr  	Exceptionprint)	rd   r   rf   ZxsZysir
  r  r.   rX  r)   r/   checkX
  s&   

z;TestConvertStructTypes.test_from_numpy_large.<locals>.check)r[   rf   rI   r  )r6   r   r   rU  r#  r8   Zrandom_sampler^  rF   r  rv   rs  r^   r  rz  )rx   Ztarget_sizerI  bsblockr$  r   r  rd   r^  rf   r)   r]  r/   test_from_numpy_large?
  s2   

z,TestConvertStructTypes.test_from_numpy_largec                 C   s   t t dt  t dt  g}tdtjfdtjfg}tjg |d}tj	t
dd t j||d W d    n1 s@w   Y  tg }tj	tdd t j||d W d    d S 1 sdw   Y  d S )	Nr.   r  rS  r   zMissing field 'y'r  r  zExpected struct array)rF   r  rv   r   rn  r6   r   r^   r   r   r   r  )rx   r  rI  r   r)   r)   r/   test_from_numpy_bad_inputx
  s$   
"z0TestConvertStructTypes.test_from_numpy_bad_inputc                 C   s   t dddgi}t dddddddgi}td	t fd
t fg}t|d }t||d |d td|fg}t	||||d d S )NZtuples)r   r2   rt   r   r   r2   r   rt   r   r   r   rm  )rS   r>   rT   )
r4   r5   rF   r  r   r6   asarrayrm   r>   rY   )rx   rR   r  Zstruct_typerd   rT   r)   r)   r/   test_from_tuples
  s   

z'TestConvertStructTypes.test_from_tuplesc                    s   ddg t g d t g d g}t jj| d}tdd |D  } fdd	|D }t|}t	|
 | |g d
}d |d< t	|
 | d S )Nr  r  )  i  rf  )r   r   Nr   c                 s   s    | ]}|  V  qd S r  )ri  )r-   r!  r)   r)   r/   	<genexpr>
  s    zCTestConvertStructTypes.test_struct_of_dictionary.<locals>.<genexpr>c                    s   g | ]	}t t |qS r)   )r_  r`  r6  r   r)   r/   r0   
  r  zDTestConvertStructTypes.test_struct_of_dictionary.<locals>.<listcomp>)r   Nr2   r   )rF   r^   r  r  r   r`  r4   r_   rP   r`   rJ   r   )rx   childrenrd   Zrows_as_tuplesZrows_as_dictsrS   r)   r   r/   test_struct_of_dictionary
  s   
z0TestConvertStructTypes.test_struct_of_dictionaryN)rL  rM  rN  rO  rG  rH  rM  rP  rV  r   ra  Zslowr  ra  rb  re  ri  r)   r)   r)   r/   rF  	  s    7rF  c                   @   sp   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdS )TestZeroCopyConversionz@
    Tests that zero-copy conversion works with some types.
    c                 C   s*   t g djdd}t|g d d S )Nr   TrD  rn   )rx   rX   r)   r)   r/   test_zero_copy_success
  s   z-TestZeroCopyConversion.test_zero_copy_successc                 C   s^   t jtddgtjdgdd}|jdd}tddg}tj	t
|t
|dd d S )	Nr   r   r   r   TrD  Frg   )rF   r   r   r6   r^   rJ   r4   r  rP   r`   r_   )rx   rd   rX   rl   r)   r)   r/   test_zero_copy_dictionaries
  s   
z2TestZeroCopyConversion.test_zero_copy_dictionariesc                 C   2   t jdgdd}t|jdd}t|| d S )NrJ  r%   r   TrD  r6   r^   rF   rJ   ro   rp   rx   rd   rX   r)   r)   r/   test_zero_copy_timestamp
     z/TestZeroCopyConversion.test_zero_copy_timestampc                 C   rm  )Nr   r'   r   TrD  rn  ro  r)   r)   r/   test_zero_copy_duration
  rq  z.TestZeroCopyConversion.test_zero_copy_durationc                 C   s>   t tj |jdd W d    d S 1 sw   Y  d S )NTrD  )r   r   rF   r  rJ   rx   rd   r)   r)   r/   check_zero_copy_failure
  s   "z.TestZeroCopyConversion.check_zero_copy_failurec                 C      |  tg d d S )N)ABCrt  rF   r^   r  r)   r)   r/   &test_zero_copy_failure_on_object_types
     z=TestZeroCopyConversion.test_zero_copy_failure_on_object_typesc                 C   ru  )N)r   r   Nry  r  r)   r)   r/   *test_zero_copy_failure_with_int_when_nulls
  r{  zATestZeroCopyConversion.test_zero_copy_failure_with_int_when_nullsc                 C   ru  )N)rl  r{   Nry  r  r)   r)   r/   ,test_zero_copy_failure_with_float_when_nulls
  r{  zCTestZeroCopyConversion.test_zero_copy_failure_with_float_when_nullsc                 C   s   |  tddg d S )NTFry  r  r)   r)   r/   $test_zero_copy_failure_on_bool_types
  r{  z;TestZeroCopyConversion.test_zero_copy_failure_on_bool_typesc                 C   s2   t jddgddggt t  d}| | d S )Nr   r2   r   r  r  )rF   r^   r  r   rt  rs  r)   r)   r/   $test_zero_copy_failure_on_list_types
  s   $z;TestZeroCopyConversion.test_zero_copy_failure_on_list_typesc                 C   &   t jdd gdd}| t| d S )Nr   r%   r   r6   r^   rt  rF   rs  r)   r)   r/   .test_zero_copy_failure_on_timestamp_with_nulls
     zETestZeroCopyConversion.test_zero_copy_failure_on_timestamp_with_nullsc                 C   r  )Nr   r'   r   r  rs  r)   r)   r/   -test_zero_copy_failure_on_duration_with_nulls
  r  zDTestZeroCopyConversion.test_zero_copy_failure_on_duration_with_nullsN)rL  rM  rN  rO  rk  rl  rp  rr  rt  rz  r|  r}  r~  r  r  r  r)   r)   r)   r/   rj  
  s    rj  c                  C   s$   t  } t| dd t| ddd d S )NFr@   TrA   rU   r=   rY   rR   r)   r)   r/   _non_threaded_conversion
     r  c                  C   s$   t  } t| dd t| ddd d S )NTr@   r  r  r  r)   r)   r/   _threaded_conversion
  r  r  c                   @   s  e Zd ZdZeje feje feje fej	e	 fej
e
 feje feje feje feje feje feje feje feje fejedfejee	 fgZdd Zdd Zdd Zd	d
 Zdd Zdd Zdd Zdd Zdd Zej !ddg diej"fdg diej"fdddgiej"fdddgiej#fdddgiej#fdddgiej#fgdd Z$dd Z%dd  Z&d!d" Z'd#d$ Z(d%d& Z)d'd( Z*d)d* Z+d+d, Z,d-d. Z-d/S )0TestConvertMiscz)
    Miscellaneous conversion tests.
    r\  c                 C   s   t dg di}t| d S )Nr   NNNr   r   r)   r)   r/   test_all_none_objects  s   z%TestConvertMisc.test_all_none_objectsc                 C   s0   t dg di}|d d|d< t| d S )Nr   r  r   )r4   r5   r   rY   r   r)   r)   r/   test_all_none_category  s   z&TestConvertMisc.test_all_none_categoryc                 C   s.   | j D ]\}}tjg |d}t||d qd S )Nr   r  )
type_pairsr6   r^   rm   )rx   r   pa_typerd   r)   r)   r/   test_empty_arrays  s   z!TestConvertMisc.test_empty_arraysc                 C   s
   t   d S r  )r  r  r)   r)   r/   test_non_threaded_conversion  s   
z,TestConvertMisc.test_non_threaded_conversionc                 C   s>   t d}z|t W |  |  d S |  |  w )Nr2   )mpZPoolapplyr  closejoin)rx   poolr)   r)   r/   %test_threaded_conversion_multiprocess  s   

z5TestConvertMisc.test_threaded_conversion_multiprocessc                 C   s   d}dd ddt jg}g d}dd ddt jg}t|| tj|| ddgd	t|| t|| tj|| g d
dd|| || || || || d
}t|}t| |D ]}t||  qZd S )Nr   r   r  rn  )r   r   r   r   r   ro  r  rq  r  )r  rn  r   Tr  r   )
Zcat_stringsZcat_strings_with_naZcat_intsZ
cat_binaryZcat_strings_orderedr  rL  rk  Zstrings2Zstrings3)r6   r^  r4   r  r5   rY   rm   )rx   rl  r{  r|  Zv3r>  rR   r5  r)   r)   r/   test_category&  s2   

zTestConvertMisc.test_categoryc                 C   sL   dd }t jg dddgdt jg dddgddg}|D ]}|| qd S )	Nc                 S   s.   t | }| }tt|t|  d S r  )rF   r^   rJ   rP   r`   r4   r_   )r.  rd   rX   r)   r)   r/   _checkB  s   
zBTestConvertMisc.test_category_implicit_from_pandas.<locals>._checkr+  r   r   r  Tr  )r4   r  )rx   r  r>  rd   r)   r)   r/   "test_category_implicit_from_pandas@  s   
z2TestConvertMisc.test_category_implicit_from_pandasc                 C   s    t dt g i}t| d S )Ncat)r4   r5   r  rY   r   r)   r)   r/   test_empty_categoryO  s   z#TestConvertMisc.test_empty_categoryc                 C   s   t  dft  dffD ]D\}}t g t t  |}| }tjg t	j
g |dd}tt|t| t d|i}| }td|i}t|| qd S )Nr   r   r   r  r   )rF   r<  r   r  
dictionaryr   rJ   r4   r  r6   r^   rP   r`   r_   rW   r5   rQ   )rx   r  r   r   rX   rS   rW   r)   r)   r/   test_category_zero_chunksT  s    z)TestConvertMisc.test_category_zero_chunkszdata,error_typer   )r   r   r|   r   Tr{   c                 C   sN   t |}d}tj||d tj| W d    d S 1 s w   Y  d S )N/Conversion failed for column a with type objectr  )r4   r5   r   r   rF   rH   rI   )rx   r   Z
error_typerR   rG  r)   r)   r/   test_mixed_types_failsa  s
   
"z&TestConvertMisc.test_mixed_types_failsc              	   C   sd  g }g d}d\}}t j|| d }g d}|D ]#}t  td ||| W d    n1 s9w   Y  q|t j	dd t
|| D td||  t j	g d	| td|| }d |d
< || |t jd|| dd||  |dktd d df }	|D ]}
tj|
|d}|d }t| t| t||	d qd S )Nr+  )r   rt   r   )
ru  rv  rw  rx  ry  rz  r{  r|  rR  rS  rB   c                 S      g | ]}t d qS r\  r   r-   r\  r)   r)   r/   r0     r1   z<TestConvertMisc.test_strided_data_import.<locals>.<listcomp>r   rL  r   r!   r#   r   r   r   r  )r6   r8   r9   rk   rL   rM   r   rV  r   r^   r:   r   Zreshaper7   r    r4   r5   rY   rm   )rx   casesr   NKZrandom_numbersZnumeric_dtypes	type_nameZboolean_objectsZstrided_maskcaserR   r  r)   r)   r/   test_strided_data_importr  sD   


z(TestConvertMisc.test_strided_data_importc                 C   sX   dd }|t jd gd td |t jtjgd td |t jd tjd gtd d S )Nc                 S   sT   t | }t|t jsJ t|dksJ |jdksJ |D ]	}|t ju s'J qd S )Nrt   )rF   r^   r   Z	NullArrayr   ri   r!  )ra   r  itemr)   r)   r/   _check_series  s   
z5TestConvertMisc.test_all_nones.<locals>._check_seriesrt   r   )r4   r_   r   r6   r^  )rx   r  r)   r)   r/   test_all_nones  s   zTestConvertMisc.test_all_nonesc                 C   s~   t dg dfdtjg dtjdfdg dfg}t|}ttdt	 tdt	 g}t
|||ddg |d d S )Nr   )r   r   r2   rt   r   r   )ir   r   r\  r   r   )r>   rS   rT   )r   r6   r^   r   r4   r5   rF   r>   rv   r   rY   )rx   r   rR   Zpartial_schemar)   r)   r/   test_partial_schema  s   




z#TestConvertMisc.test_partial_schemac                 C   s   t i }t|d d t|d dd t jt g d}t||dd t||ddd t ji g dd}t|dd t|ddd d S )	Nr   T)r?   rU   r   Fr   r   )rU   r?   )r4   r5   rY   r   )rx   rR   rS   r   r)   r)   r/    test_table_batch_empty_dataframe  s   
z0TestConvertMisc.test_table_batch_empty_dataframec              	   C   s   t jg t  d}ttjg td}t|	 ttjg tjd t jg t 
 d}t|	 | t jg t t  d}t|	 | t jg t t dt  gd}t|	 | d S )Nr  r   r   )rF   r^   r   r4   r_   r6   r   rP   r`   rJ   r<  r  r  rv   )rx   rd   Zempty_objectsr)   r)   r/   test_convert_empty_table  s   
"z(TestConvertMisc.test_convert_empty_tablec                 C   s   t dt jfdt jfg}t jddg|d}|jdksJ tj|d t d}| dd	gks2J tj|d t d}| d
dgksGJ dS )zt
        ARROW-2172: converting from a Numpy array with a stride that's
        not a multiple of itemsize.
        r.   r  )r  r   )r2   r   )r   r  r  r  r   r2   N)r6   r   r   r   r^   r  rF   ri  )rx   r   r   rd   r)   r)   r/   test_non_natural_stride  s   z'TestConvertMisc.test_non_natural_stridec                 C   sR   t jddt jdddd }tj|t d}tjg dt d}|| d S )	Nr   r\  r   r   r   r2   r  )r{   r}   r   g      @)r6   r7   r   rF   r^   r   rK   )rx   r  Zpa_arrrS   r)   r)   r/   #test_array_from_strided_numpy_array  s   z3TestConvertMisc.test_array_from_strided_numpy_arrayc                 C   s   t tdtdddd}ttdt tdt	 g}t
t tjj||d W d    n1 s:w   Y  tjj||d	d
}|djt	 ksTJ d S )Nabcr   r   rt   )rv  rw  rv  rw  r  Frg  )r4   r5   r   r6   ZlinspacerF   r>   rv   r<  r   r   r   r   rH   rI   rF  r[   rj  r)   r)   r/   test_safe_unsafe_casts  s   z&TestConvertMisc.test_safe_unsafe_castsc                 C   s   zt dt jdtjdgi}W n ty(   t dt dtjdgi}Y nw tjt	dd t
j| W d    d S 1 sBw   Y  d S )Nr   r   rt   zSparse pandas datar  )r4   r5   r>  ZSparseArrayr6   r^  AttributeErrorr   r   r  rF   rH   rI   r   r)   r)   r/   test_error_sparse  s   " "z!TestConvertMisc.test_error_sparseN).rL  rM  rN  rO  r6   r   rF   r   r   r   r   r   r   r   rU  r   r   rT  r<  rs  r  r  r  r  r  r  r  r  r  r  r  r   ra  rb  r  r  r  r  r  r  r  r  r  r  r  r  r)   r)   r)   r/   r  
  s^    	
+r  c                  C   sL   t g d} tjj| t dd}tjg dt d}||s$J d S )N)r   r2   Nr   Tr[   r   r  )r4   r_   rF   r  rI   r   r^   rK   )rl   rd   rS   r)   r)   r/   *test_safe_cast_from_float_with_nans_to_int  s   r  c                  C   s   t jt jddddttjddgtddg} t jddd}||	d	|	d
|d d d 	d	d
dddgd t ddgd 
djddgd tjdtjjddddt jddddt jddddd}t j|| dS )Nr  r   rD  r2   r   r  r   r\  r  r  r   TFr   r   r;   Z2013Mr   r   r   )r   r   r   )r   r   r2   rt   r   r   r   r   r   r  r\  r   )r4   r   r   r   repeatr6   Ztiler^   r   rJ  r   r_   rl   r8   r9   r]  period_rangeinterval_ranger5   )r   r:  r   r)   r)   r/   _fully_loaded_dataframe_example  s$   


r  r   ro  r   c                 C   s   t dt tj  krt dk rn ntd tj| d}tj|}tj|	 }|
|s3J |j
|js<J |jj|jjksFJ d S )Nr  r  r  r   )r   r4   r  r   r	  r5   rF   rH   rI   rJ   rK   r>   r   )r   rR   table1table2r)   r)   r/   !test_roundtrip_with_bytes_unicode%  s   &
r  c                 C   s>   | d }| d } | d }| d } | d }| d }t ||||S )Nr2  r  )r   )valr^  secondsminutesr  r)   r)   r/   _pytime_from_micros5  s   r  c                 C   s$   | j d | jd  | jd  | j S )Nl    $'- i r2  )hourminuter5  r1  )Zpytimer)   r)   r/   r,  ?  s   r,  c                  C   sh   G dd d} t d|  |  gi}d}tjt|d tj| W d    d S 1 s-w   Y  d S )Nc                   @   r  )z6test_convert_unsupported_type_error_message.<locals>.ANr  r)   r)   r)   r/   rv  J  r  rv  r   r  r  )r4   r5   r   r   r   rF   rH   rI   )rv  rR   rG  r)   r)   r/   +test_convert_unsupported_type_error_messageF  s   "r  c                 C   s,   |   }tj|| jdd}|| sJ d S )NTr  )rJ   rF   r^   r[   rK   )rd   ra   r   r)   r)   r/   test_array_to_pandas_roundtripX  s   r  c                 C   s   dd t | D }|| S )Nc                 S   r  r  )r   r  r)   r)   r/   r0   d  r1   z+_generate_dedup_example.<locals>.<listcomp>)r:   )nuniquerl  unique_valuesr)   r)   r/   _generate_dedup_examplec  s   r  c                 C   s   t dd | D |ksJ d S )Nc                 S   s   h | ]}t |qS r)   r  r,   r)   r)   r/   	<setcomp>i  r1   z"_assert_nunique.<locals>.<setcomp>)r   )r
  rS   r)   r)   r/   _assert_nuniqueh  s   r  c                  C   sp   d} d}t | |}tj|t dtj|t dt||gfD ]}t| |  t|jddt| q!d S )Nr   r\  r  Fdeduplicate_objects)	r  rF   r^   rs  r  r  r  rJ   r   )r  rl  rl   rd   r)   r)   r/   .test_to_pandas_deduplicate_strings_array_typesl  s   
r  c                  C   sz   d} d}t | |}t|}tj|gdg}tj|g}||fD ]}t| d |  t|jddd t	| q"d S )Nr   r\  r   Fr  )
r  rF   r^   rG   r   rH   r   r  rJ   r   )r  rl  rl   rd   r?  r@  r
  r)   r)   r/   .test_to_pandas_deduplicate_strings_table_typesx  s   

r  c                  C   sn   d} d}t tjjdd| d}d || d < t|| }t|jdd|  t|jdd	d
| d | d  d S )Nr   r\  i i ʚ;r;   r2   Tr  F)r  r  r   )r   r6   r8   r]  rF   r^   r  rJ   )r  rl  r  rd   r)   r)   r/   .test_to_pandas_deduplicate_integers_as_objects  s   r  c            	      C   s   d} d}t t| }ddddifddddifdd	i fdd
i fg}|D ]-\}}}tj|| |d}||}t|jdi ||  t|jdddi|t| q"d S )Nr   r\  r   r  r  Tr   r  r?  r>  r  r  Fr)   )r   r:   rF   r^   r  r  rJ   r   )	r  rl  r  r  Zraw_typeZ
array_typeZpandas_optionsZraw_arrZ
casted_arrr)   r)   r/   $test_to_pandas_deduplicate_date_time  s(   
r  c                  C   s   t g dtjddgd} ttjdt ddtjdt ddg}t	
t tjj| |d	 W d    d S 1 s>w   Y  d S )
N)333333?g @r   r<  r   r   r   Fre  r   r  )r4   r5   r6   r^  rF   r>   rv   r   r  r   r   r   rH   rI   )rR   r>   r)   r)   r/   /test_table_from_pandas_checks_field_nullability  s   

"r  c               	   C   s   t tdg dfdg dddgd d gfdg dfg} | g d	 }tdt fdtt fdt fg}tdt fdt fdtt fg}tjj	| d
d}tjj	|d
d}|j
|shJ |j
|spJ d S )N	partitionr   r   r   r   r>  r   rt   r   r  NNr   gffffff
@r  r  r>  Fr   )r4   r5   r   rF   r>   r   r  r   rH   rI   rK   )df1r   schema1schema2r  r  r)   r)   r/   6test_table_from_pandas_keeps_column_order_of_dataframe  s(   





r  c               	   C   s   t tdg dfdg dddgd d gfdg dfg} tdt fdtt fdt fg}| | jd	k }| | jd
k g d }tj	j
||dd}tj	j
||dd}|j|s_J |j|jshJ d S )Nr  r  r>  r   rt   r   r  r  r   r   r  Fr  )r4   r5   r   rF   r>   r   r  r   r  rH   rI   rK   )rR   r>   r  r   r  r  r)   r)   r/   3test_table_from_pandas_keeps_column_order_of_schema  s    



r  c               	   C   s   t tdg dfdg dddgd d gfdg dfg} g d	}tdtt fdt fdt fg}ddg}tdt fdt fg}tjj	| |d
d}tjj	| |d
d}|j
|sdJ |j
|slJ d S )Nr  r  r>  r   rt   r   r  r  )r>  r  r  F)r   r?   )r4   r5   r   rF   r>   r  r   r   rH   rI   rK   )rR   Zcolumns1r  Zcolumns2r  r  r  r)   r)   r/   ;test_table_from_pandas_columns_argument_only_does_filtering  s(   





r  c               	   C   s   t tdg dfdg dddgd d gfdg dfg} tdt fdtt fdt fg}ddg}t	t
 tjj| ||d	 W d    d S 1 sQw   Y  d S )
Nr  r  r>  r   rt   r   r  r  )r>   r   )r4   r5   r   rF   r>   r   r  r   r   r   r   rH   rI   )rR   r>   r   r)   r)   r/   @test_table_from_pandas_columns_and_schema_are_mutually_exclusive  s   



"r  c                  C   sz   t dg di} ttjdt ddg}tj| }|jdjdu s(J tjj| |d}|jdjdu s;J d S )Nr   r   Fre  Tr  )	r4   r5   rF   r>   rv   r   rH   rI   rf  rR   r>   rW   r)   r)   r/   /test_table_from_pandas_keeps_schema_nullability  s   r  c                  C   s  t g dg dd} tdt fdt fdt fg}tjtdd tj	j
| |d	 W d    n1 s:w   Y  d| j_t| |d
|d tjtdd tj	j
| |dd W d    n1 sfw   Y  tjtdd tj	j
| |d d W d    n1 sw   Y  t jg ddd| _t| |d |d t| |d
|d tdt fdt fdt fg}t| |d |d t| |d
|d tdt fdt fg}|  }|jd
d}t| |d ||d t| |d
||d t jjg dddgd| _tdt fdt fdt fdt fg}t| |d
|d t| |d |d tdt fdt fdt fg}|  }|jdd
d}t| |d
||d t| |d ||d d S )Nrs   r.  r   r   r   r   zname 'index' present in ther  r  T)r>   r?   rT   z'preserve_index=False' wasFr  z=name 'index' is present in the schema, but it is a RangeIndexr   r\   r1  )r>   r?   rT   rS   ))r   r   )r   r2   )r   r   Zlevel1Zlevel2r   )r4   r5   rF   r>   r   r   r   r   KeyErrorrH   rI   r   r]   rY   r   r   rk   r2  r   from_tuplesr<  )rR   r>   rS   r)   r)   r/   +test_table_from_pandas_schema_index_columns*  s   















r  c                  C   s   t g dg dd} tdt fdt fdt fg}tjj| dd}tjj| d|d	}|j	 
|s;J t jg dg ddg d
d} tj| }tjj| |d}|j	 
|scJ d S )Nrs   r.  r   r   r   r   Tr   )r?   r>   r   r   r  )r4   r5   rF   r>   r   r   SchemarI   rH   r  rK   )rR   rT   r>   rW   r)   r)   r/   :test_table_from_pandas_schema_index_columns__unnamed_index  s   


 r  c                  C   sH   t  } tj| ddi}tjj| |d}|jj	ddks"J d S )NmetaTruer  s   metas   True)
r4   r5   rF   r  rI   r  rH   r>   r   getr  r)   r)   r/   2test_table_from_pandas_schema_with_custom_metadata  s   r  c               	   C   s(  t t jddddtjdd} ttjdt	 ddtjd	tj
d
ddddg}tjj| |d}|j|s<J |jjd d }|d dksLJ |d d u sTJ |jjd d }|d d	ksdJ |d ddiksnJ | }ddi}tt jtdkrd|d	< | dd	g |}t|| d S )Nz2020-01-01T00:00:00ZHr2   )r   r   )r   r  r  Tre  r   ra   rF  rG  Fr  r   r   r]   r   r   r   r   r  zdatetime64[s, UTC])r4   r5   r   r6   r8   r9   rF   r>   rv   r   rH  rH   rI   rK   r   rJ   r   r  r   rP   rQ   )rR   r>   rW   Zmetadata_floatZmetadata_datetimerX   Zcoerce_cols_to_typesrS   r)   r)   r/   2test_table_from_pandas_schema_field_order_metadata  s,   
r  c                  C   sf   t tjg dddtjg dddtjdg dg dd} tj| }|	 }t
| | d S )	Nr   r2   rt   r   r   r   r   r   r   )r   r  Nr  rn  )FTFTF)r:  r;  c3c4Zc5)r4   r5   r6   r^   r8   r9   rF   rG   rI   rJ   rP   rQ   )r   r   rX   r)   r)   r/   test_recordbatch_from_to_pandas  s   
r  c                  C   s   t tjg dddtjg dddg dg dd} t tjd	d
gddtjddgddddgddgd}tj| }tj|}tj||g}|	 }t 
| |gjdd}t|| d S )N)r   r   r2   r   r   rz   r   )TNF)r   r  N)r:  r;  r  r  rt   r   r~   r   Tr  rn  r1  )r4   r5   r6   r^   rF   rG   rI   rH   r   rJ   r  r2  rP   rQ   )r  r  Zbatch1Zbatch2rW   rX   r   r)   r)   r/   test_recordbatchlist_to_pandas  s$   r  c                  C   sh   t jt g dgdgd} t jt g dgdgd}| d  jdks'J |d  jdks2J d S )Nr   Za0r   r   )rF   Zrecord_batchr^   rW   rJ   r]   )r?  r   r)   r)   r/   *test_recordbatch_table_pass_name_to_pandas  s   r  r[   rS   r  r    r   r   r   r   r   r   r   r   rU  r   r   r   r  r   )lengthr   zlist[list[int16]]r  rt   r$  r  r   r  rF  r   ra   r   c                 C   s   t | |ksJ d S r  )r   r  r)   r)   r/   test_logical_type  s   r  c                  C   s   d} t jtj| tjdtjjdd| dtjd}t	
  t  }| }t  || d  ks2J d }t	
  t  |ks@J t tj| tjd}t  }| }t  |ks[J d S )N'  r   r   r2   r;   r  r   )rF   r^   r6   r7   r   r8   r]  r   rn  rI  rJ  rK  rJ   )r  rd   prior_allocationr.   r)   r)   r/   test_array_uses_memory_pool  s   r  c                  C   sd   t t tjdtjdgdg} t| dd t  }|  }|d j	j
js(J t  |ks0J d S )Nr/  r   r  Tsplit_blocks)rF   rW   r^   r6   r7   r   !_check_to_pandas_memory_unchangedrK  rJ   Z_valuesflagsZ	writeable)r   r  rX   r)   r)   r/   test_singleton_blocks_zero_copy5  s   "r  c                 K   s,   t  }| jdi |}t  |ksJ d S )Nr)   )rF   rK  rJ   )r
  kwargsr  r.   r)   r)   r/   r  C  s   r  c                  C   s   t t jg dddt jg dddt jg dddt jg dddt jg dddt jg dddt jg dddt jg dddgdd	 td
D } t| d
 t| dd d S )Nr  ru  r  rw  rx  rR  rS  c                 S      g | ]}d  |qS zf{}formatr  r)   r)   r/   r0   V      z/test_to_pandas_split_blocks.<locals>.<listcomp>r   Tr  )rF   rW   r^   r:   _check_blocks_createdr  )r   r)   r)   r/   test_to_pandas_split_blocksK  s   	
r  c                 C   s   t tjt dk r| jS | jS )Nz1.1.0)r   r4   r  _dataZ_mgrr  r)   r)   r/   _get_mgr\  s   r  c                 C   s&   | j dd}tt|j|ksJ d S )NTr  )rJ   r   r  blocks)r   numberr.   r)   r)   r/   r
  c  s   r
  c                     s:   d  fdd} |  }t |ddd |  }t |dd d S )N2   c                      s(   t dd t D dd t D S )Nc                 S   s(   g | ]}t tjd ddd qS )r  Nr2   )rF   r^   r6   r8   r9   r  r)   r)   r/   r0   l  s    zEtest_to_pandas_self_destruct.<locals>._make_table.<locals>.<listcomp>c                 S   r  r  r  r  r)   r)   r/   r0   p  r	  )rF   rW   r:   r)   r  r)   r/   _make_tablek  s
   
z1test_to_pandas_self_destruct.<locals>._make_tableT)r   self_destruct)r  )r  )r  r   r)   r  r/   test_to_pandas_self_destructh  s   r  c                  C   s|   d} t tj| tjd}t |||gg d}t  }| }t  |d|  d  ks.J d }t	  t  |ks<J d S )Nr  r   )r  r  rQ  rt   r   )
rF   r^   r6   r7   r   rW   rK  rJ   rI  rJ  )r  rd   r   r  r.   r)   r)   r/   test_table_uses_memory_poolz  s   r  c                  C   sl   t ddig} |  }|jtdksJ |d }t|}t||ks(J ~t||d ks4J d S )Nr   r   r   r   )rF   r^   rJ   r   r6   sysgetrefcount)rd   r  r
  refcountr)   r)   r/   test_object_leak_in_numpy_array  s   
r  c                  C   s   t ddig} t | gdg}| d }|jtdks J |d }t|}t||ks2J ~t||d ks>J d S )Nr   r   r  r   r   )rF   r^   rW   rJ   r   r6   r  r  )rd   rW   r  r
  r  r)   r)   r/   test_object_leak_in_dataframe  s   
r  c                  C   s   ddgddgg} t  }t j| d |d}t j| t |d}tj| d tjd}ttj| d tjdtj| d tjdg}|j|ksGJ |t |sQJ |t |s[J d S )	Nr  g333333@g      "@g      E@r   r  r   r   )	rF   r   r^   r  r6   r4   r_   r[   rK   )r   r   r8  r  r  r  r)   r)   r/   test_array_from_py_float32  s   r  c            
      C   sR  t  } t| g}|jdjd}tjddd}tj	j
||d}|j|ks*J tj	j
|tdd}|d  |d  ksCJ |d  |d  ksQJ tjdgddtd	}tjd
gddtd}|jtd	kswJ td}tt || W d    n1 sw   Y  |j|dd}||sJ ttdtdtdg}	tjg dtdd}tt tj|	tdd W d    n1 sw   Y  tt tj	j
|	tdd W d    n1 sw   Y  tj	j
|	tddd}||sJ tj|	tddd}||s'J d S )Nz	tzlocal()r   r  rG  r  r   i r   r  {   ra   Fr   r   r\  r/  )r   r   r   r  )r   nowr4   r_   rI  rJ  Z
tz_convertrF   rH  r  rI   r[   rs  to_pydatetimer^   r  r   r   r   rK   rP  )
r  ra   Zs_nycZ
us_with_tzrd   r  rS   targetrX   r  r)   r)   r/   test_cast_timestamp_unit  s<   
"r   c                  C   sf   t j } |  }tj|gtjdddd}tj||gddg}|	 }t|}|
|s1J d S )Nr  r   rG  r  r   stop)r4   rP  r  r  rF   r^   rH  r  r   rJ   rK   )tsts_dtrd   r  rX   r   r)   r)   r/   (test_nested_with_timestamp_tz_round_trip  s   

r$  c            
      C   s  t j } |  }dD ]3}|dv rdd }ndd }tj| gt|d}tj| gtj|ddd}tj||gd	d
g}tj||gd	d
g}|	 }t
|d d	 tsXJ |d d	 jd u scJ t
|d d
 tsnJ |d d
 jd u syJ |	 }t
|d d	 tsJ |d d	 jd usJ |d d	 tj}	||	jd d||ksJ t
|d d
 tsJ |d d
 jd usJ td|i	 }t
|d d d	 tsJ |d d d	 jd u sJ t
|d d d
 tsJ |d d d
 jd u sJ td|i	 }t
|d d d	 tsJ |d d d	 jd us#J t
|d d d
 ts1J |d d d
 jd us?J qd S )NrE  )ra   r  c                 S   s   | j ddS )Nr   r0  r<  r.   r)   r)   r/   truncate  s    z/test_nested_with_timestamp_tz.<locals>.truncatec                 S   s   | S r  r)   r%  r)   r)   r/   r&    s    r  r   rG  r   r!  r   r  r   )r4   rP  r  r  rF   r^   rH  r  r   rJ   r   r   r  
astimezoner   r  r5  rW   )
r"  r#  r  r&  rd   r  Zarr3Zarr4rX   Zutc_dtr)   r)   r/   test_nested_with_timestamp_tz  sB   


r(  c               	   C   sH  t g dd} t jg dtd}t g d}dD ]}| |}tj||}tjj|||d}|d d	krQt	t
 |  W d    n1 sKw   Y  q| }tjj||d
}tt|t| | }	|	  dkswJ |}
|d d	kr||dd  }
tjjt |d|
|d
}tt|	t| qd S )Nr   r2   r  r   )FFTFFF)r   r   r   r   r   r   r   r   r  r   ur  r   r   )r6   r  r^   r   r   rF   r   r   r   r   r  rJ   r4   r  
from_codesrP   r`   r_   rh   rj   where)Zsrc_indicesr  rf   
index_typer   d1d2Zpandas1Z
ex_pandas1Zpandas2Zsigned_indicesZ
ex_pandas2r)   r)   r/   test_dictionary_with_pandas,  s6   

r/  c                    sd   |d ur|t jjdt|| d }nt j fddt| D td}|dkr0d |t j| |k < |S )Nr   r;   c                    s   g | ]}t  qS r)   r  r  	item_sizer)   r/   r0   T  r1   z"random_strings.<locals>.<listcomp>r   )r6   r8   r]  r   r^   r:   r   rand)r$  r1  Zpct_nullr  rX   r)   r0  r/   random_stringsP  s   r3  c                  C   s  t jd tjtdddd} tjtdddd}tjtdddd}tjt jjd	t	| d
dd| }tjt jjd	t	|d
dd|}tjt jjd	t	|d
dd|}tjt jjd	t	|d
ddt j
d
dk d}tj||}t| ||g}t||||g}	t|d|d|d|dg}
|	 }|
 }|jj| k sJ |d}d || < t|| d S )Ni90  r       r<  r  r  r  r\  r   r/  rw  r  r/  r  r+   )r6   r8   r  rF   r^   r3  r   r   r]  r   r2  Zconcat_arraysr  r  rJ   r  r  r@  r   rh   rP   r`   )r-  r.  Zd3r'  r  r8  rw  r9  Zexpected_dictr   Za_denserX   Zresult_denseZexpected_denser)   r)   r/   "test_variable_dictionary_to_pandas]  sB   
r5  c                  C   sf   t g d } t jg d| }| }ttjg dt	dtjddgt	dg}t
|| d S )N)r   r   r   r   r   r<  )r   r   r   r   r   )rF   r^   r  r  r   rJ   r4   r_   r6   r   rP   r`   )r!  rd   rX   rS   r)   r)   r/   (test_dictionary_encoded_nested_to_pandas  s   r6  c                  C   s   t g d} tt t }t| }| g dks J |j	|s(J t g d} t| }| g dks>J |j	|sFJ tj| t
g dd}| g dks\J |j	|sdJ d S )N)r   r   r   )r   r   Nr   )FFFTr  )r   r   NN)r4   r  rF   r  r   r<  r^   ri  r[   rK   r6   )r  r  rX   r)   r)   r/   test_dictionary_from_pandas  s   

r7  c                  C   sx  t jtjddgddtjddgtd} tjt t	 d}tj| |d}|j
|s/J | ddgks9J tjt t d}ttj tj| |d}W d    n1 s\w   Y  tjt t	 d	d
}d}tjt|d tj| |d}W d    n1 sw   Y  | ddgksJ tjt t	 d}tj| |tdd	gd}|j
|sJ | dd gksJ t g } tjt t	 d}tj| |d}|j
|sJ | g ksJ tjt t d}tj| |d}|j
|sJ | g ksJ t ddg} tj| t	 d}tjddgt	 d}||s/J | ddgks:J d S )Nr   r   r   r   r   r   )r,  
value_typer  T)r,  r8  r   z4The 'ordered' flag of the passed categorical values r  Fr  )r4   r  r*  r6   r^   r   rF   r  r   r<  r[   rK   ri  r   r   r   r   r  r   )r  typrX   rG  rS   r)   r)   r/   *test_dictionary_from_pandas_specified_type  sJ    
r:  c                  C   s   t tjt dk rtd tjdg didd} | d} t| d j	j
}t| d j	jj}t|tjs9J tj| d }tj||}||ksNJ d S )Nz1.3.0z:PyArrow backed string data type introduced in pandas 1.3.0r.   r   r  r   zstring[pyarrow]r   r   )r   r4   r  r   r	  r5   r   rF   r^   r  codesr  rl   r   r  rI   r   r   )rR   r   r  rS   rX   r)   r)   r/   :test_convert_categories_to_array_with_string_pyarrow_dtype  s   

r=  c                  C   s(  t dt jg dddi} t| }tg dt }|d d|s)J t	dt
 fg}tj| |d}tg dt
 }|d d|sPJ t| d }||s^J tj| d t
 d}||spJ t| d j}||sJ tj| d jt
 d}||sJ d S )Nr   rh  Int64r   r   r  r  )r4   r5   r_   rF   rW   r^   r   r  rK   r>   r   rl   )rR   rX   rS   r>   r  r)   r)   r/   test_array_protocol  s    
r?  c                       s0   e Zd Z fddZdd Zedd Z  ZS )DummyExtensionTypec                       t  t d d S )Nz,pyarrow.tests.test_pandas.DummyExtensionTypesuper__init__rF   r   r  	__class__r)   r/   rD       zDummyExtensionType.__init__c                 C      dS Nr  r)   r  r)   r)   r/   __arrow_ext_serialize__     z*DummyExtensionType.__arrow_ext_serialize__c                 C   s"   |dksJ |t  ksJ |  S rI  )rF   r   )clsZstorage_typeZ
serializedr)   r)   r/   __arrow_ext_deserialize__  s   z,DummyExtensionType.__arrow_ext_deserialize__)rL  rM  rN  rD  rJ  classmethodrM  __classcell__r)   r)   rE  r/   r@    s
    r@  c                 C   s&   t jg dt  d}t jt |S )Nrs   r  )rF   r^   r   ExtensionArrayfrom_storager@  )rx   r[   storager)   r)   r/   PandasArray__arrow_array__!  s   rS  c                 C   s   t jg dt  d}t jt |}| jtjj	dt
dd | jtjjdt
dd tjdddd	jtd
djfD ]C}t |}||sGJ t t|}||sVJ t t|}||seJ t td|idd}||s|J q9d S )Nrs   r  Z__arrow_array__F)Zraising
2012-01-01rt   Dr  r   r   r   r   )rF   r^   r   rP  rQ  r@  setattrr4   r>  ZPeriodArrayrS  ZIntervalArrayr  r  rK   r_   r   rW   r5   rF  r  )monkeypatchrR  rS   rd   rX   r)   r)   r/   *test_array_protocol_pandas_extension_types(  s(   
 rX  c                 C   s   t |tjr	|}n|d}| }tj|d dd|j|jt|  }|d }|d urAtj	
t t|d |g}t|}n	tjt|td}tjj| | dd}|S )Nr   r   r   r   F)rk   )r   rF   r  r  buffersr6   Z
frombufferoffsetr   ZBooleanArrayZfrom_buffersrn  rd  r  r    r4   r>  ZIntegerArrayrk   )rx   r^   rd   Zbuflistr   Zbitmaskrf   Zint_arrr)   r)   r/   _Int64Dtype__from_arrow__B  s    
r[  c                 C   sx  dd l m  m} tg dtjg dddg dd}t|}| }t	t
|jd |jr3J t
|jd jjtdksCJ t	t
|jd	 |jsPJ t|| td
tjg dddi}t|}| }t	t
|jd |jszJ t|| ttjtdk r| tjjjjd n
| tjjjjd | }tt
|jd	ksJ t	t
|jd |jrJ d S )Nr   rs   r  r>  r   )r   r   r   r+  r   r   r   rh  	1.3.0.dev__from_arrow__)pandas.core.internalscore	internalsr4   r5   r^   rF   rW   rJ   r   r  r  ExtensionBlockrl   r   r6   rP   rQ   r   r  delattrr>  integer_IntegerDtypeNumericDtyper   )rW  _intrR   rW   rX   r   r  r)   r)   r/   test_convert_to_extension_arrayX  s6   
 
rg  c                       s,   e Zd Z fddZdd Zdd Z  ZS )MyCustomIntegerTypec                    rA  )Nz-pyarrow.tests.test_pandas.MyCustomIntegerTyperB  r  rE  r)   r/   rD    rG  zMyCustomIntegerType.__init__c                 C   rH  rI  r)   r  r)   r)   r/   rJ    rK  z+MyCustomIntegerType.__arrow_ext_serialize__c                 C   s   t  S r  )r4   
Int64Dtyper  r)   r)   r/   r    s   z#MyCustomIntegerType.to_pandas_dtype)rL  rM  rN  rD  rJ  r  rO  r)   r)   rE  r/   rh  ~  s    rh  c                 C   s~  dd l m  m} tg dt }tjt |}t	d|i}|
 }tt|jd |js3J tjg ddd}t|| |
 }tt|jd |jsSJ tdtjg dddi}t|| ttjtdk r{| tjjjjd n
| tjjjjd |
 }tt|jd |jrJ tg d}t|| tt |
  W d    d S 1 sw   Y  d S )Nr   r   r   r>  r   r\  r]  )r^  r_  r`  rF   r^   r   rP  rQ  rh  rW   rJ   r   r  r  ra  r4   r_   rP   r`   r5   rQ   r   r  rb  r>  rc  rd  re  r   r   r   )rW  rf  rR  rd   rW   rX   rS   r)   r)   r/   /test_conversion_extensiontype_to_extensionarray  s4   
"rj  c               	   C   s4  t dt g dt  i} |  }|d jtdksJ | jt  t ij	d}t
|d jtjs7J t dt g dt  i} | jt  t ij	d}|d jtdks_J t tdtjg dddi} |  }t
|d jtjsJ | jt  tdij	d}t
|d jtjsJ d S )	Nr   rs   r   types_mapperr   r>  r   rU  )rF   rW   r^   r   rJ   r   r6   r4   ri  r  r   r   r5   ZPeriodDtype)rW   rX   r)   r)   r/   'test_to_pandas_extension_dtypes_mapping  s   "rm  c                  C   s   t tjt dk rtd tjddddjtddjfD ],} t| 	 }t
| }t|| td	| id	 	 }tj
| d	d
}t|| qd S )Nz1.1z'ExtensionDtype to_pandas method missingrT  rt   rU  r  r   r   r  r\   )r   r4   r  r   r	  r  r^   r  rF   rJ   r_   rP   r`   rW   )rd   rX   rS   r)   r)   r/   test_array_to_pandas  s   

rn  c                  C   sT   t jt jdddd} t| }| jt jdddddddddgdd	k d S )
Nr   rt   )r   endr   r   )r  r  r2   r   r   )r4   r5   r  rF   rW   rJ   r   r   )rR   rW   r)   r)   r/   5test_roundtrip_empty_table_with_extension_dtype_index  s   
rp  r   r   r   c                 C   s~   t tjt dk rtd tjddgddgddgd	tt d
	| }|
 }t|}|jtjd}t|| d S )Nz1.5.0zArrowDtype missingr   r2   rt   r   r   r   r+  r   rk  )r   r4   r  r   r	  r5   Z
ArrowDtyperF   r   r   rk   rW   rJ   rP   rQ   )r   rR   rS   rW   rX   r)   r)   r/   !test_to_pandas_types_mapper_index  s   

rq  c                  C   s   t tjt dk rtd tg dt } t t ij	}| j
|d}|jt ks0J t d ij	}| j
|d}|jtdksGJ t t ij	}| j
|d}|jtdks`J d S Nz1.2.0z$Float64Dtype extension dtype missingrs   rk  r   )r   r4   r  r   r	  rF   r^   r   ri  r  rJ   r   r6   r   Float64Dtyper   rl  rX   r)   r)   r/   !test_array_to_pandas_types_mapper  s   
ru  c                  C   s   t tjt dk rtd ttg dt g} t	| tj
s$J t t ij}| j|d}|jt ks<J t d ij}| j|d}|jtdksSJ t t ij}| j|d}|jtdkslJ d S rr  )r   r4   r  r   r	  rF   r  r^   r   r   ry  ri  r  rJ   r   r6   r   rs  rt  r)   r)   r/   )test_chunked_array_to_pandas_types_mapper  s   
rv  c                  C   s|  g d} g d}t j| dd}t j|dd}t jg ddd}d}d}tjd	| itjd
ddddd}t jj||gd	dgd}|dt	dgd d ddddidgd	d	ddd dddddd dgddi}|
 }	t|	| tjd| itjd
ddddd}
t jj||gd|gd}|dt	|gd d ddddidgd	d	ddd dd|ddd dgddi}|
 }t||
 tjd	| itjd
ddd dd}t jj||gd	|gd}|dt	|gd d ddddidgd	d	ddd dd |ddd dgddi}|
 }t|| tjd	| itjd
dddd|gd}t jj|||gd	d|gd}|dt	d|gd d ddddidgd	d	ddd dddddd dd |ddd dgddi}|
 }t|| tjd	| itjd
ddd d|gd}t jj|||gd	||gd}|dt	||gd d ddddidgd	d	ddd dd |ddd dd |ddd dgddi}|
 }t|| d S )N)r   r  Nr  )r   r   r   r   r  r  rB  r   r   Z__index_level_1__r   r   r   r2   rn  r   r   r      pandasr   r   r   r   )r]   r   r   r   r   z0.23.4)r   r   r   rB  )rF   r^   r4   r5   r   rH   r   replace_schema_metadatajsondumpsrJ   rP   rQ   )a_valuesb_valuesa_arrowb_arrowZrng_index_arrowZ
gen_name_0Z
gen_name_1e1r%  r1e2r&  r2Ze3Zt3Zr3Ze4Zt4Zr4Ze5Zt5Zr5r)   r)   r/   )test_metadata_compat_range_index_pre_0_12(  s  


r  c                  C   s   g d} g d}t j| dd}t j|dd}tj| |dtjddd	d
dd}t ||d}|dtd d d dddgd ddddd ddddgdd
dd	ddgddi}|	 }t
|| d S )Nr   )r   r   r   r  r   r  r  r   r   r   r2   rn  r   r   rw  r   zmixed-integer)r   r   r]   r   r   r   )r   r]   r   r   r   r   r:   )kindr]   r   r   r!  z0.25.0)r   r   r   rB  )rF   r^   r4   r5   r   rW   rx  ry  rz  rJ   rP   rQ   )r{  r|  r}  r~  rS   rW   rX   r)   r)   r/   'test_metadata_compat_missing_field_name  sV   r  c                  C   sL   t d} ttjtjdd| dd}|jj}|d d d dks$J d S )	Nr   r   r   r\   r   r   r]   6)	r6   r   rF   rW   r4   r5   r   r>   r   r]   rW   r   r)   r)   r/   .test_metadata_index_name_not_json_serializable  s   
r  c                  C   sF   d} t tjtjdd| dd}|jj}|d d d dks!J d S )Nr   r   r   r\   r   r   r]   )rF   rW   r4   r5   r   r>   r   r  r)   r)   r/   -test_metadata_index_name_is_json_serializable  s   r  c               
   C   s   t tddtddtddtddgtddtddtddtddgd	} | jj| jjftd
tdfksDJ | S )Nz0001-01-01 00:00r  z2012-05-02 12:35z2012-05-03 15:42z3000-05-03 15:42z1991-01-01 00:00rE  z2050-05-03 15:42)
dateTimeMs
dateTimeNsr  zM8[ns])r4   r5   r6   r  r  r   r  r  r)   r)   r/   make_df_with_timestamps  s    








r  z)ignore:Parquet format '2.0':FutureWarningc                 C   sP   t  }tj|}| d }tj||dd t|}|jdd}t	|| d S )Nztimestamps_from_pandas.parquetz2.0)versionTZtimestamp_as_object)
r  rF   rH   rI   pqZwrite_tableZ
read_tablerJ   rP   rQ   )tempdirrR   rW   filenamerX   r   r)   r)   r/    test_timestamp_as_object_parquet,  s   
r  c                  C   s.   t  } tj| }|jdd}t| | d S )NTr  )r  rF   rH   rI   rJ   rP   rQ   )rR   rW   r   r)   r)   r/   %test_timestamp_as_object_out_of_range:  s   r  
resolutionrE  r   r   rI  i  i  c                 C   s   t j|gt j| |dd}t d|i}|jdd|jddd fD ]9}|jtks+J t|d ts4J |rH|d j	d us?J |d j	
|}n|d j	d u sQJ |}|d |ks[J q"d S )NrG  r  r   Tr  r   )rF   r^   rH  rW   rJ   r   r   r   r   r  fromutc)r  r   rI  rd   rW   rX   rS   r)   r)   r/   'test_timestamp_as_object_non_nanosecondC  s   
r  c               	   C   sl   t d} dd l}| d}||ddddd}td	t|gi}|jd
d}t||ks4J d S )Nr  r   x   i  r   r  r  9   Ztimestamp_colTr  )	r   r  r   r  localizerF   rW   r^   rJ   )r  r   r   rI  rW   rX   r)   r)   r/   %test_timestamp_as_object_fixed_offset\  s   

r  c                   C   s   t d d S )Nzpandas_threaded_import.py)r
   r)   r)   r)   r/   test_threaded_pandas_importh  s   r  c                  C   s^   t d} t t j| ksJ tdt  gi}tj|}|  t t j| ks-J d S )NZm8r   )	r6   r   rS  r4   r5   rF   rH   rI   rJ   )rS   rR   r   r)   r)   r/   $test_does_not_mutate_timedelta_dtypel  s   
r  c                  C   sf   ddl m}  d| dddig}| ddd	d
g}t||d}| }|d d  |d ks1J d S )Nr   )r   timedelta_1r  r   )r  r^  rt   (   r  )r  r  r  )r  timedelta_2r  )r   r   rF   rW   rJ   Zto_pytimedelta)r   r  r  rW   rR   r)   r)   r/   %test_does_not_mutate_timedelta_nestedx  s    r  c                  C   sP  t t dt t t  t t dt  gg} t jdggd dddifgddd ifdddifgggg| d	}t	
ddggd dddifgddd ifdddifgggi}t	
dd
d igd dddiidd iddidggi}| }|jdd}t|| t|| t jj|| d	}||sJ t jj|| d	}||sJ d S )Nr   r   )r   Nr2   s   abcrt   r   s   defr  r   rc  strictmaps_as_pydicts)rF   r>   rv   r  r  r   r  rs  rW   r4   r5   rJ   rP   rQ   rH   rI   rK   )r>   rW   Zexpected_default_dfZexpected_as_pydicts_dfZ
default_dfZas_pydicts_dfZtable_default_roundtripZtable_as_pydicts_roundtripr)   r)   r/   ,test_roundtrip_nested_map_table_with_pydicts  sJ   "r  c               	      s  t ddg} t g d}t g t  }t ddgddggt t  }t g ddd	gd
dggt t  }t g t t  }t jddg| |}t jg d||}t jddg||}t t jddg|dt jddg|ddt jddg|dg}	|		 }
t
dddgfdddgfggdd
dgfggg g}|	j	dd}t
ddgddgdgdd
dgigg g}|	dd}|	 }t
dd
dgfggg g}|j	dd}t
dd
dgigg g}t ' tddt t|
| t|| t|| t|| W d   n	1 sw   Y  t t t  t t   dt
jddf fdd}||
|	 |||	 ||| ||| dS )z=
    Slightly more robust test with chunking and slicing
    r   r  )r  rn  r
  r  r   r   r   r  Nr   r  r  r   r2   )r   rt   r   r   r  r  r  r   r  rt   rB   rC   r  returnc                    s4   t t jj|  d}|jdd ||sJ d S )Nr  Tr  )rF   r  r  rI   r  rK   )r  r   Zarray_roundtripr  r)   r/   assert_roundtrip  s   zMtest_roundtrip_nested_map_array_with_pydicts_sliced.<locals>.assert_roundtrip)rF   r^   r<  r  r  r   r  r  r  rJ   r4   r_   rL   rM   rN   rO   rP   r`   r  )Zkeys_1Zkeys_2Zkeys_3Zitems_1Zitems_2Zitems_3Zmap_chunk_1Zmap_chunk_2Zmap_chunk_3r  series_defaultexpected_series_defaultseries_pydictsexpected_series_pydictsZslicedZseries_default_slicedZexpected_series_default_slicedZseries_pydicts_slicedZexpected_series_pydicts_slicedr  r)   r  r/   3test_roundtrip_nested_map_array_with_pydicts_sliced  sz   
 


r  c                  C   s6  t g d} t ddgddgddggt t  }dd	g}t j|| |}t t  t t  }tt j	j
 |jd
d W d    n1 sLw   Y  |jdd}tddgddgdg}|t jj||drqJ | }tdddgfdddgfdddgfgg}|t jj||dsJ t|t|ksJ t||D ]2\}	}
t|	t|
ksJ t|	 |
 D ]\}}|d |d ksJ t|d |d sJ qqt|t|ksJ t||D ]0\}	}
t|	t|
ksJ t|	|
D ]\}}|d |d ks
J t|d |d sJ qqd S )Nr;  r   r   r   r  r   r  r   rt   r  r  lossyr  r  r   r  r   )rF   r^   r  r<  r  r   r  r   r   r  ZArrowExceptionrJ   r4   r_   rK   r  rI   r   r`  r  r6   array_equal)r=  r  r=  mapsr  r  r  r  r  row1row2tup1tup2r)   r)   r/   4test_roundtrip_map_array_with_pydicts_duplicate_keys   sH    r  c            
   	   C   s\  t ddgddgg dgd dgddggt t  } t g d	}g d
}t j|| |}tt |j	dd W d    n1 sCw   Y  |	 }t
ddgdfddgdfgg dfdgdfd dgdfddgdfgg}t|t|ksxJ t||D ].\}}t|t|ksJ t||D ]\}}	t|d |	d sJ |d |	d ksJ qq}d S )Nr   r   r   r  r   r  r  rc  )r   r  r  rn  r
  r  )r   r2   r   r  r  r   r  r  rn  r
  r  r   r   )rF   r^   r  r<  r  r   r   r   r  rJ   r4   r_   r   r`  r6   r  )
r=  r  r=  r  r  r  r  r  r  r  r)   r)   r/   %test_unhashable_map_keys_with_pydicts4  s.    $r  c                  C   s^   t jt jdddddd} tdt| i}|d}| }|jdks'J t	
||  d S )N2012r2   r,  r-  Zdatetime_columnr\   )r4   r_   r   rF   rW   r^   rF  rJ   r]   rP   r`   )r  rW   Z	table_colrX   r)   r)   r/   )test_table_column_conversion_for_datetimeU  s   
r  c                  C   s8   t t jdddd} t| }| }t||  d S )Nr  r2   r,  r-  )r4   r_   r   rF   r^   rJ   rP   r`   )r  rd   rX   r)   r)   r/   "test_array_conversion_for_datetimed  s   
r  c            	   
   C   s  ddd} dd }| t d|||gi d|i}| t d|||gi |g}| t d|||gi |g}| t d	|||gi d|i}| t d
|||gi d|i}tt ttdt fg}td|fg}| t d|||gi|d d S )Nc                 S   sN   t jj| |d}|jdd |dj}|dksJ t|jddd|  d S )Nr  Tr  r   r   r  )r  r  )	rF   rH   rI   r  rF  rz  rP   rQ   rJ   )rR   r>   tabrz  r)   r)   r/   	roundtripv  s   
z-test_nested_chunking_valid.<locals>.roundtrip   0i T*rk  Zstruct_fieldr  r  lossolr   r  r  r  )	r4   r5   rF   r  r<  r  r  rs  r>   )	r  r.   r  r  r  r  Z
map_of_losZmap_typer>   r)   r)   r/   test_nested_chunking_validp  s&   



r  c                   C   s   t d d S )Nzarrow_39313.py)	test_utilr
   r)   r)   r)   r/   !test_is_data_frame_race_condition  s   r  )r   )NFNTNFF)NNr  r  )r   N)rI  r$  ry  multiprocessingr  r  rL   collectionsr   r   r   r   r   r   Z
hypothesisrc  Zhypothesis.strategiesZ
strategiesre  numpyr6   Znumpy.testingr  ro   r   Zpyarrow.pandas_compatr   r	   Zpyarrow.tests.utilr
   r   r   Zpyarrow.tests.strategiesZtestsrg  utilr  Zpyarrow.vendored.versionr   ZpyarrowrF   r   r  ImportErrorZpandasr4   Zpandas.testingrP   Zpandas_examplesr   r   r   r  r  Znumpy.exceptionsra  Z
pytestmarkr=   rY   re   rm   rq   rr   rP  rb  r  r  r  ri  r  r  rF  rj  r  r  r  r  r  r  r  r,  r  rd  r>  Zpandas_compatible_typesr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r#  rn  r   r   r   r   r   r   r   r   rU  r   r   r  r  rs  r<  r  r  rH  r4  r3  r  r  r  r  r  r  r
  r  r  r  r  r  r   r$  r(  r/  r3  r5  r6  r7  r:  r=  r?  ZExtensionTyper@  rS  rX  r[  rg  rh  rj  rm  rn  rp  rq  ru  rv  r  r  r  r  r  rN   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r)   r)   r)   r/   <module>   s  
      '

      Y[    # c<  	



Z%

















14$,5&&	

 *3
	 -N4!&