o
    թZhQ                     @  s  d Z ddlmZ ddlmZ ddlZddlZddlZddlmZ ddlZddl	Z	ddl
Z
ddlmZ ddlZddlZddlZddlmZ ddlZddlZddlZddlZddlmZmZ dd	lmZ dd
lmZ ddlm  m Z! ddl"Z#ddl"m$Z$m%Z%m&Z&m'Z' ddl(m)Z* ddl+m,Z, ddl-m	  m.Z/ ddl0m1Z1m2Z2 dd Z3ej45ddde6de7de8dedg de7d9dde7d9ddddd e:dj;ddde:dj;ddde:dj;ddddddddf gdd Z<dd  Z=d!d" Z>d#d$ Z?dqd(d)Z@ej45d*ejAe>d+d,ejAe#jBd-d,ejAeje#jBejCd.d/d,ejAeje#jBd0d.d1d,ejAeje#jBd2d.d3d,gej45d4e#jBe>gej45d5e@e, d6d7 ZDd8d9 ZEd:d; ZFejGd<d= ZHG d>d? d?ZIG d@dA dAZJej45dBdCgdDe%g dEfdFe#jKLg dEg dGgfgdHdI ZMdJdK ZNej45dLg dMdNdO ZOdPdQ ZPG dRdS dSejQZRdTdU ZSdVdW ZTdXdY ZUdZd[ ZVej45d\d]d^d\ed_d`d`dagdbdc ZWddde ZXdfdg ZYe!jZdhdi Z[ej45djej\ejCgdkdl Z]dmdn Z^dodp Z_dS )ra<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    )annotations)arrayN)partial)Path)Any)get_lzma_fileis_platform_little_endian)import_optional_dependency)flatten_buffer)	DataFrameIndexSeriesperiod_range)create_pickle_data)DayMonthEndc                 C  s   t |trt||  d S |drt| | d S |dkr4|tju r,| tju s*J d S | |ks2J d S ttd| dtj	}|| | d S )NZsp_	timestampZassert_Z_equal)

isinstancer   tmassert_index_equal
startswithZassert_equalpdZNaTgetattrZassert_almost_equal)resultexpectedtypZ
comparator r   R/var/www/html/lang_env/lib/python3.10/site-packages/pandas/tests/io/test_pickle.pycompare_element>   s   


r   datas   123s   123456I         B)r$   r#   r#      )r$      C)orderFc                 C  s   t | }t| d}||ksJ t| ttfr || u sJ d S t|trA|jdks,J |jdks3J |js8J |j	|j
fksCJ d S d S )NAr"   r%   )r
   
memoryviewtobytesr   bytes	bytearrayndimformat
contiguousshapenbytes)r   r   r   r   r   r   test_flatten_bufferT   s   

r5   c           
   	   C  s  t  std ttjdD ]}| |}t|}|	 D ]\}}|	 D ]\}}|| | }|dkr|dkrt
|| |jj|jjksIJ |jjjrPJ t
|dk|dk |jj}|td tdksjJ |tjdd }	t|	tjszJ |	tjddd	ksJ |tjdd
 }	t|	tjsJ |	tjdddksJ q'|dkr|dkrt
|| t|jtsJ |jt ksJ |jdksJ t
|d|d q'|dkr|dv rt
|| q'|dkr|dv rt
|| q't||| q'qqd S )Nz"known failure on non-little endianzdata/legacy_pickle/*/*.p*kl*Zseriestsr   r"   r#   )hours)daysr7   )nanoseconds)r8   r9   indexZperiodM)Zdt_tzcatframe)Zdt_mixed_tzsZ
cat_onecolZcat_and_float)r   pytestskipr   __file__parentglobr   read_pickleitemsr   assert_series_equalr:   freq	normalizer   Z	Timedeltar   r   r   freqstrshiftassert_frame_equalr   )
datapathlegacy_pickler   r   Zdvdtr   r   rF   resr   r   r   test_picklesq   sF   

rO   c                 C  @   t |d}tj| |dd W d    d S 1 sw   Y  d S Nwbprotocolopenpickledumpobjpathfhr   r   r   python_pickler      "r^   c                 C  sB   t | d}|d t|W  d    S 1 sw   Y  d S )Nrbr   )rW   seekrX   load)r\   r]   r   r   r   python_unpickler   s   
$rc   dictreturnlist[tuple[str, Any]]c                 C  s   dd |   D S )zFlatten create_pickle_datac                 S  s&   g | ]\}}|  D ]}||fq
qS r   )values).0r   ZexamplesZexampler   r   r   
<listcomp>   s    zflatten.<locals>.<listcomp>)rD   )r   r   r   r   flatten   s   rj   pickle_writerpython)idZpandas_proto_defaultrT   Zpandas_proto_highestr'   Zpandas_proto_4   Zpandas_proto_5writerztyp, expectedc              	   C  s   t  i}||| t|}t|||  t|}t|||  t|dd}||| |d W d    n1 s:w   Y  t|dd}t|}|d W d    n1 sZw   Y  t|||  W d    d S 1 spw   Y  d S )NrR   moder   r`   )r   ensure_cleanr   rC   r   rc   rW   ra   )r   r   rk   ro   r\   r   handler   r   r   test_round_trip_current   s    




"rt   c                  C  `   t dtdd ttdtdtdd tdD tdd	} t	| j
tj}t| | d S )
N皙?x      r'   ABCDZdtypec                 S     g | ]}d | qS zi-r   rh   ir   r   r   ri          z,test_pickle_path_pathlib.<locals>.<listcomp>ry   columnsr:   )r   nparangereshaper   listobjectranger   round_trip_pathlib	to_pickler   rC   rJ   dfr   r   r   r   test_pickle_path_pathlib      r   c                  C  ru   )
Nrv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri      r   z.test_pickle_path_localpath.<locals>.<listcomp>ry   r   )r   r   r   r   r   r   r   r   r   Zround_trip_localpathr   r   rC   rJ   r   r   r   r   test_pickle_path_localpath   r   r   c                   C  s   dt   dS )N__z	__.pickle)uuiduuid4r   r   r   r   get_random_path   s   r   c                   @  sT   e Zd ZejZdd Zdd Zej	
dg ddd Zd	d
 Zdd Zdd ZdS )TestCompressionc           	   	   C  s  |d u rt || d S |dkrt|d}n|dkr"t|d}n|dkrJtj|dtjd}|	|t
j| W d    n1 sDw   Y  nb|dkrt|d/}tj|dd}||t
j|}||| W d    n1 svw   Y  W d    n1 sw   Y  n!|d	krt |d}n|d
krtd|d}n	d| }t||dvrt|d*}| |	|  W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S d S )Ngzipwbz2zipcompressiontarr`   rp   xzZzstdZ	zstandardrR   zUnrecognized compression type: )r   r   )shutilcopyfiler   rW   r   BZ2FilezipfileZipFileZIP_DEFLATEDwriteosr\   basenametarfile
gettarinfoaddfiler   r	   
ValueErrorread)	selfZsrc_path	dest_pathr   fr]   r   tarinfomsgr   r   r   compress_file   sH   
"zTestCompression.compress_filec                 C  sL  |}|d }|d }t |}t |o}tdtdd ttdtdtdd	 t	d
D tdd}|j
||d t j||d$}	t|d}
|
|	  W d    n1 s\w   Y  W d    n1 skw   Y  tj|d d}t || W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )N.compressed.rawrv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri   '  r   z7TestCompression.test_write_explicit.<locals>.<listcomp>ry   r   r   rR   )r   rr   r   r   r   r   r   r   r   r   r   decompress_filerW   r   r   r   rC   rJ   )r   r   r   basepath1path2p1p2r   r   r]   df2r   r   r   test_write_explicit  s&   Pz#TestCompression.test_write_explicitr   ) NonebadZ7zc              	   C  s   t jtddM t|/}tdtdd t	t
dtdt	dd	 td
D tdd}|j||d W d    n1 s?w   Y  W d    d S W d    d S 1 sWw   Y  d S )NzUnrecognized compression typematchrv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri   >  r   z;TestCompression.test_write_explicit_bad.<locals>.<listcomp>ry   r   r   )r>   Zraisesr   r   rr   r   r   r   r   r   r   r   r   r   )r   r   r   r\   r   r   r   r   test_write_explicit_bad7  s   "z'TestCompression.test_write_explicit_badc                 C  sX  |}|| }|d }| j | }t|}t|m}tdtdd t	t
dtdt	dd td	D tdd
}	|	| tj||d$}
t|d}||
  W d    n1 sbw   Y  W d    n1 sqw   Y  tj|d d}t|	| W d    n1 sw   Y  W d    d S W d    d S 1 sw   Y  d S )Nr   rv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri   L  r   z4TestCompression.test_write_infer.<locals>.<listcomp>ry   r   r   rR   )_extension_to_compressiongetlowerr   rr   r   r   r   r   r   r   r   r   r   r   rW   r   r   r   rC   rJ   )r   compression_extr   r   r   r   r   r   r   r   r   r]   r   r   r   r   test_write_inferB  s(   
Pz TestCompression.test_write_inferc           
   	   C  s   |}|d }|d }t |b}t |D}tdtdd ttdtdtdd	 t	d
D tdd}|j
|d d | j|||d tj||d}	t ||	 W d    n1 s\w   Y  W d    d S W d    d S 1 stw   Y  d S )Nr   r   rv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri   e  r   z6TestCompression.test_read_explicit.<locals>.<listcomp>ry   r   r   )r   rr   r   r   r   r   r   r   r   r   r   r   r   rC   rJ   )
r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_explicit\  s   Pz"TestCompression.test_read_explicitc              	   C  s  |}|d }|| }| j | }t|`}t|B}tdtdd t	t
dtdt	dd td	D tdd
}	|	j|d d | j|||d t|}
t|	|
 W d    n1 sbw   Y  W d    d S W d    d S 1 szw   Y  d S )Nr   rv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri   |  r   z3TestCompression.test_read_infer.<locals>.<listcomp>ry   r   r   )r   r   r   r   rr   r   r   r   r   r   r   r   r   r   r   r   rC   rJ   )r   r   r   r   r   r   r   r   r   r   r   r   r   r   test_read_inferr  s   
PzTestCompression.test_read_inferN)__name__
__module____qualname__icomZextension_to_compressionr   r   r   r>   markparametrizer   r   r   r   r   r   r   r   r      s    

r   c                   @  s&   e Zd Zejdg ddd ZdS )TestProtocolrU   )rS   r   r"   r#   c                 C  s   t |;}tdtdd ttdtdtdd t	dD tdd	}|j
||d
 t|}t || W d    d S 1 sCw   Y  d S )Nrv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri     r   z*TestProtocol.test_read.<locals>.<listcomp>ry   r   rT   )r   rr   r   r   r   r   r   r   r   r   r   r   rC   rJ   )r   rU   r   r\   r   r   r   r   r   	test_read  s   
"zTestProtocol.test_readN)r   r   r   r>   r   r   r   r   r   r   r   r     s    r   pickle_fileexcolsztest_py27.pkl)abcztest_mi_py27.pkl)r+   r%   r(   c                 C  s*   | ddd|}t |}t|j| d S )Nior   rX   )r   rC   r   r   r   )rK   r   r   r\   r   r   r   r   test_unicode_decode_error  s   
r   c               	   C  s   t  c} tdtdd ttdtdtdd t	dD tdd	}t
| d
}|| W d    n1 s:w   Y  t
| d}t|}W d    n1 sTw   Y  t || W d    d S 1 sjw   Y  d S )Nrv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri     r   z0test_pickle_buffer_roundtrip.<locals>.<listcomp>ry   r   rR   r`   )r   rr   r   r   r   r   r   r   r   r   rW   r   r   rC   rJ   )r\   r   r]   r   r   r   r   test_pickle_buffer_roundtrip  s   
"r   mockurl)zhttp://url.comzftp://test.comzhttp://gzip.comc                   s   dd }G dd d t  F fdd}tdtdd	 ttd
tdtdd t	dD tdd}|| | 
d| t|}t || W d    d S 1 sXw   Y  d S )Nc                 S  rP   rQ   rV   rZ   r   r   r   r^     r_   z3test_pickle_generalurl_read.<locals>.python_picklerc                   @  s6   e Zd ZdddZdd Zdd Zd	d
 Zdd ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsere   r   c                 S  s0   t |d| _d|v rddi| _d S ddi| _d S )Nr`   r   zContent-Encodingr   )rW   fileheaders)r   r\   r   r   r   __init__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c                 S  s   | S Nr   r   r   r   r   	__enter__     z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c                 W  s   |    d S r   )close)r   argsr   r   r   __exit__  s   z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c                 S  
   | j  S r   )r   r   r   r   r   r   r        
z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc                 S  r   r   )r   r   r   r   r   r   r     r   z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeNre   r   )r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s    
r   c                    s    S r   r   )r   kwargsr   r\   r   r   mock_urlopen_read  s   z6test_pickle_generalurl_read.<locals>.mock_urlopen_readrv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri     r   z/test_pickle_generalurl_read.<locals>.<listcomp>ry   r   zurllib.request.urlopen)r   rr   r   r   r   r   r   r   r   r   setattrr   rC   rJ   )Zmonkeypatchr   r^   r   r   r   r   r   r   test_pickle_generalurl_read  s   


"r   c                  C  s   t d t ; d} tdtdd tt	dt
dtdd	 td
D t
dd}||  t| }t|| W d    d S 1 sGw   Y  d S )NZfsspeczmemory://mockfilerv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri     r   z0test_pickle_fsspec_roundtrip.<locals>.<listcomp>ry   r   )r>   Zimportorskipr   rr   r   r   r   r   r   r   r   r   r   r   rC   rJ   )r   r   r   r   r   r   test_pickle_fsspec_roundtrip  s   



"r   c                   @  s   e Zd ZdddZdS )MyTzre   r   c                 C  s   d S r   r   r   r   r   r   r     r   zMyTz.__init__Nr   )r   r   r   r   r   r   r   r   r     s    r   c                  C  sD   t tdt f} t| }t|d | d  t|d ts J d S )Nr{   r   r"   )r   r   r   r   round_trip_picklerE   r   )r   r   r   r   r   test_read_pickle_with_subclass  s   
r   c                 C  s   t dtdd ttdtdtdd tdD tdd	}t	 }|j
|| d
 t| }W d   n1 s<w   Y  t }|j
|| d
 |d | |ks]| dv s]J tj|| d
}|d t|| dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    rv   rw   rx   rz   r{   c                 S  r|   r}   r   r~   r   r   r   ri     r   z9test_pickle_binary_object_compression.<locals>.<listcomp>ry   r   r   Nr   )r   r   r   )r   r   r   r   r   r   r   r   r   rr   r   r   
read_bytesr   BytesIOra   getvaluer   rC   rJ   )r   r   r\   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression  s    


r   c                 C  s8   | }|}dd }|| ||j  || ||j  d S )Nc                 S  s   t | }t | | d S r   )r   r   rJ   )r=   	unpickledr   r   r   _test_roundtrip7     
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr=   r   r   r   r   +test_pickle_dataframe_with_multilevel_index0  s   
r   c                  C  sH   t dddd} ttjdt| | }t|}|j	j
dks"J d S )Nz1/1/2011z1/1/2012r;   )rF   r#   )r   r   r   randomZdefault_rngZstandard_normallenr   r   r:   rH   )Zprngr6   Znew_tsr   r   r   "test_pickle_timeseries_periodindexA  s   
r   namei	  g     H@i     )r"   r#   c                 C  s0   t ttjdtjd| d}|j| ksJ d S )N
   r{   )r   )r   r   r   r   r   Zfloat64r   )r   r   r   r   r   test_pickle_preserve_nameI  s   r  c                 C     t | }t ||  d S r   r   r   rE   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimesQ  r   r  c                 C  r  r   r  )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_stringsV  r   r  c                  C  sf   t tddjdg } t| }|jjd jdksJ |jjd j	dks(J t
|dg |  d S )Nabccategoryr   r"   )r"   T)r   r   ZastypeZilocr   r   Z_mgrblocksr0   r3   rE   )ZserrN   r   r   r    test_pickle_preserves_block_ndim[  s
   
r	  rU   c                 C  s>   t td}tt|j| |dttj|d}t|| d S )Ni )rU   r   r   )	r   r   r   r   r   r   r   rC   rJ   )rU   r   r   r   r   r   r   %test_pickle_big_dataframe_compressionh  s   r
  c                 C  sf   | t tjdddd}t|d}t|}W d    n1 s w   Y  tg g d}t|| d S )Nr   rL   z1.2.4zempty_frame_v1_2_4-GH#42345.pklr`   )r:   r   )	r   r@   rA   rW   rX   rb   r   r   rJ   )rK   r\   fdr   r   r   r   r   #test_pickle_frame_v124_unpickle_130s  s   r  c                  C  s\   t dg di} d}tjt|d t }| |d W d    d S 1 s'w   Y  d S )Nr   r!   zpStarting with pandas version 3.0 all arguments of to_pickle except for the argument 'path' will be keyword-only.r   Zinfer)r   r   Zassert_produces_warningFutureWarningr   r   r   )r   r   r   r   r   r    test_pickle_pos_args_deprecation  s   "r  )r   rd   re   rf   )`__doc__
__future__r   r   r   datetime	functoolsr   r   r   r   pathlibr   rX   r   r   typingr   r   r   numpyr   r>   Zpandas.compatr   r   Zpandas.compat._optionalr	   Zpandas.compat.compressorsr
   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r   r   r   r   Zpandas._testingZ_testingr   Z-pandas.tests.io.generate_legacy_storage_filesr   Zpandas.io.commoncommonr   Zpandas.tseries.offsetsr   r   r   r   r   r/   r,   PickleBuffercastr   r   r5   rO   r^   rc   rj   paramr   HIGHEST_PROTOCOLrt   r   r   Zfixturer   r   r   Z
MultiIndexZfrom_arraysr   r   r   r   tzinfor   r   r   r   r   r  r  r  Zskip_array_manager_invalid_testr	  DEFAULT_PROTOCOLr
  r  r  r   r   r   r   <module>   s    	$
2
	

 


)	



