o
    TZhB                     @   s|  d Z ddlZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ G dd	 d	ejZd
d Zdd Zdd Zeje
dk rJdd Znejjdd e
dje
dje
djfv rgdd Zeedd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zeje
dk reed%d& ZdS ejjdd e
dje
dje
djfv reed'd( ZdS dS ))zPExtends `dill` to support pickling more types and produce more consistent dumps.    N)BytesIO)CodeTypeFunctionType)version   )configc                       sF   e Zd Zejejj ZdZ	d	ddZ
 fddZdd Z  ZS )
PicklerFTc           	      C   s:  t |}|| jvrdtjv rdd l}||ju rt|t dtjv r2dd l}t	||j
r2t|t dtjv rFdd l}||ju rFt|t dtjv rsdd l}t	||jr[t|t ||ju rft|t t	||jjrst|d|}dtjv rdd l}t	||jrt|t |tu rt|d|}tjj| ||d	 d S )
Nregexr   spacytiktokentorchZ	_orig_modtransformersZ_torchdynamo_orig_callable)save_persistent_id)typedispatchsysmodulesr	   Patternpklregister_save_regexPatternr
   
issubclassZLanguage_save_spacyLanguager   Encoding_save_tiktokenEncodingr   ZTensor_save_torchTensor	Generator_save_torchGeneratornnModulegetattrr   ZPreTrainedTokenizerBase)_save_transformersPreTrainedTokenizerBaser   dillr   save)	selfobjr   obj_typer	   r
   r   r   r    r&   K/var/www/html/lang_env/lib/python3.10/site-packages/datasets/utils/_dill.pyr"      s:   








zPickler.savec                    sd   | j r	t |S zt|}W n ty(   ddlm  t| fddd}Y nw tj| | d S )Nr   Hasherc                    s     | d S )Nr   )hash)xr(   r&   r'   <lambda>R   s    z)Pickler._batch_setitems.<locals>.<lambda>key)	_legacy_no_dict_keys_sortingsuper_batch_setitemssorted	Exceptiondatasets.fingerprintr)   r!   r   )r#   items	__class__r(   r'   r1   H   s   zPickler._batch_setitemsc                 C   s"   t |turtj| | d S d S N)r   strr!   r   memoize)r#   r$   r&   r&   r'   r:   U   s   zPickler.memoize)T)__name__
__module____qualname__r!   _dillZMetaCatchingDictr   r   copyr/   r"   r1   r:   __classcell__r&   r&   r6   r'   r      s    
)r   c                    s    fdd}|S )z'Register a custom reducer for the type.c                    s   | t j < | S r8   )r   r   )functr&   r'   proxy^   s   
zpklregister.<locals>.proxyr&   )rC   rD   r&   rB   r'   r   [   s   r   c                 C   s   t |dd|  dS )zPickle an object to a file.T)recurseN)r   dumpr$   filer&   r&   r'   rF   e   s   rF   c                 C   s   t  }t| | | S )zPickle an object to a string.)r   rF   getvaluerG   r&   r&   r'   dumpsj   s   
rJ   z0.3.6c                 C   s   t jj| d S r8   )r!   r>   loginfopicklermsgr&   r&   r'   rK   s   s   rK      z0.3.7z0.3.8c                 C   s   t jj| | d S r8   )r!   r>   loggertracerM   r&   r&   r'   rK   |   s   c                 C   sl   t | d|  zt|f}W n ty&   ddlm} t||jdf}Y nw | jt||d t | d d S )NzSe: r   r(   r-   r$   z# Se)rK   r2   r3   r4   r)   r*   save_reduceset)rN   r$   argsr)   r&   r&   r'   	_save_set   s   rW   c                 C   sD   dd l }t| d|  |j|jf}| j|j||d t| d d S )Nr   zRe: rS   z# Re)r	   rK   patternflagsrT   compile)rN   r$   r	   rV   r&   r&   r'   r      s
   r   c                 C   sL   dd l }t| d|  |j|j|j|jf}| j|j||d t| d d S )Nr   zEnc: rS   z# Enc)r   rK   nameZ_pat_strZ_mergeable_ranksZ_special_tokensrT   r   )rN   r$   r   rV   r&   r&   r'   r      s
   r   c                    sT   dd l   fdd}t| d|  |   f}| j|||d t| d d S )Nr   c                    s
     | S r8   )Z
from_numpy)Znp_arrayr   r&   r'   create_torchTensor   s   
z-_save_torchTensor.<locals>.create_torchTensorzTo: rS   z# To)r   rK   detachcpunumpyrT   )rN   r$   r]   rV   r&   r\   r'   r      s   r   c                    sL   dd l   fdd}t| d|  | f}| j|||d t| d d S )Nr   c                    s      }||  |S r8   )r   Z	set_state)state	generatorr\   r&   r'   create_torchGenerator   s   
z3_save_torchGenerator.<locals>.create_torchGeneratorzGe: rS   z# Ge)r   rK   Z	get_staterT   )rN   r$   rc   rV   r&   r\   r'   r      s   
r   c                    sP   dd l   fdd}t| d|  |j| f}| j|||d t| d d S )Nr   c                    s(    j | d d }|| }||S )NZnlplang)utilZget_lang_classZfrom_config
from_bytes)r   bytesZlang_clsZ	lang_instr
   r&   r'   create_spacyLanguage   s   

z1_save_spacyLanguage.<locals>.create_spacyLanguagezSp: rS   z# Sp)r
   rK   r   to_bytesrT   )rN   r$   ri   rV   r&   rh   r'   r      s   r   c                 C   sX   t | d|  |j}d|v rt|d tri |d< | jt|d||d t | d d S )NzTok: cacher&   )ra   r$   z# Tok)rK   __dict__
isinstancedictrT   r   )rN   r$   ra   r&   r&   r'   r       s   r    c                 C   s^  t jjd|  |jds.t|jtj	j
dkr)|jtj	j
d ds.|jdkr0dntj	|j}d}t jjrt|drc|j|j|j|j|j|j|j|j|j|j||j||j|j|jf}n;|j|j|j|j|j|j|j|j|j||j||j|j|jf}n|j|j|j|j|j|j|j|j||j||j|j|jf}| jt||d	 t jjd
 dS )z
        From dill._dill.save_code
        This is a modified version that removes the origin (filename + line no.)
        of functions created in notebooks or shells for example.
        zCo: <   
ipykernel_<lambda> co_posonlyargcountrS   # CoN)r!   r>   rK   rL   co_filename
startswithlensplitospathsepco_namebasenamePY3hasattrco_argcountru   co_kwonlyargcount
co_nlocalsco_stacksizeco_flagsco_code	co_constsco_namesco_varnames	co_lnotabco_freevarsco_cellvarsrT   r   rN   r$   rw   co_firstlinenorV   r&   r&   r'   
_save_code   s   



r   c                 C   s(  t jj| d| |jds-t|jtj	j
dkr(|jtj	j
d ds-|jdkr/dntj	|j}d}t|drh|j|j|j|j|j|j|j|j|j|j|j||j|j||j|j|j|j|j|jf}nt|d	r|j|j|j|j|j|j|j|j|j|j|j||j|j||j|j|j|jf}nlt|d
r|j|j|j|j|j|j|j|j|j|j|j||j||j|j|jf}nDt|dr|j|j|j|j|j|j|j|j|j|j||j||j|j|jf}n|j|j|j|j|j|j|j|j|j||j||j|j|jf}| j t jj!||d t jj| d d S )NzCo: %sro   rp   rq   rr   rs   rt   co_endlinetableco_exceptiontableco_linetableru   rS   rv   )"r!   r>   rQ   rR   rw   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   ru   r   r   r   r   r   r   r   r   Zco_qualnamer   r   Zco_columntabler   r   r   rT   Z_create_coder   r&   r&   r'   	save_code;  s   






r   )__doc__r{   r   ior   typesr   r   r!   	packagingr   rt   r   r   r   rF   rJ   ZDILL_VERSIONparserK   releaserU   rW   r   r   r   r   r   r    r   r   r&   r&   r&   r'   <module>   sN   @





		
^


