o
    TZho(                     @   s   d Z ddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZ eeZG d
d dZdS )z Mock download manager interface.    N)Path)CallableListOptionalUnion)ZipFile   )cached_pathhf_github_url)
get_logger)Versionc                   @   s  e Zd ZdZdZdZ				d5dededeeef d	e	e d
e
de
de	ee  fddZedd Zedd Zedd Zdd Zedd Zedd Zedd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zd/d0 Zd1d2 Z d3d4 Z!dS )6MockDownloadManagerZ
dummy_dataZdatasetsFNTdataset_nameconfigversion	cache_diruse_local_dummy_dataload_existing_dummy_datadownload_callbacksc                 C   sH   d| _ || _|| _|| _|| _|pg | _|| _t|| _d | _	d | _
d S )Nr   )Zdownloaded_sizer   r   r   r   r   r   strversion_name_dummy_file_bucket_url)selfr   r   r   r   r   r   r    r   ^/var/www/html/lang_env/lib/python3.10/site-packages/datasets/download/mock_download_manager.py__init__&   s   



zMockDownloadManager.__init__c                 C   s   | j d u r
|  | _ | j S N)r   download_dummy_datar   r   r   r   
dummy_fileA   s   

zMockDownloadManager.dummy_filec                 C   s0   | j d urtjd| j j| jS tjd| jS )Ndummy)r   ospathjoinnamer   r   r   r   r   dummy_data_folderG   s   
z%MockDownloadManager.dummy_data_folderc                 C   s   t j| jdS )Nzdummy_data.zip)r"   r#   r$   r&   r   r   r   r   dummy_zip_fileO   s   z"MockDownloadManager.dummy_zip_filec                 C   s8   | j du r| jn| j}t|| jddd}tj|| jS )NT)r   Zextract_compressed_fileZforce_extract)	r   local_path_to_dummy_datagithub_path_to_dummy_datar	   r   r"   r#   r$   dummy_file_name)r   Zpath_to_dummy_data_dirZ
local_pathr   r   r   r   S   s   
z'MockDownloadManager.download_dummy_datac                 C   s   t j| j| j| jS r   )r"   r#   r$   datasets_scripts_dirr   r'   r   r   r   r   r(   ^   s   z,MockDownloadManager.local_path_to_dummy_datac                 C   s*   | j d u rt| j| jtjd| _ | j S )N/)r   r
   r   r'   replacer"   sepr   r   r   r   r)   b   s   
z-MockDownloadManager.github_path_to_dummy_datac                 C   s8   t j| jr
| jS d| jt jddd d S Nr,   )r"   r#   isdirr    r$   r-   r.   splitr   r   r   r   
manual_dirh   s   $zMockDownloadManager.manual_dirc                 G   sP   | j r| j}n| j}t|tr| ||S t|ttfr"| ||S | 	||S r   )
r   r    r*   
isinstancedictcreate_dummy_data_dictlisttuplecreate_dummy_data_listcreate_dummy_data_single)r   data_urlargsr    r   r   r   download_and_extractq   s   
z(MockDownloadManager.download_and_extractc                 G   
   |  |S r   r=   )r   r;   r<   r   r   r   download      
zMockDownloadManager.downloadc                 C   r>   r   r?   )r   r;   Zcustom_downloadr   r   r   download_custom   rA   z#MockDownloadManager.download_customc                 O   s   |S r   r   )r   r#   r<   kwargsr   r   r   extract      zMockDownloadManager.extractc                 C   s   i S r   r   r   r   r   r   get_recorded_sizes_checksums   rE   z0MockDownloadManager.get_recorded_sizes_checksumsc           	         s   i }|  D ]B\}}| jD ]}t|tr|D ]}|| qq|}|| qt|tr4 fdd|D }n|}tj tj	t
|j}|||< qtdd | D rktt| t| k rkdd |  D }|S )Nc              	      s(   g | ]}t j tjt|jqS r   )r"   r#   r$   urllibparse
quote_plusr   r%   ).0xpath_to_dummy_datar   r   
<listcomp>   s   ( z>MockDownloadManager.create_dummy_data_dict.<locals>.<listcomp>c                 s   s    | ]}t |tV  qd S r   )r4   r   )rJ   ir   r   r   	<genexpr>   s    z=MockDownloadManager.create_dummy_data_dict.<locals>.<genexpr>c                 S   s   i | ]	\}}||| qS r   r   )rJ   keyvaluer   r   r   
<dictcomp>   s    z>MockDownloadManager.create_dummy_data_dict.<locals>.<dictcomp>)itemsr   r4   r7   r"   r#   r$   rG   rH   rI   r   r%   allvalueslenset)	r   rM   r;   Zdummy_data_dictrQ   Zsingle_urlsdownload_callback
single_urlrR   r   rL   r   r6      s&   





&z*MockDownloadManager.create_dummy_data_dictc           	   	   C   s   g }t dd |D }t dd |D }|r#|s|r#|d gt| }|D ]!}| jD ]}|| q*tj|tj|	dd }|
| q%|S )Nc                 s   s     | ]}t td |V  qdS )z[0-9]{3,}-of-[0-9]{3,}N)boolrefindallrJ   urlr   r   r   rP      s    z=MockDownloadManager.create_dummy_data_list.<locals>.<genexpr>c                 s   s    | ]}| d V  qdS )z3https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmedN)
startswithr^   r   r   r   rP      s    

r   r,   r0   )rU   rW   r   r"   r#   r$   rG   rH   rI   r2   append)	r   rM   r;   Zdummy_data_listZis_tf_recordsZis_pubmed_recordsrZ   rY   rR   r   r   r   r9      s   

 z*MockDownloadManager.create_dummy_data_listc                 C   sN   | j D ]}|| qtj|tj|dd }tj|s#| j	s%|S |S r/   )
r   r"   r#   r$   rG   rH   rI   r2   existsr   )r   rM   r;   rY   rR   r   r   r   r:      s   

 z,MockDownloadManager.create_dummy_data_singlec                 C      d S r   r   r   r   r   r   delete_extracted_files   rE   z*MockDownloadManager.delete_extracted_filesc                 C   rc   r   r   r   r   r   r   manage_extracted_files   rE   z*MockDownloadManager.manage_extracted_filesc                 #   sj     fdd}t |} jr||n|d}|D ]}| r2|jds2|| |dfV  qd S )Nc                 3   sr    t  jj}| |}t j}| }W d    n1 s w   Y  |D ]}|| r6|	|V  q'd S r   )
r   r    parentrelative_tor   r(   namelistr`   as_posixjoinpath)r#   Zdummy_parent_pathZrelative_pathzip_filemembersmemberr   r   r   _iter_archive_members   s   

z?MockDownloadManager.iter_archive.<locals>._iter_archive_members*.__rb)	r   r   rglobis_filer%   r`   rg   ri   open)r   r#   rn   
file_paths	file_pathr   r   r   iter_archive   s   
z MockDownloadManager.iter_archivec                 c   s    t |ts	|g}|D ]:}tj|r|V  qt|D ](\}}}tj|dr+q|  t	|D ]}|dr;q3tj
||V  q3qqd S )Nrp   )r4   r7   r"   r#   isfilewalkbasenamer`   sortsortedr$   )r   pathsr#   dirpathdirnames	filenamesfilenamer   r   r   
iter_files   s"   

zMockDownloadManager.iter_files)NFTN)"__name__
__module____qualname__r*   r+   Zis_streamingr   r   r   r   r[   r   r   r   propertyr    r&   r'   r   r(   r)   r3   r=   r@   rB   rD   rF   r6   r9   r:   rd   re   ry   r   r   r   r   r   r   !   s`    








r   )__doc__r"   r\   urllib.parserG   pathlibr   typingr   r   r   r   zipfiler   Zutils.file_utilsr	   r
   Zutils.loggingr   Zutils.versionr   r   loggerr   r   r   r   r   <module>   s   