o
    +ifQ"                     @   s   d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
Z
 d dlZd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZmZ eeZdd ZG dd deZdS )    N)ArgumentParser)Path)copyfilermtree)	Generator)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)dataset_module_factoryimport_main_class)VerificationMode)ERROR
get_loggerc                 C   s4   t | j| j| j| j| j| jp| j| j| j	| j
| j
S )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cachenum_proc)args r   O/var/www/html/corbot_env/lib/python3.10/site-packages/datasets/commands/test.py_test_command_factory   s   
r   c                   @   s\   e Zd ZdZedefddZdedededed	ed
edededede	fddZ
dd ZdS )r   Fparserc                 C   s   | j ddd}|jdtd dd |jdtd dd |jd	td d
d |jdddd |jdddd |jdddd |jdddd |jdddd |jdtd dd |jdddd |jdtdd |jtd d S )NtestzTest dataset implementation.)helpz--namezDataset processing name)typedefaultr!   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr!   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz
--num_proczNumber of processesz--save_infoszalias to save_infor   zName of the dataset to download)r"   r!   )func)
add_parseradd_argumentstrintset_defaultsr   )r   test_parserr   r   r   register_subcommand&   sB   zTestCommand.register_subcommandr   r   r   r   r   r   r   r   r   r   c                 C   sf   || _ || _|| _|| _|| _|| _|| _|| _|	| _|
| _	|	r*|s*t
d td |r1d| _d S d S )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset script.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cache	_num_procprintexit)selfr   r   r   r   r   r   r   r   r   r   r   r   r   __init__K   s$   
zTestCommand.__init__c              	      s^  t dt jd urjrtd td jj}t	|t
j jr3 jr3t jnd}dttd d f f fdd}t| D ]\}}td|jj d|d  d	| d
 tjtj| tjj|_|jjsytjntjj rt!j"nt!j#dj$d |%  j&r|&  j&rtj  tjj'}t(|jd }tj||}tj)|rtj*|}	ntj)|r|}	ntj+|r|}	n	d }	td|  |	d urtj|	tjj'}
t,||
 td|
  j-r(tj+|j.rt/0d|j.  t1|j. tjj.tjj2}tj+|r(t/0d|  t1| qKtd d S )NfilelockzABoth parameters `config` and `all_configs` can't be used at once.r.   returnc                  3   s    j r7 jr7t jD ](\} }djv r$ djjdjV  q d|jjjdjV  qd S djv rL djjdjV  d S  djjdjV  d S )Nconfig_name)r   r   )r?   r   r   r   )r3   BUILDER_CONFIGS	enumeratebuilder_kwargsr1   r2   r   )iconfigbuilder_clsr?   moduler;   r   r   get_buildersv   s6   



 z%TestCommand.run.<locals>.get_builderszTesting builder 'z' (/)F)download_modeverification_modetry_from_hf_gcsr   z.pyzDataset card saved at zClearing cache at zTest successful.)3logging	getLoggersetLevelr   r0   r3   r9   r:   r/   r
   r   module_pathr@   lenr   r   rA   rD   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer6   r	   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr5   r   	NO_CHECKS
ALL_CHECKSr8   
as_datasetr4   REPOCARD_FILENAMEr   isfiledirnameisdirr   r7   r1   loggerwarningr   DOWNLOADED_DATASETS_DIR)r;   rT   
n_buildersrH   jbuilderdataset_readme_pathr   combined_pathdataset_diruser_dataset_readme_pathdownload_dirr   rE   r   runl   sn   
"$


zTestCommand.runN)__name__
__module____qualname____test__staticmethodr   r-   r)   boolr*   r<   rp   r   r   r   r   r   #   s4    $	

!r   )rN   rS   argparser   pathlibr   shutilr   r   typingr   datasets.configrX   datasets.builderr   datasets.commandsr   "datasets.download.download_managerr	   datasets.loadr
   r   datasets.utils.info_utilsr   datasets.utils.loggingr   r   rq   re   r   r   r   r   r   r   <module>   s     