o TZhBZ@sddlZddlZddlZddlZddlZddlmmZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZdd lmZm Z dd l!m"Z"ee#Z$dZ%ddZ&GdddeZ'edGdddeZ(dS)N)ArgumentParser)Path)Optional)config)BaseDatasetsCLICommand)DownloadConfig)DownloadManager)MockDownloadManager)dataset_module_factoryimport_main_class) deprecated) get_loggerset_verbosity_warning) map_nestedzutf-8c Cs*t|j|j|j|j|j|j|j|j|j SN) DummyDataCommandpath_to_dataset auto_generaten_lines json_fieldxml_tagmatch_text_fileskeep_uncompressed cache_direncoding)argsrS/var/www/html/lang_env/lib/python3.10/site-packages/datasets/commands/dummy_data.pydummy_data_command_factorysrcseZdZfddZfddZfddZ     dd ed eed eed eed eede f ddZ    ddeded ed eed eed eed eedefddZ e de fddZddZZS)!DummyDataGeneratorDownloadManagercs(tj|i|||_g|_g|_dSr)super__init__mock_download_managerdownloaded_dummy_pathsexpected_dummy_paths)selfr"rkwargs __class__rrr!*s z*DummyDataGeneratorDownloadManager.__init__cs@t|}|j|}t|jj|ddt|jj|dd|SNT)Z map_tuple)r downloadr"rr#appendr$r%Z url_or_urlsoutputZ dummy_outputr'rrr*0s  z*DummyDataGeneratorDownloadManager.downloadcsHtt|}|j|}t|jj|ddt|jj|dd|Sr))r extractr*r"rr#r+r$r,r'rrdownload_and_extract7s  z6DummyDataGeneratorDownloadManager.download_and_extractNrrrrrreturnc Cstjtj|jj|jj|jjdddd}d|j_t |j |j D]!\}}tj|jj|jj|jj|} ||j || |||||d7}q!|dkrLt d|dkS)N dummy_dataTexist_okrFrrrrrzDummy data generation failed: no dummy files were created. Make sure the data files format is supported by the auto-generation.)osmakedirspathjoinr"datasets_scripts_dir dataset_namedummy_data_folderload_existing_dummy_datazipr#r$_create_dummy_dataloggererror) r%rrrrrtotalsrc_pathZrelative_dst_pathdst_pathrrrauto_generate_dummy_data_folder>s@   zADummyDataGeneratorDownloadManager.auto_generate_dummy_data_folderrCrDc sX|pt}tj|rftd|t|jgd}tfdd|D} |durAtj |} | dD] } | t | | O} q6| rt|j j dddt||dH} t|d |d(} g}t| D]\}}|krln||qb| d |Wdn1swYWdd SWdd S1swYd Sd vr7t||d|} t| }|dur||}t|trtd d|Dstdt|dfdd|D}n|d}|dur||i}t|j j dddt|d |d} t|| Wdn1swYWdd SWdd S1s0wYd StfdddDr[|durOtdd S|j ||||dd Std|ddStj!|rd}t"|D]3\}}}|D]*}|#dstj||}tj|t|$|}||j%||||||d7}q{qt|SdS)Nz#Trying to generate dummy data file )z.txtz.csvz.jsonlz.tsvc3|]}|vVqdSrr.0 extensiondst_path_extensionsrr wzGDummyDataGeneratorDownloadManager._create_dummy_data..,Tr4parentsrwz.jsoncss|]}t|tVqdSr) isinstancelist)rHvrrrrLszCouldn't parse columns z\. Maybe specify which json field must be used to read the data with --json_field .csi|] \}}||dqSrr)rHkrW)rrr szHDummyDataGeneratorDownloadManager._create_dummy_data..c3rFrrrGrJrrrLrM)z.xmlz.txmzEFound xml file but 'xml_tag' is set to None. Please provide --xml_tag)rrzCouldn't generate dummy file 'z9'. Ignore that if this file is not useful for dummy data.r.r5)&DEFAULT_ENCODINGr6r8isfiler@debugrsuffixesanybasenamesplitfnmatchparentmkdiropen enumerater+writer9stripjsonloadrUdictallvalues ValueErrorrVkeysitemsdumpwarning_create_xml_dummy_dataisdirwalk startswith relative_tor?)r%rCrDrrrrrZline_by_line_extensionsZis_line_by_line_text_file file_namepatternsrc_fileZdst_fileZ first_linesilineZ json_dataZfirst_json_datarBr8_filesnameZ src_file_pathZ dst_file_pathr)rKrrr?hs                 z4DummyDataGeneratorDownloadManager._create_dummy_datac Cst|jjdddt||dI}d}g}tj|ddD])\}} |dkr*|| q|} | j|krE||kr<|d7}q|rE|d  | qtj | d j ||dWddS1s\wYdS) NTrOrQr)startend)eventsrrT)element) rrcrdreETZ iterparser+poptagremove ElementTreerg) rCrDrrrrzZn_linerPeventelemr}rrrrss    "z8DummyDataGeneratorDownloadManager._create_xml_dummy_datacCsTtj||jj}tj|d}d}td|dt|d||t |dS)Nr2z"Compressing dummy data folder to 'z.zip'r>) r6r8r9r"r<r@infoshutil make_archivermtree)r%rroot_dir base_namebase_dirrrr!compress_autogenerated_dummy_datas zCDummyDataGeneratorDownloadManager.compress_autogenerated_dummy_data)r0NNNN)NNNN)__name__ __module__ __qualname__r!r*r/intrstrboolrEr? staticmethodr[rsr __classcell__rrr'rr)s\     / TrzThe `datasets` repository does not host the dataset scripts anymore. Therefore, dummy data is no longer needed to test their loading with CI.c@seZdZedefddZdededede ede ed e ed ed e ed e efd dZ ddZ de efddZ ddZ dS)rparsercCs|jddd}|jdddd|jdtd d d |jd tdd d |jdtddd |jdtddd |jdddd|jdtddd |jdtddtd |jdtdd|jtddS)Nr2zGenerate dummy data.)helpz--auto_generate store_truez!Automatically generate dummy data)actionrz --n_linesr0zBNumber of lines or samples to keep when auto-generating dummy data)typedefaultrz --json_fieldzOptional, json field to read the data from when auto-generating dummy data. In the json data files, this field must point to a list of samples as json objects (ex: the 'data' field for squad-like files)z --xml_tagz[Optional, xml tag name of the samples inside the xml files when auto-generating dummy data.z--match_text_fileszOptional, a comma separated list of file patterns that looks for line-by-line text files other than *.txt or *.csv. Example: --match_text_files *.labelz--keep_uncompressedzWhether to leave the dummy data folders uncompressed when auto-generating dummy data. Useful for debugging for to do manual adjustements before compressing.z --cache_dirzKCache directory to download and cache files when auto-generating dummy dataz --encodingz=Encoding to use when auto-generating dummy data. Defaults to rz/Path to the dataset (example: ./datasets/squad))rr)func) add_parser add_argumentrrr[ set_defaultsr)rZ test_parserrrrregister_subcommandsTz$DummyDataCommand.register_subcommandrrrrrrrrrc Cs||_tj|r|tjddd|_n |tjddd|_tj|p+t j }||_ ||_ ||_ ||_||_||_||_| |_dS)N/r)_path_to_datasetr6r8rtreplacesepra _dataset_name expanduserrZHF_DATASETS_CACHE_auto_generate_n_lines _json_field_xml_tag_match_text_files_keep_uncompressed _cache_dir _encoding) r%rrrrrrrrrrrrr! s  zDummyDataCommand.__init__c Cs>tt|j}t|j}|jpdg}g}t|}|D];}|r#|jnd}|||j |d}|r2|j n|j j } t |j || ddd} |jrP||j|| |jdq|j|| dq|jr}|jst|rltd|jdn!td |jdWddSWddSWddSWddS1swYdS) N) config_namehashrTF)r;rversionZuse_local_dummy_datar=)dataset_buildermock_dl_managerr)rrz>Automatic dummy data generation succeeded for all configs of ''z.zEDummy data generation done and dummy data test succeeded for config 'z''.csg|] }|dkr|qS)rr)rHZ split_nameZn_examples_per_splitrr wsz=DummyDataCommand._autogenerate_dummy_data..zCDummy data generation done but dummy data test failed since splits z have 0 examples for config 'z#Dummy data generated in directory 'zg' but kept uncompressed. Please compress this directory into a zip file to use it for dummy data tests.)$rr6r8r9rZDOWNLOADED_DATASETS_DIRZDOWNLOADED_DATASETS_PATHrrr_split_generatorsr=rErrrrrr:r;rr7Z_prepare_splitZ split_infoZ num_examplesrOSErrorr@rArrlrmrrrr<r) r%rrrZ dl_cache_dirrZ dl_managerZpath_do_datasetZsplit_generatorsZsplit_generatoreZ empty_splitsZgenerated_dummy_data_dirrrrrKsl      z)DummyDataCommand._autogenerate_dummy_datac Cstj|j|j}td|dtj|ddz||}Wn#t yC}zt d|j d|j d|j dWYd}~nd}~wwt}g}|j}|D]h} td | j|| j| j} |jd5i| } z4d } |j durzd |j jd nd } | d| |j d|d|d7} | D]\}}q| d|d7} WqNt y}z ||j WYd}~qNd}~wwd|}t|dkrt|dkrtt||kr| dtt|d|d|d7} |}nd|}| d|d|d7} | d|d7} | d|d 7} t|dkr>tt||kr>| d!|d"|d#|d$7} | d%|d&|d$7} | d'|d(|d)|d*7} n'| d+|d,|d#|d-7} | d.|d/|d$7} | d0|d(|d)|d*7} | d1|d2|d37} | d47} t | dS)6Nz$Creating dummy folder structure for z... Tr3zDataset z with config a seems to already open files in the method `_split_generators(...)`. You might consider to instead only open files in the method `_generate_examples(...)` instead. If this is not possible the dummy data has to be created with less guidance. Make sure you create the file rZz/Collecting dummy data file paths to create for zU ==============================DUMMY DATA INSTRUCTIONS============================== zconfig z of rSz(- In order to create the dummy data for z, please go into the folder 'z ' with `cd z` . za- It appears that the function `_generate_examples(...)` expects one or more files in the folder z using the function `glob.glob(...)`. In this case, please refer to the `_generate_examples(...)` method to see under which filename the dummy data files should be created. z, rrTz1- Please create a single dummy data file called 'z' from the folder 'zV'. Make sure that the dummy data file provides at least one example for the split(s) 'z' z0- Please create the following dummy data files 'z' z- For each of the splits 'zU', make sure that one or more of the dummy data files provide at least one example z- If the method `_generate_examples(...)` includes multiple `open()` statements, you might have to create other files in addition to 'zG'. In this case please refer to the `_generate_examples(...)` method z@- After the dummy data file is created, it should be zipped to 'z.zip' with the command `zip z.zip z` z- You can now delete the file 'z' with the command `rm z- To get the file 'z;' back for further changes to the dummy data, simply unzip z.zip with the command `unzip z.zip` zP- After all dummy data files are created, they should be zipped recursively to 'z.zip' with the command `zip -r z/` z!- You can now delete the folder 'z' with the command `rm -r z- To get the folder 'z'- Make sure you have created the file 'z .zip' in 'z' zT=================================================================================== r)r6r8r9rr<r@rr7rFileNotFoundErrorrrrfilenamesetdummy_file_namerr+ gen_kwargsZ_generate_examplesaddlennextiter)r%rrr<Zgenerator_splitsrZfiles_to_createZ split_namesrrar generatorZdummy_data_guidance_printZ config_stringkeyrecordZ files_stringrrrrs|   $   z/DummyDataCommand._print_dummy_data_instructionsN)rrrrrrrrrrr!rrrrrrrrs2,  ' :r))rbrir6rrZxml.etree.ElementTreeetreerrargparserpathlibrtypingrZdatasetsrZdatasets.commandsrZ!datasets.download.download_configrZ"datasets.download.download_managerrZ'datasets.download.mock_download_managerr Z datasets.loadr r Z datasets.utils.deprecation_utilsr Zdatasets.utils.loggingr rZdatasets.utils.py_utilsrrr@r[rrrrrrrs6          0