o +if@szddlZddlZddlZddlmZmZddlmZddlm Z dZ dZ gdZ gdZ d efd d ZGd d d eZdS)N)ArgumentParser Namespace)BaseDatasetsCLICommand) get_loggerz><<<<<<< This should probably be modified because it mentions: z======= >>>>>>> )TextEncoderConfigByteTextEncoderSubwordTextEncoderencoder_configmaybe_build_from_corpus manual_dir) )z tfds\.coredatasets)ztf\.io\.gfile\.GFileopen)z tf\.([\w\d]+)zdatasets.Value('\1'))ztfds\.features\.Text\(\)zdatasets.Value('string'))ztfds\.features\.Text\(zdatasets.Value('string'),)z+features\s*=\s*tfds.features.FeaturesDict\(zfeatures=datasets.Features()ztfds\.features\.FeaturesDict\(zdict()zThe TensorFlow Datasets AuthorszDThe TensorFlow Datasets Authors and the HuggingFace Datasets Authors)ztfds\.z datasets.)zdl_manager\.manual_dirzself.config.data_dir)zself\.builder_configz self.configargscCst|j|jS)zz Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. Returns: ConvertCommand )ConvertCommand tfds_pathdatasets_directory)rrR/var/www/html/corbot_env/lib/python3.10/site-packages/datasets/commands/convert.pyconvert_command_factory*src@s8eZdZedefddZdedefddZdd Zd S) rparsercCsB|jddd}|jdtddd|jdtdd d|jtd d S) z Register this command to argparse so it's available for the datasets-cli Args: parser: Root parser to register command-specific arguments convertzHConvert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.)helpz --tfds_pathTzQPath to a TensorFlow Datasets folder to convert or a single tfds file to convert.)typerequiredrz--datasets_directoryz(Path to the HuggingFace Datasets folder.)funcN) add_parser add_argumentstr set_defaultsr)r train_parserrrrregister_subcommand4sz"ConvertCommand.register_subcommandrrcGstd|_||_||_dS)Nzdatasets-cli/converting)r_logger _tfds_path_datasets_directory)selfrrrrrr__init__Ks  zConvertCommand.__init__c sNtj|jrtj|j}ntj|jrtj|j}ntdtj|j}|j d|d|g}g}i}tj|jrHt |}ntj |jg}|D]}|j d|tj ||}tj ||} tj|r~d|vs~d|vs~d|vr|j dqRt|d d  } | } Wdn1swYg} d } d }g}| D]}|d vrqd vrqdvrqdvrdnWdvrdqdvrdnIdvr܈ddn>tfddtDr d}ttfddt}| tt|d| | tqtD] \}}t||q dvrz%ConvertCommand.run..Tcs|vSr*r)er-rrsz$ConvertCommand.run.. tensorflow_datasetsz/from\stensorflow_datasets.*import\s([^\.\r\n]+)css|]}|VqdSr*)stripr+imprrrr/r0,zfrom . import ztf.ztfds.zError converting GeneratorBasedBuilderBeamBasedBuilderwmt)exist_okzAdding directory csi|]}|qSrrr6) output_dirrr sz&ConvertCommand.run..wz Converted in zMoving z#Cannot find destination folder for z. Please copy manually.z!You need to manually update file z4 to remove configurations using 'TextEncoderConfig'.))ospathisdirr"abspathisfiledirname ValueErrorr#r!infolistdirbasenamejoinr readlinesreplaceany TO_HIGHLIGHTlistfilterappendHIGHLIGHT_MESSAGE_PRErHIGHLIGHT_MESSAGE_POST TO_CONVERTresubmatchextendgroupsplitr5makedirsupdate writelinesshutilcopyKeyErrorerrorwarning)r$ abs_tfds_pathabs_datasets_path utils_fileswith_manual_updateimports_to_builder_map file_namesf_name input_file output_fileflines out_lines is_builderneeds_manual_update tfds_importsline to_removepattern replacementrXdir_name utils_file dest_folder file_pathr)r.r>rrunQs  $               zConvertCommand.runN) __name__ __module__ __qualname__ staticmethodrr rr%r{rrrrr3s  r)rArVr_argparserrdatasets.commandsrdatasets.utils.loggingrrSrTrOrUrrrrrrs