o
    TZh                     @   sv   d dl Z d dlmZ d dlmZ d dlZd dlZd dlm	Z	 ej
jeZeG dd dejZG dd dejZdS )	    N)	dataclass)Optional)
table_castc                   @   s$   e Zd ZU dZdZeej ed< dS )ArrowConfigzBuilderConfig for Arrow.Nfeatures)	__name__
__module____qualname____doc__r   r   datasetsFeatures__annotations__ r   r   \/var/www/html/lang_env/lib/python3.10/site-packages/datasets/packaged_modules/arrow/arrow.pyr      s   
 r   c                   @   s>   e Zd ZeZdd Zdd ZdejdejfddZ	d	d
 Z
dS )Arrowc                 C   s   t j| jjdS )N)r   )r   ZDatasetInfoconfigr   )selfr   r   r   _info   s   zArrow._infoc              
      s,  | j jstd| j j  | j j}t|tttfr;|}t|tr&|g} fdd|D }tj	tj
jd|idgS g }| D ]R\}}t|trM|g} fdd|D }| jjdu rtj|D ]$}t|d}tjtj|j| j_W d   n1 sw   Y   |tj	|d|id qA|S )	z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=c                       g | ]}  |qS r   Z
iter_files.0file
dl_managerr   r   
<listcomp>%       z+Arrow._split_generators.<locals>.<listcomp>files)nameZ
gen_kwargsc                    r   r   r   r   r   r   r   r   ,   r   Nrb)r   
data_files
ValueErrorZdownload_and_extract
isinstancestrlisttupler   ZSplitGeneratorZSplitZTRAINitemsinfor   	itertoolschainfrom_iterableopenr   Zfrom_arrow_schemapaipcopen_streamZschemaappend)r   r   r    r   ZsplitsZ
split_namer   fr   r   r   _split_generators   s,   

zArrow._split_generatorspa_tablereturnc                 C   s    | j jd urt|| j jj}|S )N)r'   r   r   Zarrow_schema)r   r2   r   r   r   _cast_table6   s   zArrow._cast_tablec           	      c   s    t tj|D ]Z\}}t|dI}z#t tj|D ]\}}tj	|g}| d| | 
|fV  qW n tyS } ztd| dt| d|   d }~ww W d    n1 s^w   Y  q	d S )Nr   _zFailed to read file 'z' with error z: )	enumerater(   r)   r*   r+   r,   r-   r.   TableZfrom_batchesr4   r!   loggererrortype)	r   r   Zfile_idxr   r0   Z	batch_idxZrecord_batchr2   er   r   r   _generate_tables=   s"    zArrow._generate_tablesN)r   r   r	   r   ZBUILDER_CONFIG_CLASSr   r1   r,   r7   r4   r<   r   r   r   r   r      s    r   )r(   dataclassesr   typingr   Zpyarrowr,   r   Zdatasets.tabler   utilsloggingZ
get_loggerr   r8   ZBuilderConfigr   ZArrowBasedBuilderr   r   r   r   r   <module>   s    