o
    TZh                     @   sp   d dl Z d dlmZ d dlmZ d dlZd dlZd dl	Z	d dl
mZ eG dd de	jZG dd de	jZdS )	    N)	dataclass)Optional)
table_castc                   @   s$   e Zd ZU dZdZeej ed< dS )PandasConfigzBuilderConfig for Pandas.Nfeatures)	__name__
__module____qualname____doc__r   r   datasetsZFeatures__annotations__ r   r   ^/var/www/html/lang_env/lib/python3.10/site-packages/datasets/packaged_modules/pandas/pandas.pyr      s   
 r   c                   @   s>   e Zd ZeZdd Zdd ZdejdejfddZ	d	d
 Z
dS )Pandasc                 C   s   t j| jjdS )N)r   )r   ZDatasetInfoconfigr   )selfr   r   r   _info   s   zPandas._infoc                    s   | j jstd| j j  | j j}t|tttfr;|}t|tr&|g} fdd|D }tj	tj
jd|idgS g }| D ]!\}}t|trM|g} fdd|D }|tj	|d|id qA|S )z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=c                       g | ]}  |qS r   Z
iter_files.0file
dl_managerr   r   
<listcomp>#       z,Pandas._split_generators.<locals>.<listcomp>files)nameZ
gen_kwargsc                    r   r   r   r   r   r   r   r   *   r   )r   
data_files
ValueErrorZdownload_and_extract
isinstancestrlisttupler   ZSplitGeneratorZSplitZTRAINitemsappend)r   r   r   r   ZsplitsZ
split_namer   r   r   _split_generators   s    

zPandas._split_generatorspa_tablereturnc                 C   s    | j jd urt|| j jj}|S )N)r   r   r   Zarrow_schema)r   r'   r   r   r   _cast_table.   s   zPandas._cast_tablec              	   c   sl    t tj|D ]*\}}t|d}tjt	|}|| 
|fV  W d    n1 s.w   Y  q	d S )Nrb)	enumerate	itertoolschainfrom_iterableopenpaTableZfrom_pandaspdZread_pickler)   )r   r   ir   fr'   r   r   r   _generate_tables5   s   zPandas._generate_tablesN)r   r   r	   r   ZBUILDER_CONFIG_CLASSr   r&   r0   r1   r)   r5   r   r   r   r   r      s    r   )r,   dataclassesr   typingr   Zpandasr2   Zpyarrowr0   r   Zdatasets.tabler   ZBuilderConfigr   ZArrowBasedBuilderr   r   r   r   r   <module>   s    