o
    TZh                     @   s   d dl Z d dlZd dlmZmZmZ d dlZddlmZm	Z	m
Z
mZ ddlmZ ddlmZ ddlmZ ddlmZmZ d	d
lmZ G dd deZG dd dZdS )    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Json)tqdm)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                       sj   e Zd Z							ddee dee dee dede	de	d	ee d
ee
 f fddZdd Z  ZS )JsonDatasetReaderNFpath_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingfieldnum_procc	           
   	      s^   t  j|f||||||d|	 || _t|tr|n| j|i}td||||d|	| _d S )N)r   r   r   r   r   r   )r   Z
data_filesr   r    )super__init__r   
isinstancedictr   r   builder)
selfr   r   r   r   r   r   r   r   kwargs	__class__r   G/var/www/html/lang_env/lib/python3.10/site-packages/datasets/io/json.pyr      s,   
zJsonDatasetReader.__init__c                 C   s\   | j r| jj| jd}|S d }d }d }d }| jj||||| jd | jj| j|| jd}|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r'   Z	in_memory)r   r   Zas_streaming_datasetr   Zdownload_and_preparer   Z
as_datasetr   )r    datasetr%   r&   r'   r(   r   r   r$   read0   s$   
zJsonDatasetReader.read)NNNFFNN)__name__
__module____qualname__r   r   r   r   r   strboolintr   r*   __classcell__r   r   r"   r$   r      s4    	 r   c                
   @   sd   e Zd Z		ddedeeef dee dee fddZ	defd	d
Z
dd ZdedefddZdS )JsonDatasetWriterNr)   path_or_buf
batch_sizer   c                 K   sR   |d ur|dkrt d| d|| _|| _|r|ntj| _|| _d| _|| _d S )Nr   z	num_proc z must be an integer > 0.zutf-8)	
ValueErrorr)   r3   r	   ZDEFAULT_MAX_BATCH_SIZEr4   r   encodingto_json_kwargs)r    r)   r3   r4   r   r7   r   r   r$   r   J   s   
zJsonDatasetWriter.__init__returnc                 C   s.  | j dd }| j dd}| j d|dkrdnd}d| j vr)|dv r)d| j d< t| jtttjfr5d	nd }| j d
|}|dvrJtd| dt| jtttjfr}t	j
| jd|d}| jd|||d| j }W d    |S 1 svw   Y  |S |rtd| d| jd| j||d| j }|S )Nr3   orientrecordslinesTFindex)r   tableinfercompression)Nr>   gzipbz2xzz&`datasets` currently does not support z compressionwb)r?   )file_objr9   r;   zUThe compression parameter is not supported when writing to a buffer, but compression=z1 was passed. Please provide a local path instead.r   )r7   popr   r3   r.   bytesosr   NotImplementedErrorfsspecopen_write)r    _r9   r;   Zdefault_compressionr?   bufferwrittenr   r   r$   write\   s,   

		
zJsonDatasetWriter.writec                 C   sh   |\}}}}t | jjt||| j | jjd}| jdd ||d|}|ds.|d7 }|	| j
S )N)r=   keyindices)r3   r9   r;   
r   )r
   r)   dataslicer4   Z_indicesZ	to_pandasto_jsonendswithencoder6   )r    argsoffsetr9   r;   r7   batchjson_strr   r   r$   _batch_jsonv   s   
zJsonDatasetWriter._batch_jsonrD   c              
      s  d}| j du s| j dkr0ttdt| j| jdddD ]}| | f}|||7 }q|S t| j| j}}	t	| j 9}
t|

| j fddtd||	D ||	 r^||	 d n||	 ddd	D ]	}|||7 }qfW d   |S 1 s{w   Y  |S )
zWrites the pyarrow table as JSON lines to a binary file handle.

        Caller is responsible for opening and closing the handle.
        r   Nr   bazCreating json from Arrow format)unitdescc                    s   g | ]}| fqS r   r   ).0rY   r;   r9   r7   r   r$   
<listcomp>   s    z,JsonDatasetWriter._write.<locals>.<listcomp>)totalr^   r_   )r   hf_tqdmrangelenr)   r4   r\   rO   multiprocessingZPoolimap)r    rD   r9   r;   r7   rN   rY   r[   Znum_rowsr4   poolr   ra   r$   rK      s6   

	
zJsonDatasetWriter._write)NN)r+   r,   r-   r   r   r   r   r   r0   r   rO   r\   rK   r   r   r   r$   r2   I   s&    

r2   )rg   rG   typingr   r   r   rI    r   r   r   r	   Z
formattingr
   Zpackaged_modules.json.jsonr   utilsr   rd   Zutils.typingr   r   abcr   r   r2   r   r   r   r$   <module>   s    :