o
    TZh                     @   s   d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ er:d dlZd dlZG d
d deZG dd dZdS )    N)TYPE_CHECKINGOptionalUnion   )DatasetFeaturesconfig)query_table)Sql)tqdm   )AbstractDatasetInputStreamc                       sZ   e Zd Z			ddeedf deedddf d	ee d
edef
 fddZdd Z	  Z
S )SqlDatasetReaderNFsqlzsqlalchemy.sql.Selectableconsqlalchemy.engine.Connectionsqlalchemy.engine.Enginesqlite3.Connectionfeatures	cache_dirkeep_in_memoryc                    s8   t  jd|||d| td||||d|| _d S )N)r   r   r   )r   r   r   r    )super__init__r
   builder)selfr   r   r   r   r   kwargs	__class__r   F/var/www/html/lang_env/lib/python3.10/site-packages/datasets/io/sql.pyr      s   	zSqlDatasetReader.__init__c                 C   s<   d }d }d }d }| j j||||d | j jd|| jd}|S )N)download_configdownload_modeverification_mode	base_pathtrain)splitr"   Z	in_memory)r   Zdownload_and_prepareZ
as_datasetr   )r   r    r!   r"   r#   datasetr   r   r   read$   s   	zSqlDatasetReader.read)NNF)__name__
__module____qualname__r   strr   r   boolr   r'   __classcell__r   r   r   r   r      s     
r   c                   @   sh   e Zd Z		ddededeedddf dee d	ee f
d
dZdefddZ	dd Z
defddZdS )SqlDatasetWriterNr&   namer   r   r   r   
batch_sizenum_procc                 K   sR   |d ur|dkrt d| d|| _|| _|| _|r|ntj| _|| _|| _d S )Nr   z	num_proc z must be an integer > 0.)	
ValueErrorr&   r/   r   r   ZDEFAULT_MAX_BATCH_SIZEr0   r1   to_sql_kwargs)r   r&   r/   r   r0   r1   r3   r   r   r   r   :   s   	
zSqlDatasetWriter.__init__returnc                 C   sD   | j dd }| j dd }| j dd}| jdd|i| j }|S )Nr   r   indexFr   )r3   pop_write)r   _r5   writtenr   r   r   writeM   s
   zSqlDatasetWriter.writec                 C   sx   |\}}}|dkri |ddin|}t | jjt||| j | jjd}| }|j| j| j	fd|i|}|p;t
|S )Nr   Z	if_existsappend)tablekeyindicesr5   )r	   r&   dataslicer0   Z_indicesZ	to_pandasZto_sqlr/   r   len)r   argsoffsetr5   r3   batchdfnum_rowsr   r   r   
_batch_sqlU   s   
zSqlDatasetWriter._batch_sqlc              
      s   d}| j du s| j dkr*ttdt| j| jdddD ]}|| | f7 }q|S t| j| j}}t| j 5}t|	| j fddtd||D || rW|| d n|| ddd	D ]}||7 }q_W d   |S 1 sqw   Y  |S )
zWrites the pyarrow table as SQL to a database.

        Caller is responsible for opening and closing the SQL connection.
        r   Nr   bazCreating SQL from Arrow format)unitdescc                    s   g | ]}| fqS r   r   ).0rC   r5   r3   r   r   
<listcomp>u   s    z+SqlDatasetWriter._write.<locals>.<listcomp>)totalrI   rJ   )
r1   hf_tqdmrangerA   r&   r0   rG   multiprocessingZPoolimap)r   r5   r3   r9   rC   rF   r0   poolr   rL   r   r7   a   s4   


	
zSqlDatasetWriter._write)NN)r(   r)   r*   r   r+   r   r   intr   r:   rG   r7   r   r   r   r   r.   9   s"    
r.   )rQ   typingr   r   r    r   r   r   Z
formattingr	   Zpackaged_modules.sql.sqlr
   utilsr   rO   abcr   sqlite3Z
sqlalchemyr   r.   r   r   r   r   <module>   s    (