o
    +if                     @   s   d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ er:d dlZd dlZG d
d deZG dd dZdS )    N)TYPE_CHECKINGOptionalUnion   )DatasetFeaturesconfig)query_table)Sql)tqdm   )AbstractDatasetInputStreamc                       sZ   e Zd Z			ddeedf deedddf d	ee d
edef
 fddZdd Z	  Z
S )SqlDatasetReaderNFsqlzsqlalchemy.sql.Selectableconsqlalchemy.engine.Connectionsqlalchemy.engine.Enginesqlite3.Connectionfeatures	cache_dirkeep_in_memoryc                    s8   t  jd|||d| td||||d|| _d S )N)r   r   r   )r   r   r   r    )super__init__r
   builder)selfr   r   r   r   r   kwargs	__class__r   H/var/www/html/corbot_env/lib/python3.10/site-packages/datasets/io/sql.pyr      s   	zSqlDatasetReader.__init__c                 C   s<   d }d }d }d }| j j||||d | j jd|| jd}|S )N)download_configdownload_modeverification_mode	base_pathtrain)splitr"   	in_memory)r   download_and_prepare
as_datasetr   )r   r    r!   r"   r#   datasetr   r   r   read$   s   zSqlDatasetReader.read)NNF)__name__
__module____qualname__r   strr   r   boolr   r*   __classcell__r   r   r   r   r      s     
r   c                   @   sh   e Zd Z		ddededeedddf dee d	ee f
d
dZdefddZ	dd Z
defddZdS )SqlDatasetWriterNr)   namer   r   r   r   
batch_sizenum_procc                 K   sR   |d ur|dkrt d| d|| _|| _|| _|r|ntj| _|| _|| _d S )Nr   z	num_proc z must be an integer > 0.)	
ValueErrorr)   r2   r   r   DEFAULT_MAX_BATCH_SIZEr3   r4   to_sql_kwargs)r   r)   r2   r   r3   r4   r7   r   r   r   r   9   s   	
zSqlDatasetWriter.__init__returnc                 C   sD   | j dd }| j dd }| j dd}| jdd|i| j }|S )Nr   r   indexFr   )r7   pop_write)r   _r9   writtenr   r   r   writeL   s
   zSqlDatasetWriter.writec                 C   sx   |\}}}|dkri |ddin|}t | jjt||| j | jjd}| }|j| j| j	fd|i|}|p;t
|S )Nr   	if_existsappend)tablekeyindicesr9   )r	   r)   dataslicer3   _indices	to_pandasto_sqlr2   r   len)r   argsoffsetr9   r7   batchdfnum_rowsr   r   r   
_batch_sqlT   s   
zSqlDatasetWriter._batch_sqlc              
      s   d}| j du s| j dkr*ttdt| j| jdddD ]}|| | f7 }q|S t| j| j}}t| j 5}t|	| j fddtd||D || rW|| d n|| ddd	D ]}||7 }q_W d   |S 1 sqw   Y  |S )
zWrites the pyarrow table as SQL to a database.

        Caller is responsible for opening and closing the SQL connection.
        r   Nr   bazCreating SQL from Arrow format)unitdescc                    s   g | ]}| fqS r   r   ).0rK   r9   r7   r   r   
<listcomp>t   s    z+SqlDatasetWriter._write.<locals>.<listcomp>)totalrQ   rR   )
r4   hf_tqdmrangerI   r)   r3   rO   multiprocessingPoolimap)r   r9   r7   r=   rK   rN   r3   poolr   rT   r   r;   `   s4   


	
zSqlDatasetWriter._write)NN)r+   r,   r-   r   r.   r   r   intr   r>   rO   r;   r   r   r   r   r1   8   s"    
r1   )rY   typingr   r   r    r   r   r   
formattingr	   packaged_modules.sql.sqlr
   utilsr   rW   abcr   sqlite3
sqlalchemyr   r1   r   r   r   r   <module>   s    '