o
    +if                     @   s\   d dl mZ d dlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ G d	d
 d
eZdS )    )OptionalN   )Features
NamedSplit)DownloadMode)Spark   )AbstractDatasetReaderc                       sl   e Zd ZdZ								ddejjdee dee	 d	e
d
ede
dede
def fddZdd Z  ZS )SparkDatasetReaderzA dataset reader that reads from a Spark DataFrame.

    When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
    provided. Streaming is not currently supported.
    NTFarrowdfsplitfeatures	streaming	cache_dirkeep_in_memoryworking_dirload_from_cache_filefile_formatc
                    sH   t  jd|||||d|
 || _|	| _td||||d|
| _d S )N)r   r   r   r   r   )r   r   r   r    )super__init___load_from_cache_file_file_formatr   builder)selfr   r   r   r   r   r   r   r   r   kwargs	__class__r   J/var/www/html/corbot_env/lib/python3.10/site-packages/datasets/io/spark.pyr      s&   zSparkDatasetReader.__init__c                 C   sH   | j r| jj| jdS | jrd ntj}| jj|| jd | jj	| jdS )N)r   )download_moder   )
r   r   as_streaming_datasetr   r   r   FORCE_REDOWNLOADdownload_and_preparer   
as_dataset)r   r    r   r   r   read1   s   zSparkDatasetReader.read)NNTNFNTr   )__name__
__module____qualname____doc__pysparksql	DataFramer   r   r   boolstrr   r%   __classcell__r   r   r   r   r
      s<    		
r
   )typingr   r*    r   r   downloadr   packaged_modules.spark.sparkr   abcr	   r
   r   r   r   r   <module>   s    