o
    T©Zhí  ã                   @   s`   d dl Z d dlmZ d dlmZ ddlmZ dZeeƒZ	G dd„ deƒZ
dd
d„Zddd„ZdS )é    N)ÚFileSystems)ÚPipelineé   )Ú
get_loggeri    c                   @   s   e Zd ZdZdd„ ZdS )ÚBeamPipelinez<Wrapper over `apache_beam.pipeline.Pipeline` for conveniencec                 C   s   | j  ¡  d¡}|dv S )NÚrunner)NZDirectRunnerZPortableRunner)Ú_optionsZget_all_optionsÚget)Úselfr   © r   úP/var/www/html/lang_env/lib/python3.10/site-packages/datasets/utils/beam_utils.pyÚis_local   s   zBeamPipeline.is_localN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r      s    r   Fc              	   C   sÌ   t }| |¡r|rt d|› d¡ nt d|› d¡ dS | |¡9}t| dƒ}| t¡}|r=| |¡ | t¡}|s1W d  ƒ n1 sGw   Y  W d  ƒ dS W d  ƒ dS 1 s_w   Y  dS )zJUse the Beam Filesystems to upload to a remote directory on gcs/s3/hdfs...zRemote path already exist: ú&. Overwriting it as force_upload=True.ú$. Skipping it as force_upload=False.NÚrb)	r   ÚexistsÚloggerÚinfoÚcreateÚopenÚreadÚ
CHUNK_SIZEÚwrite)Úlocal_file_pathÚremote_file_pathZforce_uploadÚfsÚremote_fileÚ
local_fileÚchunkr   r   r   Úupload_local_to_remote   s$   



þ€þÿ"ÿr#   c              	   C   sÎ   t }tj |¡r|rt d| › d¡ nt d| › d¡ dS | | ¡9}t|dƒ}| t¡}|r>| 	|¡ | t¡}|s2W d  ƒ n1 sHw   Y  W d  ƒ dS W d  ƒ dS 1 s`w   Y  dS )zNUse the Beam Filesystems to download from a remote directory on gcs/s3/hdfs...zLocal path already exist: r   r   NÚwb)
r   ÚosÚpathr   r   r   r   r   r   r   )r   r   Zforce_downloadr   r    r!   r"   r   r   r   Údownload_remote_to_local&   s$   


þ€þÿ"ÿr'   )F)r%   Zapache_beam.io.filesystemsr   Zapache_beam.pipeliner   Úloggingr   r   r   r   r   r#   r'   r   r   r   r   Ú<module>   s    
