o
    Zh                  
   @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZ d dl	m	Z	 d dl
mZ d dlmZ d dlmZmZmZmZmZmZmZ d dlmZmZ d dlZd d	lmZmZmZ d d
lmZ d dl m!Z! ddl"m#Z#m$Z$ ddl%m&Z&m'Z'm(Z(m)Z)m*Z* ddl+m,Z,m-Z- ddl.m/Z/m0Z0m1Z1 ddl2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8 e9dej:Z;eG dd dZ<G dd dej=Z>G dd dej?j@ZAG dd dej?j@ZBdeCdeCfddZDdeCdeCfd d!ZEd"eCd#eeF defd$d%ZGd&e>d"eCd'eCd(eHd)eCf
d*d+ZIdS ),    N)deque)	dataclassfield)datetime)chain)Path)AnyDictListNoReturnOptionalTupleUnion)quoteunquote)_DEFAULT_CALLBACKNoOpCallbackTqdmCallback)
isfilelike)Response   )CommitOperationCopyCommitOperationDelete)DEFAULT_REVISIONENDPOINTREPO_TYPE_MODELREPO_TYPES_MAPPINGREPO_TYPES_URL_PREFIXES)
hf_hub_urlhttp_get)HfApiLastCommitInfoRepoFile)EntryNotFoundErrorHFValidationErrorRepositoryNotFoundErrorRevisionNotFoundErrorhf_raise_for_statushttp_backoffzy
    (^refs\/convert\/\w+)     # `refs/convert/parquet` revisions
    |
    (^refs\/pr\/\d+)          # PR revisions
    c                   @   sX   e Zd ZU dZeed< eed< eed< eed< edddZee ed	< d
efddZ	dS )HfFileSystemResolvedPathzUData structure containing information about a resolved Hugging Face file system path.	repo_typerepo_idrevisionpath_in_repoNF)defaultrepr_raw_revisionreturnc                 C   sz   t | jd| j }| jr| d| j d| j dS | jtkr2| dt	| j d| j dS | d| j dS )N @/)
r   getr*   r+   r0   r-   rstripr,   r   safe_revision)self	repo_path r:   U/var/www/html/lang_env/lib/python3.10/site-packages/huggingface_hub/hf_file_system.py	unresolve=   s   
"z"HfFileSystemResolvedPath.unresolve)
__name__
__module____qualname____doc__str__annotations__r   r0   r   r<   r:   r:   r:   r;   r)   1   s   
 r)   c                       s  e Zd ZdZdZdZddddee dee f fdd	Zd
ededee de	e
ee f fddZdEdedee defddZdEdee ddfddZ			dFdededee dee ddf
ddZdEdedee ddfddZ			dGdede
dee dee ddf
d d!Z	dHded#e
d$e
dee deeeeeef f  f
d%d&Z				"dIdede
d$e
dee d'e
f
d(d)Z fd*d+Z					dJdedee d,e
d#e
d$e
dee deee eeeeef f f f fd-d.ZdEd/ed0edee ddfd1d2Zdedefd3d4ZdKded$e
dee deeef fd5d6Zd7d8 Z d9d: Z!d;d< Z"dedefd=d>Z#e$dfdL fd?d@Z%e&dAdB Z'dCdD Z(  Z)S )MHfFileSystema  
    Access a remote Hugging Face Hub repository as if were a local file system.

    Args:
        token (`str`, *optional*):
            Authentication token, obtained with [`HfApi.login`] method. Will default to the stored token.

    Usage:

    ```python
    >>> from huggingface_hub import HfFileSystem

    >>> fs = HfFileSystem()

    >>> # List files
    >>> fs.glob("my-username/my-model/*.bin")
    ['my-username/my-model/pytorch_model.bin']
    >>> fs.ls("datasets/my-username/my-dataset", detail=False)
    ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json']

    >>> # Read/write files
    >>> with fs.open("my-username/my-model/pytorch_model.bin") as f:
    ...     data = f.read()
    >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f:
    ...     f.write(data)
    ```
    r2   ZhfNendpointtokenrE   rF   c                   s:   t  j|i | |pt| _|| _t||d| _i | _d S )NrD   )super__init__r   rE   rF   r    _api_repo_and_revision_exists_cache)r8   rE   rF   argsZstorage_options	__class__r:   r;   rH   g   s   
zHfFileSystem.__init__r*   r+   r,   r1   c              
   C   s   |||f| j vroz| jj|||d W nK ttfy; } zd|f| j |||f< d|f| j ||d f< W Y d }~n8d }~w ty^ } zd|f| j |||f< d| j ||d f< W Y d }~nd }~ww d| j |||f< d| j ||d f< | j |||f S )N)r,   r*   F)TN)rJ   rI   Z	repo_infor%   r$   r&   )r8   r*   r+   r,   er:   r:   r;   _repo_and_revision_existy   s    z%HfFileSystem._repo_and_revision_existpathc                 C   sh  dt t dt t dt t fdd}| |}|std|dd d t v r>d|vr1td|dd	\}}t| }nt}|	ddkrd
|v r|d
d	\}}d|v rt
|}|d urv|d | fv rvt
d|d}| }n|dd	\}}nd}|t||}| |||\}	}
|	st||
 nd }d|dd d }d|ddd  }|dd }d|dd	d  }|}|}| |||\}	}
|	st|
ttfr|}|}| |||\}	}|	st||
 n1t||
 n+|}d}d
|v r|d
d	\}}|t||}nd }| |||\}	}|	s"td|d ur)|nt}t|||||dS )Nrevision_in_pathr,   r1   c                 S   s:   |d ur| d ur| |krt d|  d| d|S | }|S )NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.)
ValueError)rQ   r,   r:   r:   r;   %_align_revision_in_path_with_revision   s   zHHfFileSystem.resolve_path.<locals>._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r4   r   r   r3   r2      )r0   )r   rA   Z_strip_protocolNotImplementedErrorsplitr   valuesr   r   countSPECIAL_REFS_REVISION_REGEXsearchgroupsublstripr   rO   _raise_file_not_foundjoin
isinstancer%   r$   r   r)   )r8   rP   r,   rS   r*   r+   rQ   matchr-   Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r:   r:   r;   resolve_path   sx   








zHfFileSystem.resolve_pathc                 C   sR   |s| j   | j  d S | | }|r'| j |d  | |}|sd S d S N)dircacheclearrJ   rd   r<   pop_parentr8   rP   r:   r:   r;   invalidate_cache   s   

zHfFileSystem.invalidate_cacherbmode
block_sizeHfFileSystemFilec                 K   sL   d|v rt d|dkrt| |f|||d|S t| |f|||d|S )Naz/Appending to remote files is not yet supported.r   )rm   r,   rn   )rU   HfFileSystemStreamFilero   )r8   rP   rm   r,   rn   kwargsr:   r:   r;   _open   s
   zHfFileSystem._openc              
   K   sR   | j ||d}| jj|j|j| j|j|j|d|dd | j	|
 d d S )Nr,   commit_messagecommit_description)r-   r+   rF   r*   r,   ru   rv   rP   )rd   rI   Zdelete_filer-   r+   rF   r*   r,   r5   rk   r<   )r8   rP   r,   rr   resolved_pathr:   r:   r;   _rm   s   	zHfFileSystem._rmF	recursivemaxdepthc              
      s    j ||d} j||||d} fdd|D }dd |D }	d| d}
|
|r+dnd	7 }
|
|d ur9d
| dnd	7 }
 jj|j|j j|	|j|d|
|dd  j	|
 d d S )Nrt   )rz   r{   r,   c                    s"   g | ]}  |s |jqS r:   )isdirrd   r-   ).0rP   r8   r:   r;   
<listcomp>     " z#HfFileSystem.rm.<locals>.<listcomp>c                 S   s   g | ]}t |d qS ))r-   )r   )r}   r-   r:   r:   r;   r         zDelete  zrecursively r2   zup to depth ru   rv   )r+   r*   rF   
operationsr,   ru   rv   rw   )rd   Zexpand_pathrI   create_commitr+   r*   rF   r,   r5   rk   r<   )r8   rP   rz   r{   r,   rr   rx   pathsZpaths_in_repor   ru   r:   r~   r;   rm   s"   
	zHfFileSystem.rmTdetailrefreshc                    s   | j  |d}|  d|i|}z| j f||d|}W n4 tyS   |js.t d | j|  f||d|} fdd|D }t|dkrQt d Y nw |rX|S dd |D S )	z!List the contents of a directory.rt   expand_info)r   r,   Nc                       g | ]
}|d   kr|qS namer:   r}   orw   r:   r;   r   %      z#HfFileSystem.ls.<locals>.<listcomp>r   c                 S   s   g | ]}|d  qS r   r:   r   r:   r:   r;   r   (  s    )rd   r<   _ls_treer#   r-   r^   ri   len)r8   rP   r   r   r,   rr   rx   outr:   rw   r;   ls  s   

zHfFileSystem.lsr   c              	      sR  j ||d}| }t|j|j|jd|jd }g }|jv r|sЈj| }	||	 g }
|rgt	dd |	D }|rg|
 }|d jvrO|
|d  nj|d  }	||	 |dd |	D  |s<g }|rtfdd|D }|rx|
s||r|rtj|
| }|d	s||ks|t|
|v r|d	n|  fd
d|D }jD ]}| d	 rj|d  qj d  |j |d||d nTjj|j|j|||j|jd}|D ]A}t|tr|d	 |j |jd|j|j|j|jd}n|d	 |j dd|j |jd}|d }j!|g | || qt"#|S )Nrt   r2   r-   r0   c                 S      g | ]
}|d  dkr|qS type	directoryr:   r}   	path_infor:   r:   r;   r   E  r   z)HfFileSystem._ls_tree.<locals>.<listcomp>r   c                 S   r   r   r:   r   r:   r:   r;   r   O  r   c                    s&   g | ]}|d  du r  |d qS )last_commitNr   )ri   r   r~   r:   r;   r   U  s   & r4   c                    s"   g | ]}|d    d s|qS )r   r4   )
startswithr   )common_pathr:   r;   r   c  r   T)rz   r   r,   r   )rz   expandr,   r*   filer   sizer   blob_idlfsr   securityr   r   r   r   r   tree_idr   )$rd   r<   r)   r*   r+   r,   r0   rf   extendr   popleftappendosrP   commonprefixendswithr   r6   ri   r   rh   r   rI   Zlist_repo_treer-   r`   r"   r   r   r   r   r   r   
setdefaultcopydeepcopy)r8   rP   rz   r   r,   r   rx   	root_pathr   Zcached_path_infosZdirs_not_in_dircacheZdirs_to_visitdir_infoZdirs_not_expandedcommon_prefixZcached_pathtreer   Zcache_path_infoparent_pathr:   )r   r8   r;   r   *  s   







zHfFileSystem._ls_treec                    s@   d| ddi|}| j|| dd }t j|fi |S )Nr   r   Fr,   rt   )r5   rd   r<   rG   globr8   rP   rr   rL   r:   r;   r     s   zHfFileSystem.globwithdirsc                    s  |rt  j|f|||||d|S | j||d}| }d|i|}z| j|fd||jd| W n tyR   | j|fd|i|d dkrN|i i ni  Y n+w |s]d	d
  D  n| j|fd|ji|}	|	d dkrt|	g  n  dd  D  t }
|s|
S  fdd|
D S )N)r{   r   r   r   r,   rt   r   T)rz   r   r,   r,   r   r   c                 S   s   g | ]
}|d  dkr|qS r   r:   r   r:   r:   r;   r     r   z%HfFileSystem.find.<locals>.<listcomp>r   c                 S   s   i | ]}|d  |qS r   r:   r   r:   r:   r;   
<dictcomp>  r   z%HfFileSystem.find.<locals>.<dictcomp>c                       i | ]}| | qS r:   r:   )r}   r   r   r:   r;   r     r   )	rG   findrd   r<   r   r,   r#   infosorted)r8   rP   r{   r   r   r   r,   rr   rx   r   namesrL   r   r;   r     s8   

 
zHfFileSystem.findpath1path2c                 K   s,  | j ||d}| j ||d}|j|jko|j|jk}|rEd| d| }| jj|j|j|j|d||ddt|j|j|jdgd n?| j	|d	|jd}	|	
 }
W d    n1 s]w   Y  d| d| }| jj|
|j|j| j|j|j|d||dd
 | j| d | j| d d S )Nrt   zCopy z to ru   rv   r2   )Zsrc_path_in_repor-   Zsrc_revision)r+   r*   r,   ru   rv   r   rl   Zpath_or_fileobjr-   r+   rF   r*   r,   ru   rv   rw   )rd   r*   r+   rI   r   r,   r5   r   r-   openreadupload_filerF   rk   r<   )r8   r   r   r,   rr   Zresolved_path1Zresolved_path2Z	same_reporu   fcontentr:   r:   r;   cp_file  sF   




zHfFileSystem.cp_filec                 K   s   | j |fi |}|d d S )Nr   dater   )r8   rP   rr   r   r:   r:   r;   modified  s   zHfFileSystem.modifiedc                    s  | j |d}| |dd}|js<ddd |r;| jj|j|j|jdd }i  d t	|j
|j|jd	d
 nd  | }|| jv r_fdd| j| D }	|	s[td  |	d  |so d u so|r̈ r̈ d d u r| jj|j|j||j|jd}
|
std  |
d }t|j|j|jd|jd }t|tr|d |j |jd|j|j|j|jd n|d |j dd|j|jd |ṡ fdddD   d usJ t S )Nrt   r   Tr   r   )r   r   r   )r*   r,   )oidtitler   )r   r   c                    r   r   r:   r   rw   r:   r;   r   	  r   z%HfFileSystem.info.<locals>.<listcomp>r   )r   r,   r*   r2   r   r4   r   r   r   c                    r   r:   r:   )r}   kr   r:   r;   r   2  r   z%HfFileSystem.info.<locals>.<dictcomp>)rd   r<   r5   r-   rI   Zlist_repo_commitsr+   r*   r,   r!   	commit_idr   Z
created_atri   rf   r^   Zget_paths_infor)   r0   r`   r"   rP   r   r   r   r   r   r   r   r   )r8   rP   r   r,   rr   rx   r   r   r   Zout1Z
paths_infor   r   r:   )r   rP   r;   r     s   


 


zHfFileSystem.infoc                 K   s2   z| j |fi i |ddi W dS    Y dS )z!Is there a file at the given pathr   FTr   r   r:   r:   r;   exists6  s
   zHfFileSystem.existsc                 C   s.   z| j |ddd dkW S  ty   Y dS w )zIs this entry directory-like?Fr   r   r   )r   OSErrorrj   r:   r:   r;   r|   ?  s
   zHfFileSystem.isdirc                 C   s&   z| j |ddd dkW S    Y dS )zIs this entry file-like?Fr   r   r   r   rj   r:   r:   r;   isfileF  s   zHfFileSystem.isfilec                 C   sB   |  |}t|j|j|j|j| jd}| |r|ddd}|S )z"Get the HTTP URL of the given path)r*   r,   rE   z	/resolve/z/tree/r   )	rd   r   r+   r-   r*   r,   rE   r|   replace)r8   rP   rx   urlr:   r:   r;   r   M  s   

zHfFileSystem.urlc              
      sb  | d}t| dh }t|ttfrt|dkr)t j||f||d|S t	|r0|}n| 
|r>tj|dd dS t|ttfrPtjtj|dd d}|du r]t|d}d}| }	| j||d	}
| j||d	d
 }|| z2tt|
j|
j|
j|
j| jd|||d| j t|tr|jndd | |	 W |r|!  dS dS |r|!  w w )z!Copy single remote file to local.r,   r   )callbackoutfileT)exist_okNFwbrt   r   r+   r,   filenamer*   rE   )r   	temp_fileZdisplayed_filenameexpected_sizeZresume_sizeheadersZ	_tqdm_bar)"r5   setkeysr`   r   r   r   rG   get_filer   r|   r   makedirsrA   r   rP   dirnamer   tellrd   r   Zset_sizer   r   r+   r,   r-   r*   rE   rI   _build_hf_headersZtqdmseekclose)r8   ZrpathZlpathr   r   rr   r,   Zunhandled_kwargsZ
close_fileZinitial_posZresolve_remote_pathr   rL   r:   r;   r   [  sR   




zHfFileSystem.get_filec                 C      t d)zA context within which files are committed together upon exit

        Requires the file class to implement `.commit()` and `.discard()`
        for the normal and exception cases.
        (Transactional commits are not supported.rU   r~   r:   r:   r;   transaction  s   	zHfFileSystem.transactionc                 C   r   )z@Begin write transaction for deferring files, non-context versionr   r   r~   r:   r:   r;   start_transaction  s   zHfFileSystem.start_transactionre   )rl   NN)FNN)TFN)FFNT)NFFFN)FNr1   N)*r=   r>   r?   r@   Zroot_markerprotocolr   rA   rH   r   bool	ExceptionrO   r)   rd   rk   intrs   ry   r   r
   r   r	   r   r   r   r   r   r   r   r   r   r   r|   r   r   r   r   propertyr   r   __classcell__r:   r:   rL   r;   rC   G   s    
L



h		 ')(J	4

rC   c                       s   e Zd Zddededee f fddZ fddZd	ed
ede	fddZ
dddZddeddfddZd fdd	ZdefddZ  ZS )ro   NfsrP   r,   c              
      st   z
|j ||d| _W n ty' } zd|ddv r"t| d| d }~ww t j|| j fi | |  d S )Nrt   wrm   r2   B.
Make sure the repository and revision exist before writing data.)rd   rx   FileNotFoundErrorr5   rG   rH   r<   )r8   r   rP   r,   rr   rN   rL   r:   r;   rH     s   zHfFileSystemFile.__init__c                       t | dsd S t  S Nrx   hasattrrG   __del__r~   rL   r:   r;   r        

zHfFileSystemFile.__del__startendr1   c                 C   sh   dd| d|d  i| j j }t| jj| jj| jj| jj| j j	d}t
d||dd}t| |jS )	Nrangezbytes=-r   r   GETi  i  i  )r   retry_on_status_codes)r   rI   r   r   rx   r+   r,   r-   r*   rE   r(   r'   r   )r8   r   r   r   r   rr:   r:   r;   _fetch_range  s   
zHfFileSystemFile._fetch_rangec                 C   s   t jddd| _d S )Nzhffs-F)prefixdelete)tempfileNamedTemporaryFiler   r~   r:   r:   r;   _initiate_upload  s   z!HfFileSystemFile._initiate_uploadFfinalc                 C   s   | j d | j  }| j| |rN| j  | jjj| jj	| j
j| j
j| jj| j
j| j
j| jd| jdd t| jj	 | jj| j
 d d S d S )Nr   ru   rv   r   rw   )bufferr   r   r   writer   r   rI   r   r   rx   r-   r+   rF   r*   r,   rr   r5   r   removerk   r<   )r8   r  blockr:   r:   r;   _upload_chunk  s(   





zHfFileSystemFile._upload_chunkr   c                    sl   | j dkr0|du s|dkr0| jdkr0| jj| jddd}| W  d   S 1 s+w   Y  t |S )a)  Read remote file.

        If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if
        `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a
        temporary file and read from there.
        rl   Nr   r   )rn   )rm   locr   r   rP   r   rG   )r8   lengthr   rL   r:   r;   r     s
   $ zHfFileSystemFile.readc                 C      | j | jS re   r   r   rP   r~   r:   r:   r;   r        zHfFileSystemFile.urlre   r   )Fr   )r=   r>   r?   rC   rA   r   rH   r   r   bytesr  r  r   r  r   r   r   r:   r:   rL   r;   ro     s     
ro   c                       s   e Zd Z				ddedededee d	ed
ef fddZddedefddZddefddZ	defddZ
 fddZdd Z  ZS )rq   rl   Nr   noner   rP   rm   r,   rn   
cache_typec           	   
      s   |dkrt d| |dkrt d| d|v r"t d| dz
|j||d| _W n" tyN } zd|d	d
v rDt| d|W Y d }~nd }~ww | j d d| _t j|| j f|||d| d | _	|  d S )Nr   z:HfFileSystemStreamFile only supports block_size=0 but got r  z?HfFileSystemStreamFile only supports cache_type='none' but got r   z;HfFileSystemStreamFile only supports reading but got mode=''rt   rm   r2   r   )r   r   rm   rn   r  )
rR   rd   rx   r   r5   r<   detailsrG   rH   response)	r8   r   rP   rm   r,   rn   r  rr   rN   rL   r:   r;   rH     s8   

zHfFileSystemStreamFile.__init__r  whencec                 C   s2   |dkr
|dkr
d S || j kr|dkrd S td)Nr   r   zCannot seek streaming HF file)r  rR   )r8   r  r  r:   r:   r;   r     s
   zHfFileSystemStreamFile.seekr   r  c                 C   sB  |dkr|fnd}| j d u s| j j r:t| jj| jj| jj| jj| j	j
d}td|| j	j ddd| _ t| j  z	| j jj| }W nR ty   | j   t| jj| jj| jj| jj| j	j
d}td|dd	| j i| j	j ddd| _ t| j  z	| j jj| }W n ty   | j    w Y nw |  jt|7  _|S )
Nr   r:   r   r  r  T)r   r  streamRangez	bytes=%d-)r  rawisclosedr   rx   r+   r,   r-   r*   r   rE   r(   rI   r   r'   r   r   r   r  r   )r8   r  Z	read_argsr   r   r:   r:   r;   r     s\   




zHfFileSystemStreamFile.readr1   c                 C   r  re   r  r~   r:   r:   r;   r   B  r  zHfFileSystemStreamFile.urlc                    r   r   r   r~   rL   r:   r;   r   E  r   zHfFileSystemStreamFile.__del__c                 C   s   t | j| j| j| j| jjffS re   )reopenr   rP   rm   	blocksizecacher   r~   r:   r:   r;   
__reduce__K  s   z!HfFileSystemStreamFile.__reduce__)rl   Nr   r  )r   r  )r=   r>   r?   rC   rA   r   r   rH   r   r   r   r   r'  r   r:   r:   rL   r;   rq     s.    /rq   r,   r1   c                 C   s   t | r| S t| S re   )rY   ra   
safe_quotert   r:   r:   r;   r7   O  s   r7   sc                 C   s   t | ddS )Nr2   )safe)r   )r)  r:   r:   r;   r(  S  s   r(  rP   rb   c                 C   sN   | }t |tr|  d}nt |tr|  d}n
t |tr"|  d}t||)Nz (repository not found)z (revision not found)z (invalid repository id))r`   r%   r&   r$   r   )rP   rb   msgr:   r:   r;   r^   W  s   




r^   r   rm   rn   r  c                 C   s   | j ||||dS )Nr  )r   )r   rP   rm   rn   r  r:   r:   r;   r$  b  s   r$  )Jr   r   rer	  collectionsr   dataclassesr   r   r   	itertoolsr   pathlibr   typingr   r	   r
   r   r   r   r   urllib.parser   r   ZfsspecZfsspec.callbacksr   r   r   Zfsspec.utilsr   requestsr   Z_commit_apir   r   	constantsr   r   r   r   r   Zfile_downloadr   r   Zhf_apir    r!   r"   utilsr#   r$   r%   r&   r'   r(   compileVERBOSErY   r)   ZAbstractFileSystemrC   specZAbstractBufferedFilero   rq   rA   r7   r(  r   r^   r   r$  r:   r:   r:   r;   <module>   sJ    $ 
    ^Kc"