o
    NZhmp                  	   @  s  U d dl mZ d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
mZ d dlmZmZmZmZmZmZmZmZ erPd dlZd dlmZ edZnedZedZedZeeegef ZG d	d
 d
ZG dd deZ G dd deZ!G dd deZ"G dd deZ#G dd deZ$G dd deZ%G dd deZ&G dd deeef Z'G dd deZ(deiZ)de*d< d(d)d&d'Z+ee e$e!e#e"e%e&e(f	D ]Z,e+e, qdS )*    )annotationsN)FutureThreadPoolExecutor)TYPE_CHECKINGAnyCallableClassVarGeneric
NamedTupleOrderedDictTypeVar)	ParamSpecPTZfsspecc                   @  s2   e Zd ZU dZdZded< dddZdddZdS )	BaseCacheag  Pass-though cache: doesn't keep anything, calls every time

    Acts as base class for other cachers

    Parameters
    ----------
    blocksize: int
        How far to read ahead in numbers of bytes
    fetcher: func
        Function of the form f(start, end) which gets bytes from remote as
        specified
    size: int
        How big this file is
    noneClassVar[str]name	blocksizeintfetcherFetchersizereturnNonec                 C  s   || _ || _|| _d S N)r   r   r   selfr   r   r    r   E/var/www/html/lang_env/lib/python3.10/site-packages/fsspec/caching.py__init__9   s   
zBaseCache.__init__start
int | Nonestopbytesc                 C  s<   |d u rd}|d u r| j }|| j ks||krdS | ||S Nr       )r   r   r   r!   r#   r   r   r   _fetch>   s   zBaseCache._fetchNr   r   r   r   r   r   r   r   r!   r"   r#   r"   r   r$   )__name__
__module____qualname____doc__r   __annotations__r    r(   r   r   r   r   r   '   s
   
 
r   c                      sT   e Zd ZdZdZ		d d! fddZd"ddZd#ddZd$ddZd%ddZ	  Z
S )&	MMapCachezmemory-mapped sparse file cache

    Opens temporary file, which is filled blocks-wise when data is requested.
    Ensure there is enough disc space in the temporary location.

    This cache method might only work on posix
    mmapNr   r   r   r   r   location
str | Noneblocksset[int] | Noner   r   c                   s8   t  ||| |d u rt n|| _|| _|  | _d S r   )superr    setr4   r2   	_makefilecache)r   r   r   r   r2   r4   	__class__r   r   r    S   s   zMMapCache.__init__mmap.mmap | bytearrayc                 C  s   dd l }dd l}| jdkrt S | jd u stj| jsB| jd u r*| }t	 | _
nt| jd}|| jd  |d |  nt| jd}| | | jS )Nr   zwb+      1zr+b)r1   tempfiler   	bytearrayr2   ospathexistsTemporaryFiler7   r4   openseekwriteflushfileno)r   r1   r?   fdr   r   r   r8   `   s   




zMMapCache._makefiler!   r"   endr$   c           	        s   t d| d|  |d u rd}|d u r j}| jks!||kr#dS | j }| j } fddt||d D }|rq|d}| j }t| j  j}t d| d	| d|   || j||<  j	
| |s= j|| S )
NzMMap cache fetching -r   r&   c                   s   g | ]	}| j vr|qS r   )r4   ).0ir   r   r   
<listcomp>   s    z$MMapCache._fetch.<locals>.<listcomp>r=   zMMap get block #z ()loggerdebugr   r   rangepopminr   r9   r4   add)	r   r!   rK   Zstart_blockZ	end_blockZneedrN   Zsstartsendr   rO   r   r(   v   s&   




zMMapCache._fetchdict[str, Any]c                 C  s   | j  }|d= |S )Nr9   )__dict__copyr   stater   r   r   __getstate__   s   
zMMapCache.__getstate__r\   c                 C  s   | j | |  | _d S r   )rY   updater8   r9   r[   r   r   r   __setstate__   s   zMMapCache.__setstate__)NN)r   r   r   r   r   r   r2   r3   r4   r5   r   r   )r   r<   r!   r"   rK   r"   r   r$   r   rX   r\   rX   r   r   )r+   r,   r-   r.   r   r    r8   r(   r]   r_   __classcell__r   r   r:   r   r0   H   s    


r0   c                      0   e Zd ZdZdZd fd
dZdddZ  ZS )ReadAheadCachea!  Cache which reads only when we get beyond a block of data

    This is a much simpler version of BytesCache, and does not attempt to
    fill holes in the cache or keep fragments alive. It is best suited to
    many small reads in a sequential order (e.g., reading lines from a file).
    Z	readaheadr   r   r   r   r   r   r   c                   s&   t  ||| d| _d| _d| _d S )Nr&   r   )r6   r    r9   r!   rK   r   r:   r   r   r       s   
zReadAheadCache.__init__r!   r"   rK   r$   c                 C  s  |d u rd}|d u s|| j kr| j }|| j ks||krdS || }|| jkr8|| jkr8| j|| j || j  S | j|  krD| jk rZn n| j|| j d  }|t|8 }| j}nd}t| j || j }| ||| _|| _| jt| j | _|| jd |  S r%   )r   r!   rK   r9   lenrU   r   r   )r   r!   rK   lpartr   r   r   r(      s&   zReadAheadCache._fetchr)   r`   r+   r,   r-   r.   r   r    r(   rc   r   r   r:   r   re      s
    re   c                      rd   )FirstChunkCachezCaches the first block of a file only

    This may be useful for file types where the metadata is stored in the header,
    but is randomly accessed.
    firstr   r   r   r   r   r   r   c                   s   t  ||| d | _d S r   )r6   r    r9   r   r:   r   r   r       s   
zFirstChunkCache.__init__r!   r"   rK   r$   c                 C  s   |pd}|p| j }|| jk rK| jd u r4|| jkr,| d|}|d | j | _||d  S | d| j| _| j|| }|| jkrI|| | j|7 }|S | ||S Nr   )r   r   r9   r   )r   r!   rK   datarh   r   r   r   r(      s   




zFirstChunkCache._fetchr)   r`   ri   r   r   r:   r   rj      s
    rj   c                      sr   e Zd ZdZdZ	d&d' fddZd(ddZdd Zd)ddZd*ddZ	d+ddZ
d, fd d!Zd-d$d%Z  ZS ).
BlockCachea  
    Cache holding memory as a set of blocks.

    Requests are only ever made ``blocksize`` at a time, and are
    stored in an LRU cache. The least recently accessed block is
    discarded when more than ``maxblocks`` are stored.

    Parameters
    ----------
    blocksize : int
        The number of bytes to store in each block.
        Requests are only ever made for ``blocksize``, so this
        should balance the overhead of making a request against
        the granularity of the blocks.
    fetcher : Callable
    size : int
        The total size of the file being cached.
    maxblocks : int
        The maximum number of blocks to cache for. The maximum memory
        use for this cache is then ``blocksize * maxblocks``.
    Z
blockcache    r   r   r   r   r   	maxblocksr   r   c                   s<   t  ||| t|| | _|| _t|| j| _	d S r   )
r6   r    mathceilnblocksrp   	functools	lru_cache_fetch_block_fetch_block_cachedr   r   r   r   rp   r:   r   r   r       s   zBlockCache.__init__strc                 C     d| j  d| j d| j dS )Nz<BlockCache blocksize=, size=
, nblocks=>r   r   rs   rO   r   r   r   __repr__     
zBlockCache.__repr__c                 C  
   | j  S z
        The statistics on the block cache.

        Returns
        -------
        NamedTuple
            Returned directly from the LRU Cache used internally.
        rw   
cache_inforO   r   r   r   r   	     
	zBlockCache.cache_inforX   c                 C  s   | j }|d= |S )Nrw   rY   r[   r   r   r   r]     s   zBlockCache.__getstate__r\   c                 C  s&   | j | t|d | j| _d S )Nrp   )rY   r^   rt   ru   rv   rw   r[   r   r   r   r_     s   
zBlockCache.__setstate__r!   r"   rK   r$   c                 C  st   |d u rd}|d u r| j }|| j ks||krdS || j }|| j }t||d D ]}| | q)| j||||dS )Nr   r&   r=   start_block_numberend_block_number)r   r   rS   rw   _read_cache)r   r!   rK   r   r   block_numberr   r   r   r(     s    

zBlockCache._fetchr   c                   sT   || j krtd| d| j  d|| j }|| j }td| t ||}|S )=
        Fetch the block of data for `block_number`.
        'block_number=(' is greater than the number of blocks ()zBlockCache fetching block %drs   
ValueErrorr   rQ   infor6   r(   )r   r   r!   rK   block_contentsr:   r   r   rv   6  s   


zBlockCache._fetch_blockr   r   c           
      C     || j  }|| j  }||kr| |}||| S g }|| ||d  t|d |D ]
}	|| |	 q.|| |d|  d|S z
        Read from our block cache.

        Parameters
        ----------
        start, end : int
            The start and end byte positions.
        start_block_number, end_block_number : int
            The start and end block numbers.
        Nr=   r&   r   rw   appendrS   join
r   r!   rK   r   r   	start_posend_posblockoutr   r   r   r   r   F     



zBlockCache._read_cachero   
r   r   r   r   r   r   rp   r   r   r   r   ry   ra   rb   r`   )r   r   r   r$   
r!   r   rK   r   r   r   r   r   r   r$   )r+   r,   r-   r.   r   r    r   r   r]   r_   r(   rv   r   rc   r   r   r:   r   rn      s    



rn   c                      sH   e Zd ZU dZdZded< 	dd fddZdddZdddZ  Z	S )
BytesCacheaK  Cache which holds data in a in-memory bytes object

    Implements read-ahead by the block size, for semi-random reads progressing
    through the file.

    Parameters
    ----------
    trim: bool
        As we read more data, whether to discard the start of the buffer when
        we are more than a blocksize ahead of it.
    r$   r   r   Tr   r   r   r   r   trimboolr   r   c                   s,   t  ||| d| _d | _d | _|| _d S )Nr&   )r6   r    r9   r!   rK   r   )r   r   r   r   r   r:   r   r   r    z  s
   
zBytesCache.__init__r!   r"   rK   c                 C  sL  |d u rd}|d u r| j }|| j ks||krdS | jd ur<|| jkr<| jd ur<|| jk r<|| j }| j||| |  S | jrIt| j || j }n|}||ksT|| j krVdS | jd u s`|| jk ru| jd u sj|| jkru| ||| _|| _nm| jd us|J | jd usJ || jk r| jd u s| j| | jkr| ||| _|| _nB| || j}|| _|| j | _n1| jd ur|| jkr| j| j krn || j | jkr| ||| _|| _n| | j|}| j| | _| jt| j | _|| j }| j||| |  }| jr$| j| j | jd  }|dkr$|  j| j| 7  _| j| j| d  | _|S )Nr   r&   r=   )	r   r!   rK   r9   r   rU   r   rf   r   )r   r!   rK   offsetZbendnewr   numr   r   r   r(     s^   







zBytesCache._fetchc                 C  s
   t | jS r   )rf   r9   rO   r   r   r   __len__  s   
zBytesCache.__len__)T)
r   r   r   r   r   r   r   r   r   r   r`   )r   r   )
r+   r,   r-   r.   r   r/   r    r(   r   rc   r   r   r:   r   r   k  s   
 
	Ar   c                      sD   e Zd ZU dZdZded< 				dd fddZdddZ  ZS )AllBytesz!Cache entire contents of the fileallr   r   Nr   r"   r   Fetcher | Noner   rm   bytes | Noner   r   c                   s0   t  ||| |d u r| d| j}|| _d S rl   )r6   r    r   r   rm   )r   r   r   r   rm   r:   r   r   r      s   
zAllBytes.__init__r!   r#   r$   c                 C  s   | j || S r   )rm   r'   r   r   r   r(     s   zAllBytes._fetch)NNNN)
r   r"   r   r   r   r"   rm   r   r   r   r*   	r+   r,   r-   r.   r   r/   r    r(   rc   r   r   r:   r   r     s   
 r   c                      sD   e Zd ZU dZdZded< i dfd fddZd fddZ  ZS )KnownPartsOfAFilea  
    Cache holding known file parts.

    Parameters
    ----------
    blocksize: int
        How far to read ahead in numbers of bytes
    fetcher: func
        Function of the form f(start, end) which gets bytes from remote as
        specified
    size: int
        How big this file is
    data: dict
        A dictionary mapping explicit `(start, stop)` file-offset tuples
        with known bytes.
    strict: bool, default True
        Whether to fetch reads that go beyond a known byte-range boundary.
        If `False`, any read that ends outside a known part will be zero
        padded. Note that zero padding will not be used for reads that
        begin outside a known byte-range.
    partsr   r   Tr   r   r   r   r   rm   dict[tuple[int, int], bytes]strictr   _r   c                   s   t  ||| || _|rdt| }|d g}||d g}	|dd  D ]3\}
}|d \}}|
|krH||f|d< |	d  ||
|f7  < q&||
|f |	||
|f q&tt||	| _	d S || _	d S )Nr   r=   )
r6   r    r   sortedkeysrT   r   dictziprm   )r   r   r   r   rm   r   r   Zold_offsetsoffsetsr4   r!   r#   Zstart0Zstop0r:   r   r   r      s   	

zKnownPartsOfAFile.__init__r!   r"   r#   r   r$   c                   s  |d u rd}|d u r| j }d}| j D ]C\\}}}||  kr$|k rWn q|| }|||| |  }| jrC||  krA|krSn n|d|| t|  7 }|  S |} nq| jd u rgtd||f dtd||f d t	
d| d|  |t || S )	Nr   r&       z&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching rL   )r   rm   itemsr   rf   r   r   warningswarnrQ   rR   r6   r(   )r   r!   r#   r   Zloc0loc1rm   offr:   r   r   r(     s,   
zKnownPartsOfAFile._fetch)r   r   r   r   r   r   rm   r   r   r   r   r   r*   r   r   r   r:   r   r     s   
 r   c                   @  sT   e Zd ZdZG dd deZdd ddZd!ddZd"ddZd#ddZ	d$ddZ
dS )%UpdatableLRUzg
    Custom implementation of LRU cache that allows updating keys

    Used by BackgroudBlockCache
    c                   @  s.   e Zd ZU ded< ded< ded< ded< dS )UpdatableLRU.CacheInfor   hitsmissesmaxsizecurrsizeN)r+   r,   r-   r/   r   r   r   r   	CacheInfoF  s
   
 r      funcCallable[P, T]max_sizer   r   r   c                 C  s0   t  | _|| _|| _d| _d| _t | _	d S rl   )
collectionsr   _cache_func	_max_size_hits_misses	threadingLock_lock)r   r   r   r   r   r   r    L  s   
zUpdatableLRU.__init__argsP.argskwargsP.kwargsr   c                 O  s   |rt d|  | j& || jv r-| j| |  jd7  _| j| W  d    S W d    n1 s7w   Y  | j|i |}| j, || j|< |  jd7  _t| j| j	krk| jj
dd W d    |S W d    |S 1 svw   Y  |S )Nz Got unexpected keyword argument r=   Flast)	TypeErrorr   r   r   move_to_endr   r   r   rf   r   popitem)r   r   r   resultr   r   r   __call__T  s.   



zUpdatableLRU.__call__r   r   c                 G  s4   | j  || jv W  d    S 1 sw   Y  d S r   )r   r   )r   r   r   r   r   is_key_cachedg  s   $zUpdatableLRU.is_key_cachedr   c                 G  sd   | j % || j|< t| j| jkr | jjdd W d    d S W d    d S 1 s+w   Y  d S )NFr   )r   r   rf   r   r   )r   r   r   r   r   r   add_keyk  s   
"zUpdatableLRU.add_keyr   c                 C  sH   | j  | j| jt| j| j| jdW  d    S 1 sw   Y  d S )N)r   r   r   r   )r   r   r   rf   r   r   r   rO   r   r   r   r   q  s   $zUpdatableLRU.cache_infoN)r   )r   r   r   r   r   r   )r   r   r   r   r   r   )r   r   r   r   )r   r   r   r   r   r   r   r   )r+   r,   r-   r.   r
   r   r    r   r   r   r   r   r   r   r   r   ?  s    


r   c                      s   e Zd ZU dZdZded< 	d*d+ fddZd,ddZd-ddZd.ddZ	d/ddZ
d0dd Zd1d2 fd$d%Zd3d(d)Z  ZS )4BackgroundBlockCachea  
    Cache holding memory as a set of blocks with pre-loading of
    the next block in the background.

    Requests are only ever made ``blocksize`` at a time, and are
    stored in an LRU cache. The least recently accessed block is
    discarded when more than ``maxblocks`` are stored. If the
    next block is not in cache, it is loaded in a separate thread
    in non-blocking way.

    Parameters
    ----------
    blocksize : int
        The number of bytes to store in each block.
        Requests are only ever made for ``blocksize``, so this
        should balance the overhead of making a request against
        the granularity of the blocks.
    fetcher : Callable
    size : int
        The total size of the file being cached.
    maxblocks : int
        The maximum number of blocks to cache for. The maximum memory
        use for this cache is then ``blocksize * maxblocks``.
    
backgroundr   r   ro   r   r   r   r   r   rp   r   r   c                   sZ   t  ||| t|| | _|| _t| j|| _t	dd| _
d | _d | _t | _d S )Nr=   max_workers)r6   r    rq   rr   rs   rp   r   rv   rw   r   _thread_executor_fetch_future_block_number_fetch_futurer   r   _fetch_future_lockrx   r:   r   r   r      s   zBackgroundBlockCache.__init__ry   c                 C  rz   )Nz <BackgroundBlockCache blocksize=r{   r|   r}   r~   rO   r   r   r   r     r   zBackgroundBlockCache.__repr__r   c                 C  r   r   r   rO   r   r   r   r     r   zBackgroundBlockCache.cache_inforX   c                 C  s(   | j }|d= |d= |d= |d= |d= |S )Nrw   r   r   r   r   r   r[   r   r   r   r]     s   z!BackgroundBlockCache.__getstate__c                 C  sD   | j | t| j|d | _tdd| _d | _d | _t	
 | _d S )Nrp   r=   r   )rY   r^   r   rv   rw   r   r   r   r   r   r   r   r[   r   r   r   r_     s   z!BackgroundBlockCache.__setstate__r!   r"   rK   r$   c           
      C  s  |d u rd}|d u r| j }|| j ks||krdS || j }|| j }d }d }| jM | jd uro| jd us6J | j rRtd | j	| j
 | j d | _d | _nt|| j  ko]|kn  }|ro| j}| j}d | _d | _W d    n1 syw   Y  |d urtd | j	|
 | t||d D ]}| | q|d }	| j% | jd u r|	| jkr| j|	s|	| _| j| j|	d| _W d    n1 sw   Y  | j||||dS )Nr   r&   z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.r=   asyncr   )r   r   r   r   r   donerQ   r   rw   r   r   r   rS   rs   r   r   submitrv   r   )
r   r!   rK   r   r   Zfetch_future_block_numberZfetch_futureZ	must_joinr   Zend_block_plus_1r   r   r   r(     sv   








zBackgroundBlockCache._fetchsyncr   log_infoc                   sV   || j krtd| d| j  d|| j }|| j }td|| t ||}|S )r   r   r   r   z!BlockCache fetching block (%s) %dr   )r   r   r   r!   rK   r   r:   r   r   rv     s   


z!BackgroundBlockCache._fetch_blockr   r   c           
      C  r   r   r   r   r   r   r   r   "  r   z BackgroundBlockCache._read_cacher   r   r   r   ra   )r   r   r`   )r   )r   r   r   ry   r   r$   r   )r+   r,   r-   r.   r   r/   r    r   r   r]   r_   r(   rv   r   rc   r   r   r:   r   r   {  s   
 



	
Lr   z!dict[str | None, type[BaseCache]]cachesFclstype[BaseCache]clobberr   r   r   c                 C  s6   | j }|s|tv rtd|dt|  | t|< dS )z'Register' cache implementation.

    Parameters
    ----------
    clobber: bool, optional
        If set to True (default is False) - allow to overwrite existing
        entry.

    Raises
    ------
    ValueError
    zCache with name z is already known: N)r   r   r   )r   r   r   r   r   r   register_cacheM  s   r   )F)r   r   r   r   r   r   )-
__future__r   r   rt   loggingrq   rA   r   r   concurrent.futuresr   r   typingr   r   r   r   r	   r
   r   r   r1   Ztyping_extensionsr   r   r   	getLoggerrQ   r   r$   r   r   r0   re   rj   rn   r   r   r   r   r   r   r/   r   cr   r   r   r   <module>   sZ    (

!Q* 
]b< O
