U
    Ý@·f?!  ã                   @  sÆ   U d dl mZ d dlZd dlZd dlZd dlmZ d dlmZ zd dl	Z
W n  ek
rj   esfd dl
Z
Y nX er´d dlmZmZmZmZ d dlmZ ddlmZ eeef Zd	ed
< G dd„ dƒZdS )é    )ÚannotationsN)ÚTYPE_CHECKING)Úatomic_write)ÚAnyÚDictÚIteratorÚLiteral)Ú	TypeAliasé   )ÚCachingFileSystemr	   ÚDetailc                   @  sÄ   e Zd ZdZddœdd„Zdddœd	d
„Zddddœdd„Zd0dddœdd„Zddddœdd„Zdddœdd„Z	ddœd d!„Z
d"ddd#œd$d%„Zdd&d'œd(d)„Zddœd*d+„Zdddd,œd-d.„Zd/S )1ÚCacheMetadataa\  Cache metadata.

    All reading and writing of cache metadata is performed by this class,
    accessing the cached files and blocks is not.

    Metadata is stored in a single file per storage directory in JSON format.
    For backward compatibility, also reads metadata stored in pickle format
    which is converted to JSON when next saved.
    z	list[str])Ústoragec                 C  s$   |st dƒ‚|| _i g| _d| _dS )zá

        Parameters
        ----------
        storage: list[str]
            Directories containing cached files, must be at least one. Metadata
            is stored in the last of these directories by convention.
        z3CacheMetadata expects at least one storage locationFN)Ú
ValueErrorÚ_storageÚcached_filesÚ_force_save_pickle)Úselfr   © r   úI/tmp/pip-unpacked-wheel-yjyh6wby/fsspec/implementations/cache_metadata.pyÚ__init__%   s
    	zCacheMetadata.__init__Ústrr   )ÚfnÚreturnc                 C  sŒ   z$t |dƒ}t |¡}W 5 Q R X W n4 tk
rX   t |dƒ}t |¡}W 5 Q R X Y nX | ¡ D ]$}t| d¡tƒrbt	|d ƒ|d< qb|S )z6Low-level function to load metadata from specific fileÚrÚrbÚblocks)
ÚopenÚjsonÚloadr   ÚpickleÚvaluesÚ
isinstanceÚgetÚlistÚset)r   r   ÚfZloadedÚcr   r   r   Ú_load8   s    zCacheMetadata._loadÚNone)Úmetadata_to_saver   r   c              	   C  sP   | j r(t|ƒ}t ||¡ W 5 Q R X n$t|dd}t ||¡ W 5 Q R X dS )z4Low-level function to save metadata to specific fileÚw)ÚmodeN)r   r   r    Údumpr   )r   r*   r   r&   r   r   r   Ú_saveE   s
    
zCacheMetadata._saveFÚboolzIterator[tuple[str, str, bool]])Úwritable_onlyr   c                 c  sN   t | jƒ}t| jƒD ]4\}}||d k}|r2|s2qtj |d¡||fV  qdS )a  Yield locations (filenames) where metadata is stored, and whether
        writable or not.

        Parameters
        ----------
        writable: bool
            Set to True to only yield writable locations.

        Returns
        -------
        Yields (str, str, bool)
        r
   ÚcacheN)Úlenr   Ú	enumerateÚosÚpathÚjoin)r   r0   ÚnÚir   Úwritabler   r   r   Ú_scan_locationsN   s    
zCacheMetadata._scan_locationszCachingFileSystem | Nonez#Literal[False] | tuple[Detail, str])r5   Úcfsr   c                 C  s¦   t |  ¡ | jƒD ]\\}}}}||kr(q||  ¡ }|dk	rv|jrX|d |j |¡krXq|jrvt ¡ |d  |jkrvqt	j
 ||d ¡}t	j
 |¡r||f  S qdS )zõIf path is in cache return its details, otherwise return ``False``.

        If the optional CachingFileSystem is specified then it is used to
        perform extra checks to reject possible matches, such as if they are
        too old.
        NÚuidÚtimer   F)Úzipr:   r   ÚcopyZcheck_filesÚfsZukeyZexpiryr=   r4   r5   r6   Úexists)r   r5   r;   r   ÚbaseÚ_r1   Údetailr   r   r   Ú
check_filed   s    	zCacheMetadata.check_fileÚintztuple[list[str], bool])Úexpiry_timer   c                 C  sÄ   g }| j d  ¡  ¡ D ]h\}}t ¡ |d  |kr| dd¡}|sPtd|› ƒ‚tj | j	d |¡}| 
|¡ | j d  |¡ q| j d r°tj | j	d d¡}|  | j d |¡ | j d  }||fS )zûRemove expired metadata from the cache.

        Returns names of files corresponding to expired metadata and a boolean
        flag indicating whether the writable cache is empty. Caller is
        responsible for deleting the expired files.
        éÿÿÿÿr=   r   Ú z)Cache metadata does not contain 'fn' for r1   )r   r?   Úitemsr=   r#   ÚRuntimeErrorr4   r5   r6   r   ÚappendÚpopr.   )r   rG   Zexpired_filesr5   rD   r   Ú
cache_pathZwritable_cache_emptyr   r   r   Úclear_expired   s     ÿ

zCacheMetadata.clear_expired)r   c                 C  sP   g }|   ¡ D ]2\}}}tj |¡r4| |  |¡¡ q| i ¡ q|pHi g| _dS )z>Load all metadata from disk and store in ``self.cached_files``N)r:   r4   r5   rA   rL   r(   r   )r   r   r   rC   r   r   r   r   ™   s    zCacheMetadata.loadr   )r&   r5   r   c                 C  s>   | j d | }|d dk	r:t|d ƒ|j |jkr:d|d< dS )z‹Perform side-effect actions on closing a cached file.

        The actual closing of the file is the responsibility of the caller.
        rH   r   TN)r   r2   Ú	blocksizeÚsize)r   r&   r5   r'   r   r   r   Úon_close_cached_file¤   s    $z"CacheMetadata.on_close_cached_filez
str | None)r5   r   c                 C  sR   |   |d¡}|sdS |\}}| | jd ¡rF| jd  |¡ |  ¡  ntdƒ‚|S )zÒRemove metadata of cached file.

        If path is in the cache, return the filename of the cached file,
        otherwise return ``None``.  Caller is responsible for deleting the
        cached file.
        NrH   z<Can only delete cached file in last, writable cache location)rE   Ú
startswithr   r   rM   ÚsaveÚPermissionError)r   r5   ÚdetailsrC   r   r   r   r   Úpop_file®   s    
ÿzCacheMetadata.pop_filec           	      C  sN  t |  ¡ | jƒD ]6\\}}}}|s&qtj |¡rì|  |¡}| ¡ D ]‚\}}||krD|d dksp|| d dkrzd|d< n"|| d }| |d ¡ ||d< t	|d || d ƒ|d< || d |d< qD| ¡ D ]\}}||krÐ|||< qÐn|}dd„ | ¡ D ƒ}| 
¡ D ]&}t|d tƒr
t|d ƒ|d< q
|  ||¡ || jd< qdS )	zSave metadata to diskr   Tr=   r<   c                 S  s   i | ]\}}||  ¡ “qS r   )r?   )Ú.0ÚkÚvr   r   r   Ú
<dictcomp>ß   s      z&CacheMetadata.save.<locals>.<dictcomp>rH   N)r>   r:   r   r4   r5   rA   r(   rJ   ÚupdateÚmaxr!   r"   r%   r$   r.   )	r   r   rC   r9   r1   r   rY   r'   r   r   r   r   rT   Â   s0     

zCacheMetadata.save)r5   rD   r   c                 C  s   || j d |< dS )z8Update metadata for specific file in memory, do not saverH   N)r   )r   r5   rD   r   r   r   Úupdate_fileæ   s    zCacheMetadata.update_fileN)F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r(   r.   r:   rE   rO   r   rR   rW   rT   r^   r   r   r   r   r      s   

 ÿ
$r   )Ú
__future__r   r4   r    r=   Útypingr   Zfsspec.utilsr   Zujsonr   ÚImportErrorr   r   r   r   Ztyping_extensionsr	   Úcachedr   r   r   Ú__annotations__r   r   r   r   r   Ú<module>   s"    