U @fp{ @sUddlmZddlZddlZddlZddlZddlZddlZddlZddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZe rddlZddlmZedZnedZedZedZeeegefZGd d d Z Gd d d e Z!Gd dde Z"Gddde Z#Gddde Z$Gddde Z%Gddde Z&Gddde Z'GdddeeefZ(Gddde Z)de iZ*de+d<d&d d!d"d#d$d%Z,e e!e%e"e$e#e&e'e)f D]Z-e,e-qdS)') annotationsN)FutureThreadPoolExecutor) TYPE_CHECKINGAnyCallableClassVarGeneric NamedTupleOptional OrderedDictTypeVar) ParamSpecPTZfsspecc@sneZdZUdZdZded<dddddd d Zd d d d ddZddddZddddZ ddddZ dS) BaseCacheagPass-though cache: doesn't keep anything, calls every time Acts as base class for other cachers Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is none ClassVar[str]nameintFetcherNone blocksizefetchersizereturncCs.||_d|_||_||_d|_d|_d|_dSNr)rnblocksrr hit_count miss_counttotal_requested_bytesselfrrrr$2/tmp/pip-unpacked-wheel-yjyh6wby/fsspec/caching.py__init__:szBaseCache.__init__ int | NonebytesstartstoprcCs<|dkr d}|dkr|j}||jks,||kr0dS|||S)Nr)rrr#r*r+r$r$r%_fetchDszBaseCache._fetchrcCsd|_d|_d|_dS)zAReset hit and miss counts for a more ganular report e.g. by file.rN)rr r!r#r$r$r% _reset_statsMszBaseCache._reset_statsstrcCs0|jdkr|jdkrdSd|j|j|j|jfS)z2Return a formatted string of the cache statistics.rz3 , %s: %d hits, %d misses, %d total requested bytes)rr rr!r0r$r$r% _log_statsSszBaseCache._log_statscCs@d|jjd|jd|jd|jd|jd|jd|jdS) Nz ) __class____name__rrrrr r!r0r$r$r%__repr___szBaseCache.__repr__N) r6 __module__ __qualname____doc__r__annotations__r&r.r1r4r7r$r$r$r%r(s     rcsteZdZdZdZdddddddd fd d Zd d ddZddddddZdd ddZdddddZ Z S) MMapCachezmemory-mapped sparse file cache Opens temporary file, which is filled blocks-wise when data is requested. Ensure there is enough disc space in the temporary location. This cache method might only work on posix mmapNrrz str | Nonezset[int] | Noner)rrrlocationblocksrcs8t||||dkrtn||_||_||_dSN)superr&setr?r> _makefilecache)r#rrrr>r?r5r$r%r&wszMMapCache.__init__zmmap.mmap | bytearrayr/cCsddl}ddl}|jdkr tS|jdks8tj|js|jdkrT|}t |_ n t |jd}| |jd| d|n t |jd}|||jS)Nrzwb+1zr+b)r=tempfiler bytearrayr>ospathexists TemporaryFilerBr?openseekwriteflushfileno)r#r=rHfdr$r$r%rCs       zMMapCache._makefiler'r(r*endrc sDtd|d||dkr"d}|dkr0j}|jksB||krFdS|j}|j}fddt||dD}fddt||dD}jt|7_jt|7_|r6|d}|j}t |jj} j | |7_ td |d |d| d  || j || <j |qj ||S) NzMMap cache fetching -rr,csg|]}|jkr|qSr$r?.0ir0r$r% s z$MMapCache._fetch..rFcsg|]}|jkr|qSr$rWrXr0r$r%r[s zMMap get block #z ())loggerdebugrrranger lenrpopminr!rrDr?add) r#r*rUZ start_blockZ end_blockZneedhitsrZZsstartsendr$r0r%r.s,    zMMapCache._fetchdict[str, Any]cCs|j}|d=|S)NrD)__dict__copyr#stater$r$r% __getstate__s zMMapCache.__getstate__rjrcCs|j|||_dSr@)rgupdaterCrDrir$r$r% __setstate__s zMMapCache.__setstate__)NN) r6r8r9r:rr&rCr.rkrn __classcell__r$r$rEr%r<ls r<csBeZdZdZdZdddddfdd Zd d d d d d ZZS)ReadAheadCachea!Cache which reads only when we get beyond a block of data This is a much simpler version of BytesCache, and does not attempt to fill holes in the cache or keep fragments alive. It is best suited to many small reads in a sequential order (e.g., reading lines from a file). Z readaheadrrrrcs&t|||d|_d|_d|_dS)Nr,r)rAr&rDr*rUr"rEr$r%r&szReadAheadCache.__init__r'r(rTcCs>|dkr d}|dks||jkr$|j}||jks6||kr:dS||}||jkr~||jkr~|jd7_|j||j||jS|j|kr|jkrnn6|jd7_|j||jd}|t|8}|j}n|jd7_d}t|j||j}|j ||7_ | |||_||_|jt|j|_||jd|SNrr,rF) rr*rUrrDr r`rbrr!r)r#r*rUlpartr$r$r%r.s. zReadAheadCache._fetchr6r8r9r:rr&r.ror$r$rEr%rpsrpcsBeZdZdZdZdddddfdd Zd d d d d d ZZS)FirstChunkCachezCaches the first block of a file only This may be useful for file types where the metadata is stored in the header, but is randomly accessed. firstrrrrcs&||kr |}t|||d|_dSr@)rAr&rDr"rEr$r%r&szFirstChunkCache.__init__r'r(rTcCs0|pd}||jkr tddSt||j}||jkr|jdkr|jd7_||jkr|j|7_|d|}|d|j|_||dS|d|j|_|j|j7_|j||}||jkr|j||j7_|||j|7}|j d7_ |S|jd7_|j||7_|||SdS)Nrz,FirstChunkCache: requested start > file sizer,rF) rr]r^rbrrDr r!rr)r#r*rUdatarsr$r$r%r.s0         zFirstChunkCache._fetchrtr$r$rEr%rusrucseZdZdZdZdddddddfdd Zd d Zd d ddZd ddddZddddddZ dddfdd Z ddddddddZ Z S) BlockCachea Cache holding memory as a set of blocks. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z blockcache rrrrrr maxblocksrcs<t|||t|||_||_t||j|_ dSr@) rAr&mathceilrr{ functools lru_cache _fetch_block_fetch_block_cachedr#rrrr{rEr$r%r&7szBlockCache.__init__cCs |jSz The statistics on the block cache. Returns ------- NamedTuple Returned directly from the LRU Cache used internally. r cache_infor0r$r$r%r?s zBlockCache.cache_inforfr/cCs|j}|d=|S)Nrrgrir$r$r%rkJszBlockCache.__getstate__rlcCs&|j|t|d|j|_dS)Nr{)rgrmr~rrrrir$r$r%rnOs  zBlockCache.__setstate__r'r(rTcCst|dkr d}|dkr|j}||jks,||kr0dS||j}||j}t||dD]}||qR|j||||dS)Nrr,rFstart_block_numberend_block_number)rrr_r _read_cache)r#r*rUrr block_numberr$r$r%r.Us    zBlockCache._fetch)rrcst||jkr"td|d|jd||j}||j}|j||7_|jd7_td|t||}|S)= Fetch the block of data for `block_number`. 'block_number=(' is greater than the number of blocks (r\rFzBlockCache fetching block %d) r ValueErrorrr!r r]inforAr.)r#rr*rUblock_contentsrEr$r%rls    zBlockCache._fetch_blockr*rUrrrc Cs||j}||j}|jd7_||kr@||}|||S|||dg}|t|jt|d||||d|d|SdSz Read from our block cache. Parameters ---------- start, end : int The start and end byte positions. start_block_number, end_block_number : int The start and end block numbers. rFNr,rrrextendmapr_appendjoin r#r*rUrrZ start_posZend_posblockoutr$r$r%r~s     zBlockCache._read_cache)ry) r6r8r9r:rr&rrkrnr.rrror$r$rEr%rxs rxcs^eZdZUdZdZded<dddddd d fd d Zd d ddddZddddZZ S) BytesCacheaKCache which holds data in a in-memory bytes object Implements read-ahead by the block size, for semi-random reads progressing through the file. Parameters ---------- trim: bool As we read more data, whether to discard the start of the buffer when we are more than a blocksize ahead of it. r(rrTrrboolr)rrrtrimrcs,t|||d|_d|_d|_||_dS)Nr,)rAr&rDr*rUr)r#rrrrrEr$r%r&s zBytesCache.__init__r'rTcCs|dkr d}|dkr|j}||jks,||kr0dS|jdk r||jkr|jdk r||jkr||j}|jd7_|j||||S|jrt|j||j}n|}||ks||jkrdS|jdks||jkr|jdks||jkr|j||7_|jd7_| |||_||_nF|jdk s.t |jdk s>t |jd7_||jkr|jdksv|j||jkr|j||7_| |||_||_n4|j|j|7_| ||j}||_||j|_n|jdk rd||jkrd|j|jkrnh||j|jkr6|j||7_| |||_||_n.|j||j7_| |j|}|j||_|jt |j|_||j}|j||||}|j r|j|j|jd}|dkr|j|j|7_|j|j|d|_|Srq) rr*rUrrDrrbr!r rAssertionErrorr`r)r#r*rUoffsetZbendnewrnumr$r$r%r.s|      zBytesCache._fetchr/cCs t|jSr@)r`rDr0r$r$r%__len__szBytesCache.__len__)T) r6r8r9r:rr;r&r.rror$r$rEr%rs   IrcsPeZdZUdZdZded<dddddd d fd d Zddd dddZZS)AllBytesz!Cache entire contents of the fileallrrNr'zFetcher | Nonez bytes | Noner)rrrrwrcsNt||||dkrD|jd7_|j|j7_|d|j}||_dS)NrFr)rAr&r r!rrrw)r#rrrrwrEr$r%r&s zAllBytes.__init__r(r)cCs|jd7_|j||S)NrF)rrwr-r$r$r%r.szAllBytes._fetch)NNNN r6r8r9r:rr;r&r.ror$r$rEr%r s  rcsVeZdZUdZdZded<ddddd d d d fd d Zddddfdd ZZS)KnownPartsOfAFilea Cache holding known file parts. Parameters ---------- blocksize: int How far to read ahead in numbers of bytes fetcher: func Function of the form f(start, end) which gets bytes from remote as specified size: int How big this file is data: dict A dictionary mapping explicit `(start, stop)` file-offset tuples with known bytes. strict: bool, default True Whether to fetch reads that go beyond a known byte-range boundary. If `False`, any read that ends outside a known part will be zero padded. Note that zero padding will not be used for reads that begin outside a known byte-range. partsrrNTrrz&Optional[dict[tuple[int, int], bytes]]rr)rrrrwstrict_c st|||||_|rt|}|dg}||dg} |ddD]f\} } |d\} } | | kr| | f|d<| d|| | f7<qL|| | f| || | fqLtt|| |_ ni|_ dS)NrrF) rAr&rsortedkeysrardictziprw)r#rrrrwrrZ old_offsetsoffsetsr?r*r+Zstart0Zstop0rEr$r%r&=s     zKnownPartsOfAFile.__init__r'r(r)cs:|dkr d}|dkr|j}d}|jD]\\}}}||krH|kr(nq(||}|||||}|jr||kr|krnn.|d||t|7}|jd7_|S|}qq(|jdkrtd||fdt d||fdt d|d ||j ||7_ |j d7_ |t||S) Nrr,rFz&Read is outside the known file parts: z. z%. IO/caching performance may be poor!z!KnownPartsOfAFile cache fetching rV)rrwitemsrr`rrrwarningswarnr]r^r!r rAr.)r#r*r+rZloc0loc1rwoffrEr$r%r.[s0 zKnownPartsOfAFile._fetch)NTrr$r$rEr%r$s  rc@sveZdZdZGdddeZdddddd d Zd d d dddZdddddZd dddddZ ddddZ dS) UpdatableLRUzg Custom implementation of LRU cache that allows updating keys Used by BackgroudBlockCache c@s.eZdZUded<ded<ded<ded<dS)UpdatableLRU.CacheInforrdmissesmaxsizecurrsizeN)r6r8r9r;r$r$r$r% CacheInfos rzCallable[P, T]rr)funcmax_sizercCs0t|_||_||_d|_d|_t|_ dSr) collectionsr _cache_func _max_size_hits_misses threadingLock_lock)r#rrr$r$r%r&s  zUpdatableLRU.__init__zP.argszP.kwargsr)argskwargsrc Os|rtd||j@||jkrX|j||jd7_|j|W5QRSW5QRX|j||}|j<||j|<|jd7_t|j|j kr|jj ddW5QRX|S)Nz Got unexpected keyword argument rFFlast) TypeErrorrrr move_to_endrrrr`rpopitem)r#rrresultr$r$r%__call__s     zUpdatableLRU.__call__rr)rrc Gs(|j||jkW5QRSQRXdSr@)rr)r#rr$r$r% is_key_cachedszUpdatableLRU.is_key_cached)rrrc Gs>|j.||j|<t|j|jkr0|jjddW5QRXdS)NFr)rrr`rr)r#rrr$r$r%add_keys zUpdatableLRU.add_keyrr/c Cs<|j,|j|jt|j|j|jdW5QRSQRXdS)N)rrrdr)rrrr`rrrr0r$r$r%rszUpdatableLRU.cache_infoN)r) r6r8r9r:r rr&rrrrr$r$r$r%rsrcseZdZUdZdZded<d"dddddd fd d Zd d ddZdd ddZdd ddZ ddddddZ d#ddddfdd Z ddddddd d!Z Z S)$BackgroundBlockCachea Cache holding memory as a set of blocks with pre-loading of the next block in the background. Requests are only ever made ``blocksize`` at a time, and are stored in an LRU cache. The least recently accessed block is discarded when more than ``maxblocks`` are stored. If the next block is not in cache, it is loaded in a separate thread in non-blocking way. Parameters ---------- blocksize : int The number of bytes to store in each block. Requests are only ever made for ``blocksize``, so this should balance the overhead of making a request against the granularity of the blocks. fetcher : Callable size : int The total size of the file being cached. maxblocks : int The maximum number of blocks to cache for. The maximum memory use for this cache is then ``blocksize * maxblocks``. Z backgroundrrryrrrrzcsZt|||t|||_||_t|j||_t dd|_ d|_ d|_ t |_dS)NrF max_workers)rAr&r|r}rr{rrrr_thread_executor_fetch_future_block_number _fetch_futurerr_fetch_future_lockrrEr$r%r&s zBackgroundBlockCache.__init__rr/cCs |jSrrr0r$r$r%rs zBackgroundBlockCache.cache_inforfcCs(|j}|d=|d=|d=|d=|d=|S)Nrrrrrrrir$r$r%rksz!BackgroundBlockCache.__getstate__cCsD|j|t|j|d|_tdd|_d|_d|_t |_ dS)Nr{rFr) rgrmrrrrrrrrrrrir$r$r%rns   z!BackgroundBlockCache.__setstate__r'r(rTc Cs|dkr d}|dkr|j}||jks,||kr0dS||j}||j}d}d}|j|jdk r|jdk slt|jrtd|j |j |jd|_d|_n:t ||jko|kn}|r|j}|j}d|_d|_W5QRX|dk rtd|j | |t ||dD]}| |q|d} |jF|jdkr~| |jkr~|j | s~| |_|j|j| d|_W5QRX|j||||dS)Nrr,z3BlockCache joined background fetch without waiting.z(BlockCache waiting for background fetch.rFasyncr)rrrrrrdoner]rrrrrr_rrrZsubmitrr) r#r*rUrrZfetch_future_block_numberZ fetch_futureZ must_joinrZend_block_plus_1r$r$r%r. sz        zBackgroundBlockCache._fetchsyncr2)rlog_inforcsv||jkr"td|d|jd||j}||j}td|||j||7_|jd7_t||}|S)rrrr\z!BlockCache fetching block (%s) %drF) rrrr]rr!r rAr.)r#rrr*rUrrEr$r%rVs   z!BackgroundBlockCache._fetch_blockrc Cs||j}||j}|jd7_||kr@||}|||S|||dg}|t|jt|d||||d|d|SdSrrrr$r$r%rhs     z BackgroundBlockCache._read_cache)ry)r)r6r8r9r:rr;r&rrkrnr.rrror$r$rEr%rs     Lrz!dict[str | None, type[BaseCache]]cachesFztype[BaseCache]rr)clsclobberrcCs6|j}|s*|tkr*td|dt||t|<dS)z'Register' cache implementation. Parameters ---------- clobber: bool, optional If set to True (default is False) - allow to overwrite existing entry. Raises ------ ValueError zCache with name z is already known: N)rrr)rrrr$r$r%register_caches  r)F). __future__rrr~loggingr|rJrrconcurrent.futuresrrtypingrrrrr r r r r r=Ztyping_extensionsrrr getLoggerr]rr(rrr<rprurxrrrrrrr;rcr$r$r$r%sZ ,    DV.. ee<Q