U diҼ@sddlZddlZddlZddlmZddlmZmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZmZddlZdd lmZmZmZdd lmZdd l m!Z!d d l"m#Z#d dl$m%Z%m&Z&d dl'm(Z(m)Z)m*Z*d dl+m,Z,m-Z-d dl.m/Z/m0Z0m1Z1d dl2m3Z3m4Z4m5Z5e6dej7Z8eGdddZ9Gdddej:Z;GdddejGdddejZ)d?d@Z*dAdBZ+Z,S)M HfFileSystema% Access a remote Hugging Face Hub repository as if were a local file system. > [!WARNING] > [`HfFileSystem`] provides fsspec compatibility, which is useful for libraries that require it (e.g., reading > Hugging Face datasets directly with `pandas`). However, it introduces additional overhead due to this compatibility > layer. For better performance and reliability, it's recommended to use `HfApi` methods when possible. Args: token (`str` or `bool`, *optional*): A valid user access token (string). Defaults to the locally saved token, which is the recommended method for authentication (see https://huggingface.co/docs/huggingface_hub/quick-start#authentication). To disable authentication, pass `False`. endpoint (`str`, *optional*): Endpoint of the Hub. Defaults to . Usage: ```python >>> from huggingface_hub import HfFileSystem >>> fs = HfFileSystem() >>> # List files >>> fs.glob("my-username/my-model/*.bin") ['my-username/my-model/pytorch_model.bin'] >>> fs.ls("datasets/my-username/my-dataset", detail=False) ['datasets/my-username/my-dataset/.gitattributes', 'datasets/my-username/my-dataset/README.md', 'datasets/my-username/my-dataset/data.json'] >>> # Read/write files >>> with fs.open("my-username/my-model/pytorch_model.bin") as f: ... data = f.read() >>> with fs.open("my-username/my-model/pytorch_model.bin", "wb") as f: ... f.write(data) ``` r0ZhfN)endpointtoken block_sizecsDtj|||ptj|_||_t||d|_||_i|_ i|_ dS)N)rCrD) super__init__rZENDPOINTrCrDr _apirE_repo_and_revision_exists_cachedircache)r8rCrDrEargsstorage_options __class__r9r:rGcs zHfFileSystem.__init__)r'r(r)r/c Cs|||f|jkrz|jj|||tjdWnttfk rx}z,d|f|j|||f<d|f|j||df<W5d}~XYnftk r}z(d|f|j|||f<d|j||df<W5d}~XYn"Xd|j|||f<d|j||df<|j|||fS)N)r)r'timeoutF)TN)rIrHZ repo_inforZHF_HUB_ETAG_TIMEOUTrr#r)r8r'r(r)er9r9r:_repo_and_revision_existys" $"z%HfFileSystem._repo_and_revision_exist)pathr)r/cCs|ttttttddd}||}|s6tdnN|dddtjkr~d|krbtd|dd\}}tj|}ntj }| ddkrd|kr>|dd\}}d|krt |}|d k r|d | fkrt d |d}| }n|dd\}}nd }|t||}||||\} } | st|| nd }d|dd d } d|dd d } |dd} d|ddd }| }| }||||\} } | sVt| ttfr| }|}||||\} }| st|| n t|| nV|}d }d|kr2|dd\}}|t||}nd }||||\} }| sVtd|d k rd|ntj}t|||||d S) a Resolve a Hugging Face file system path into its components. Args: path (`str`): Path to resolve. revision (`str`, *optional*): The revision of the repo to resolve. Defaults to the revision specified in the path. Returns: [`HfFileSystemResolvedPath`]: Resolved path information containing `repo_type`, `repo_id`, `revision` and `path_in_repo`. Raises: `ValueError`: If path contains conflicting revision information. `NotImplementedError`: If trying to list repositories. )revision_in_pathr)r/cSs8|dk r0|dk r4||kr4td|d|dn|}|S)NzRevision specified in path ("z ") and in `revision` argument ("z") are not the same.) ValueError)rSr)r9r9r:%_align_revision_in_path_with_revisionszHHfFileSystem.resolve_path.._align_revision_in_path_with_revisionz0Access to repositories lists is not implemented.r2rrr1Nr0)r-)r r@Z_strip_protocolNotImplementedErrorsplitrr3valuesZREPO_TYPES_MAPPINGZREPO_TYPE_MODELcountSPECIAL_REFS_REVISION_REGEXsearchgroupsublstriprrQ_raise_file_not_foundjoin isinstancerr#r6r&)r8rRr)rUr'r(rSmatchr*Zrepo_and_revision_existerrZrepo_id_with_namespaceZpath_in_repo_with_namespaceZrepo_id_without_namespaceZpath_in_repo_without_namespace_r9r9r: resolve_pathsl           zHfFileSystem.resolve_path)rRr/cCs|s|j|jnh||}|}|rJ|j|d||}q,|js|j|j|j dfd|j|j|j |j fddS)ac Clear the cache for a given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.invalidate_cache). Args: path (`str`, *optional*): Path to clear from cache. If not provided, clear the entire cache. N) rJclearrIrfr;pop_parentr*r'r(r))r8rR resolved_pathr9r9r:invalidate_caches     zHfFileSystem.invalidate_cacherbHfFileSystemFile)rRmoder)rEr/cKsn|dk r |n|j}|dk r"||d<d|kr2td|dkrRt||f||d|St||f||d|SdS)NrEaz/Appending to remote files is not yet supported.r)rnr))rErWHfFileSystemStreamFilerm)r8rRrnr)rEkwargsr9r9r:_openszHfFileSystem._openc KsR|j||d}|jj|j|j|j|j|j|d|dd|j | ddS)Nr)commit_messagecommit_description)r*r(rDr'r)rtrurR) rfrHZ delete_filer*r(rDr'r)r4rkr;)r8rRr)rqrjr9r9r:_rms zHfFileSystem._rmF)rR recursivemaxdepthr)r/c sj||d}j||||d}fdd|D}dd|D} d|d} | |rVdnd 7} | |d k rrd |dnd 7} jj|j|jj| |j|d | |d dj | dd S)a Delete files from a repository. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.rm). > [!WARNING] > Note: When possible, use `HfApi.delete_file()` for better performance. Args: path (`str`): Path to delete. recursive (`bool`, *optional*): If True, delete directory and all its contents. Defaults to False. maxdepth (`int`, *optional*): Maximum number of subdirectories to visit when deleting recursively. revision (`str`, *optional*): The git revision to delete from. rs)rxryr)cs"g|]}|s|jqSr9)isdirrfr*).0rRr8r9r: Cs z#HfFileSystem.rm..cSsg|]}t|dqS))r*)r)r{r*r9r9r:r}DszDelete  z recursively r0Nz up to depth rtru)r(r'rD operationsr)rtrurv) rfZ expand_pathrH create_commitr(r'rDr)r4rkr;) r8rRrxryr)rqrjpathsZ paths_in_reporrtr9r|r:rm&s"   zHfFileSystem.rmT)rRdetailrefreshr)r/c s|j|d}|z|jf||d|}Wnjtk r|jsRtd|j|f||d|}fdd|D}t|dkrtdYnX|r|Sdd|DS)a List the contents of a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.ls). > [!WARNING] > Note: When possible, use `HfApi.list_repo_tree()` for better performance. Args: path (`str`): Path to the directory. detail (`bool`, *optional*): If True, returns a list of dictionaries containing file information. If False, returns a list of file paths. Defaults to True. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `List[Union[str, Dict[str, Any]]]`: List of file paths (if detail=False) or list of file information dictionaries (if detail=True). rs)rr)Ncsg|]}|dkr|qSnamer9r{orvr9r:r}ws z#HfFileSystem.ls..rcSsg|] }|dqSrr9rr9r9r:r}zs)rfr;_ls_treerr*r`rilen)r8rRrrr)rqrjoutr9rvr:lsTs  zHfFileSystem.ls)rRrxrr) expand_inforyc sj||d}|}t|j|j|jd|jd}g} |jkr|sj|} | | g} |rdt fdd| D} | r| \} |dks|kr| djkr| | dqj| d} | | | fdd| Dqg}|r fd d| D}|r| s$|r|rt j | |}|d sZ||ksZ|t| |krd|d n||dk rt|dd }||8}fd d| D} tjD]$}|d rj|dqjd| j|d |||d nЈjj|j|j|||j|jd}|D]}|d |j }t|trf||jd|j|j |j!|j"d}n|dd|j#|j!d}|d}j$|g ||t|dd |dksƈ|kr(| |q(| S)Nrsr0r*r-rVcs g|]}|ddkr|fqStype directoryr9r{ path_infodepthr9r:r}s z)HfFileSystem._ls_tree..rcs$g|]}|ddkrd|fqS)rrrr9rrr9r:r}s cs&g|]}|ddkr|dqS) last_commitNr)rirr|r9r:r}s r2cs"g|]}|dds|qS)rr2) startswithr) common_pathr9r:r}sT)rxrr)rry)rxexpandr)r'filersizerblob_idlfsrsecurityrrrrrtree_idr)%rfr;r&r'r(r)r-rJextendrpopleftappendosrR commonprefixendswithrr5rirrZlistrrhrrHZlist_repo_treer*rbr"rrrrrr setdefault)r8rRrxrr)rryrj root_pathrZcached_path_infosZdirs_not_in_dircacheZ dirs_to_visitZdir_infoZdirs_not_expanded common_prefixZcommon_path_depthZ cached_pathtreer cache_pathZcache_path_info parent_pathr9)rrr8r:r|s            zHfFileSystem._ls_treec/s6|j||dd}tj|f||EdHdS)a Return all files below the given path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.walk). Args: path (`str`): Root path to list files from. Returns: `Iterator[Tuple[str, List[str], List[str]]]`: An iterator of (path, list of directory names, list of file names) tuples. r)rsN)rfr4r;rFwalk)r8rRrKrqrMr9r:rs zHfFileSystem.walkc s(|j||dd}tj|f|S)ah Find files by glob-matching. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.glob). Args: path (`str`): Path pattern to match. Returns: `List[str]`: List of paths matching the pattern. r)rs)rfr4r;rFglobr8rRrqrMr9r:rs zHfFileSystem.glob)rRrywithdirsrrr)r/c s,|dk r|dkrtd|j||d}|}z"|j|fd||j|d|Wn^tk rz0|j|fd|i|dd kr|iiniWntk riYnXYnVX|sd d Dn2|j|fd|ji|} | dd kr| gnd dDt} |s| Sfdd| DSdS)a List all files below path. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.find). Args: path (`str`): Root path to list files from. maxdepth (`int`, *optional*): Maximum depth to descend into subdirectories. withdirs (`bool`, *optional*): Include directory paths in the output. Defaults to False. detail (`bool`, *optional*): If True, returns a dict mapping paths to file information. Defaults to False. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to list from. Returns: `Union[List[str], Dict[str, Dict[str, Any]]]`: List of paths or dict of file information. Nrzmaxdepth must be at least 1rsT)rxrr)ryr)rrcSsg|]}|ddkr|qSrr9rr9r9r:r}Es z%HfFileSystem.find..rcSsi|]}|d|qSrr9rr9r9r: Jsz%HfFileSystem.find..csi|]}||qSr9r9)r{rrr9r:rOs) rTrfr;rr)rinfoFileNotFoundErrorsorted) r8rRryrrrr)rqrjrnamesr9rr:finds>   zHfFileSystem.find)path1path2r)r/c Ks|j||d}|j||d}|j|jko2|j|jk}|rd|d|}|jj|j|j|j|d||ddt|j|j|jdgdnj|j |d |jd} | } W5QRXd|d|}|jj | |j|j|j |j|j|d||dd |j |d |j |d d S) a Copy a file within or between repositories. > [!WARNING] > Note: When possible, use `HfApi.upload_file()` for better performance. Args: path1 (`str`): Source path to copy from. path2 (`str`): Destination path to copy to. revision (`str`, *optional*): The git revision to copy from. rszCopy z to rtrur0)Zsrc_path_in_repor*Z src_revision)r(r'r)rtrurrlZpath_or_fileobjr*r(rDr'r)rtrurvN)rfr'r(rHrr)r4rr*openread upload_filerDrkr;) r8rrr)rqZresolved_path1Zresolved_path2Z same_reportfcontentr9r9r:cp_fileQsD    zHfFileSystem.cp_filecKs"|j|f|ddi}|ddS)ai Get the last modified time of a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.modified). Args: path (`str`): Path to the file. Returns: `datetime`: Last commit date of the file. rTrdater)r8rRrqrr9r9r:modifieds zHfFileSystem.modified)rRrr)r/c  s|j|d}||dd}|jsxdddd|rt|jj|j|j|jdd }dt |j |j |j d d nDd| }|s||jkr||||jkrևfd d |j|D} | std| d|sdks|rrddkr|jj|j|j||j|jd} | s*td| d} t|j|j|jd|jd} t| tr| d| j| jd| j| j| j| jdn| d| jdd| j| jd|sfdddDdk stS)a Get information about a file or directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.info). > [!WARNING] > Note: When possible, use `HfApi.get_paths_info()` or `HfApi.repo_info()` for better performance. Args: path (`str`): Path to get info for. refresh (`bool`, *optional*): If True, bypass the cache and fetch the latest data. Defaults to False. revision (`str`, *optional*): The git revision to get info from. Returns: `Dict[str, Any]`: Dictionary containing file information (type, size, commit info, etc.). rsrFrrN)rrrr)r'r))oidtitler)rrcsg|]}|dkr|qSrr9rrvr9r:r}s z%HfFileSystem.info..r)rr)r'r0rr2rrrcsi|]}||qSr9r9)r{krr9r:rsz%HfFileSystem.info..)rrr)rfr;r4r*rHZlist_repo_commitsr(r'r)r!Z commit_idrZ created_atrirJrr`Zget_paths_infor&r-rbr"rRrrrrrrAssertionError) r8rRrr)rqrjrrrZout1Z paths_inforrr9)rrRr:rs     &     zHfFileSystem.infocKs>z*|ddr|||j|f|WdSYdSXdS)a Check if a file exists. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists). > [!WARNING] > Note: When possible, use `HfApi.file_exists()` for better performance. Args: path (`str`): Path to check. Returns: `bool`: True if file exists, False otherwise. rFTN)r4rkrrr9r9r:existss  zHfFileSystem.existscCs0z||ddkWStk r*YdSXdS)ai Check if a path is a directory. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isdir). Args: path (`str`): Path to check. Returns: `bool`: True if path is a directory, False otherwise. rrFN)rOSErrorr8rRr9r9r:rzs zHfFileSystem.isdircCs(z||ddkWSYdSXdS)a` Check if a path is a file. For more details, refer to [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.isfile). Args: path (`str`): Path to check. Returns: `bool`: True if path is a file, False otherwise. rrFNrrr9r9r:isfile&s zHfFileSystem.isfilecCsB||}t|j|j|j|j|jd}||r>|ddd}|S)z Get the HTTP URL of the given path. Args: path (`str`): Path to get URL for. Returns: `str`: HTTP URL to access the file or directory on the Hub. )r'r)rCz /resolve/z/tree/r) rfrr(r*r'r)rCrzreplace)r8rRrjurlr9r9r:r8s  zHfFileSystem.urlr.c sV|d}t|dh}t|ttfr6t|dkrRtj||f||d|St |r`|}n| |r|t j |dddSt|t tfrt j t j|ddd}|dkrt|d}d}|} |j||d } |j||d d } || zTtt| j| j| j| j|jd ||| d|jt|tr,|j ndd |!| W5|rP|XdS) aC Copy single remote file to local. > [!WARNING] > Note: When possible, use `HfApi.hf_hub_download()` for better performance. Args: rpath (`str`): Remote path to download from. lpath (`str`): Local path to download to. callback (`Callback`, *optional*): Optional callback to track download progress. Defaults to no callback. outfile (`IO`, *optional*): Optional file-like object to write to. If provided, `lpath` is ignored. r)r)callbackoutfileT)exist_okNFwbrsrr(r)filenamer'rC)r temp_fileZdisplayed_filename expected_sizeZ resume_sizeheadersZ _tqdm_bar)"r4setkeysrbrrrrFget_filerrzrmakedirsr@rrRdirnamertellrfrZset_sizecloserrr(r)r*r'rCrH_build_hf_headersZtqdmseek) r8ZrpathZlpathrrrqr)Zunhandled_kwargsZ close_fileZ initial_posZresolve_remote_pathrrMr9r:rOsL    zHfFileSystem.get_filecCs tddS)zA context within which files are committed together upon exit Requires the file class to implement `.commit()` and `.discard()` for the normal and exception cases. (Transactional commits are not supported.NrWr|r9r9r: transactions zHfFileSystem.transactioncCs tddS)z@Begin write transaction for deferring files, non-context versionrNrr|r9r9r:start_transactionszHfFileSystem.start_transactioncCs"tt||j|j|j|jdffS)N)rJrI) make_instancerZ storage_argsrLrJrIr|r9r9r: __reduce__szHfFileSystem.__reduce__)N)N)rlNN)N)FNN)TFN)FFNFN)NFFFN)N)FN)-r<r=r>r?Z root_markerprotocolr r@rboolintrGr ExceptionrQr&rfrkrrrwrr r rrrr rrrrrrrrrzrrrrpropertyrrr __classcell__r9r9rMr:rB:s%  _  / + v* ?8"bD rBcseZdZdeeeedfdd ZfddZeee ddd Z dd d d Z de ddddZ dfdd Zed ddZZS)rmN)fsrRr)c szz|j||d|_WnDtk rX}z&d|ddkrFt|d|W5d}~XYnXtj||jf||dS)Nrswrnr0B. Make sure the repository and revision exist before writing data.)rfrjrr4rFrGr;)r8rrRr)rqrPrMr9r:rGszHfFileSystemFile.__init__cst|dsdStSNrjhasattrrF__del__r|rMr9r:rs zHfFileSystemFile.__del__)startendr/cCsjdd|d|di|jj}t|jj|jj|jj|jj|jj d}t d||t j d}t ||jS)Nrangezbytes=-rrGET)rrO)rrHrrrjr(r)r*r'rCr%rHF_HUB_DOWNLOAD_TIMEOUTr$r)r8rrrrrr9r9r: _fetch_ranges zHfFileSystemFile._fetch_ranger.cCstjddd|_dS)Nzhffs-F)prefixdelete)tempfileNamedTemporaryFilerr|r9r9r:_initiate_uploadsz!HfFileSystemFile._initiate_uploadF)finalr/c Cs|jd|j}|j||r|j|jjj|jj |j j |j j |jj |j j|j j|jd|jddt|jj |jj|j ddS)Nrrtrurrv)bufferrrrwriterrrHrrrjr*r(rDr'r)rqr4rremoverkr;)r8rblockr9r9r: _upload_chunks&       zHfFileSystemFile._upload_chunkrc sv|jdkrj|dks|dkrj|jdkrj|jj|jddd,}|}|jt|7_|W5QRSQRXt|S)a)Read remote file. If `length` is not provided or is -1, the entire file is downloaded and read. On POSIX systems and if `hf_transfer` is not enabled, the file is loaded in memory directly. Otherwise, the file is downloaded to a temporary file and read from there. rlNrr)rE)rnlocrrrRrrrF)r8lengthrrrMr9r:rs $zHfFileSystemFile.readcCs|j|jSNrrrRr|r9r9r:rszHfFileSystemFile.url)N)F)r)r<r=r>rBr@r rGrrbytesrrrrrrrr9r9rMr:rms rmcsveZdZdeeeeeeedfdd Zdeedd d Zded d dZ edddZ fddZ ddZ Z S)rprlNrnone)rrRrnr)rE cache_typec s|dkrtd||dkr,td|d|krDtd|dz|j||d|_WnBtk r}z$d|d d krt|d |W5d}~XYnX|jdd |_tj||jf|||d |d|_ |dS)Nrz:HfFileSystemStreamFile only supports block_size=0 but got rz?HfFileSystemStreamFile only supports cache_type='none' but got rz;HfFileSystemStreamFile only supports reading but got mode=''rsrnr0r)rrrnrEr) rTrfrjrr4r;detailsrFrGresponse) r8rrRrnr)rErrqrPrMr9r:rGs8 zHfFileSystemStreamFile.__init__)r whencecCs6|dkr|dkrdS||jkr*|dkr*dStddS)NrrzCannot seek streaming HF file)r rT)r8r rr9r9r:rs zHfFileSystemStreamFile.seekr)r c CsZ|dkr|fnd}|jdkrjt|jj|jj|jj|jj|jjd}t d||jj dt j d|_t|jzd|jj_|jjj|}Wntk rB|jt|jj|jj|jj|jj|jjd}t d|dd|ji|jj dt j d|_t|jzd|jj_|jjj|}Wn"tk r<|jYnXYnX|jt|7_|S) Nrr9rrT)rstreamrOZRangez bytes=%d-)rrrjr(r)r*r'rrCr%rHrrrr$rawdecode_contentrrrr r)r8r Z read_argsrrr9r9r:r%sZ        zHfFileSystemStreamFile.readr.cCs|j|jSr r r|r9r9r:rVszHfFileSystemStreamFile.urlcst|dsdStSrrr|rMr9r:rYs zHfFileSystemStreamFile.__del__cCst|j|j|j|j|jjffSr )reopenrrRrn blocksizecacherr|r9r9r:r_sz!HfFileSystemStreamFile.__reduce__)rlNrr)r)r)r<r=r>rBr@r rrGrrrrrrr9r9rMr:rps"1 rp)r)r/cCst|r|St|Sr )r[rc safe_quotersr9r9r:r7csr7)sr/cCs t|ddS)Nr0)safe)r)rr9r9r:rgsr)rRrdr/cCsR|}t|tr|d}n*t|tr0|d}nt|trD|d}t||dS)Nz (repository not found)z (revision not found)z (invalid repository id))rbrrr#r)rRrdmsgr9r9r:r`ks      r`rrRrnrErcCs|j||||dS)Nr)rrr9r9r:rvsrcCs,|||}|D]\}}t|||q|Sr )itemssetattr)clsrKrqZinstance_cache_attributes_dictrattrZ cached_valuer9r9r:rzs r)Hrrer collectionsrZ dataclassesrrr itertoolsrpathlibrtypingrr r r r r rr urllib.parserrZfsspecZfsspec.callbacksrrrZ fsspec.utilsrrequestsrr0rZ _commit_apirrerrorsrrrZ file_downloadrrZhf_apir r!r"utilsr#r$r%compileVERBOSEr[r&ZAbstractFileSystemrBspecZAbstractBufferedFilermrpr@r7rrr`rrrr9r9r9r:sN    (    }Me