o
    2ôjdJ  ã                    @   sÔ  d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	m
Z
mZmZ d dlmZmZ d dlmZ d dlmZ d dlmZmZ d d	lmZmZmZmZmZ d d
lmZ ddlm Z m!Z!m"Z" eƒ Z#edddddddddf
de$dee$ dee$edf deee	e$f  dee% dee deee$e
e$ f  deee$e
e$ f  dee$ deee
e$ e$f  deee
e$ e$f  de$fdd„Z&edddddddddf
de$dee$ dee$edf dee$ deee	e$f  dee% dee deee$e
e$ f  deee$e
e$ f  deee
e$ e$f  deee
e$ e$f  de$fdd„Z'deddddddddddœd e$d!ee$ dee$ dee$edf deee	e$f  dee% dee deee$e
e$ f  deee$e
e$ f  dee$ deee
e$ e$f  deee
e$ e$f  fd"d#„Z(d$e$fd%d&„Z)d'ee$e
e$ f fd(d)„Z*d'e
e$ fd*d+„Z+dedddddfd,e
e$ d-ed.e$d e$d/ed0e$d1e$d!ee$ dee$ dee deee$e
e$ f  deee$e
e$ f  deee
e$ e$f  deee
e$ e$f  fd2d3„Z,dS )4é    N)Ú	CookieJar©ÚPath)ÚDictÚListÚOptionalÚUnion)ÚHubApiÚModelScopeConfig)ÚInvalidParameter)ÚModelFileSystemCache)Úget_model_masked_directoryÚmodel_id_to_group_owner_name)ÚDEFAULT_DATASET_REVISIONÚDEFAULT_MODEL_REVISIONÚREPO_TYPE_DATASETÚREPO_TYPE_MODELÚREPO_TYPE_SUPPORT)Ú
get_loggeré   )Ú$create_temporary_directory_and_cacheÚdownload_fileÚget_file_download_urlFÚmodel_idÚrevisionÚ	cache_dirÚ
user_agentÚlocal_files_onlyÚcookiesÚignore_file_patternÚallow_file_patternÚ	local_dirÚallow_patternsÚignore_patternsÚreturnc                 C   s    t | t|||||||||
|	d�S )aË
  Download all files of a repo.
    Downloads a whole snapshot of a repo's files at the specified revision. This
    is useful when you want all files from a repo, because you don't know which
    ones you will need a priori. All files are nested inside a folder in order
    to keep their actual filename relative to that folder.

    An alternative would be to just clone a repo but this would require that the
    user always has git and git-lfs installed, and properly configured.

    Args:
        model_id (str): A user or an organization name and a repo name separated by a `/`.
        revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
            commit hash. NOTE: currently only branch and tag name is supported
        cache_dir (str, Path, optional): Path to the folder where cached files are stored, model will
            be save as cache_dir/model_id/THE_MODEL_FILES.
        user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        cookies (CookieJar, optional): The cookie of the request, default None.
        ignore_file_pattern (`str` or `List`, *optional*, default to `None`):
            Any file pattern to be ignored in downloading, like exact file names or file extensions.
        allow_file_pattern (`str` or `List`, *optional*, default to `None`):
            Any file pattern to be downloading, like exact file names or file extensions.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.
        allow_patterns (`str` or `List`, *optional*, default to `None`):
            If provided, only files matching at least one pattern are downloaded, priority over allow_file_pattern.
            For hugging-face compatibility.
        ignore_patterns (`str` or `List`, *optional*, default to `None`):
            If provided, files matching any of the patterns are not downloaded, priority over ignore_file_pattern.
            For hugging-face compatibility.
    Raises:
        ValueError: the value details.

    Returns:
        str: Local folder path (string) of repo snapshot

    Note:
        Raises the following errors:
        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
        if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
        ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
        if some parameter value is invalid
    ©Ú	repo_typer   r   r   r   r   r   r    r!   r#   r"   )Ú_snapshot_downloadr   )r   r   r   r   r   r   r   r    r!   r"   r#   © r(   ús/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/modelscope/hub/snapshot_download.pyÚsnapshot_download   s   :ôr*   Ú
dataset_idc                 C   s    t | t|||||||||
|	d�S )að
  Download raw files of a dataset.
    Downloads all files at the specified revision. This
    is useful when you want all files from a dataset, because you don't know which
    ones you will need a priori. All files are nested inside a folder in order
    to keep their actual filename relative to that folder.

    An alternative would be to just clone a dataset but this would require that the
    user always has git and git-lfs installed, and properly configured.

    Args:
        dataset_id (str): A user or an organization name and a dataset name separated by a `/`.
        revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
            commit hash. NOTE: currently only branch and tag name is supported
        cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset will
            be save as cache_dir/dataset_id/THE_DATASET_FILES.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.
        user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        cookies (CookieJar, optional): The cookie of the request, default None.
        ignore_file_pattern (`str` or `List`, *optional*, default to `None`):
            Any file pattern to be ignored in downloading, like exact file names or file extensions.
            Use regression is deprecated.
        allow_file_pattern (`str` or `List`, *optional*, default to `None`):
            Any file pattern to be downloading, like exact file names or file extensions.
        allow_patterns (`str` or `List`, *optional*, default to `None`):
            If provided, only files matching at least one pattern are downloaded, priority over allow_file_pattern.
            For hugging-face compatibility.
        ignore_patterns (`str` or `List`, *optional*, default to `None`):
            If provided, files matching any of the patterns are not downloaded, priority over ignore_file_pattern.
            For hugging-face compatibility.
    Raises:
        ValueError: the value details.

    Returns:
        str: Local folder path (string) of repo snapshot

    Note:
        Raises the following errors:
        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
        if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
        ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
        if some parameter value is invalid
    r%   )r'   r   )r+   r   r   r!   r   r   r   r   r    r"   r#   r(   r(   r)   Údataset_snapshot_downloadd   s   ;ôr,   )r&   r   r   r   r   r   r   r    r!   r"   r#   Úrepo_idr&   c                C   s  |st }|tvrtd|tf ƒ‚t| |	||d�\}}|d ur |nt dt ¡  dd¡¡}|rDt	|j
ƒdkr9tdƒ‚t d| ¡ | ¡ S d	tj|d
�i}dtjvrYtt ¡ ƒ|d< tƒ }|d u rdt ¡ }g }|t k�rtj |d| ¡}td|› �ƒ |j| ||d�}|d }dtjv r�|ni |¥ddi¥}|jd urŸ|j|d< |j| |d|d u rªdn||d�}t|||| |d d ||||||||
d� d| v �rt|| ƒ}t d|› d|› d�¡ zt  tj !|¡|¡ W n– t"�y } zt d|› d|› d|› �¡ W Y d }~nzd }~ww nt|t#k�r{tj |d| ¡}td|› �ƒ t$| ƒ\}}|�s&t%}|}d}d }	 |j&|||d!d||d"�}d#|v �rE|d# d$k�sTtd%| |d& |d' f ƒ d S |d( d) }t|||| |||||||||||
d� t	|ƒ|k �run|d7 }�q-|j'|d*� tj | ¡ ¡S )+Nz'Invalid repo type: %s, only support: %s)r!   r   r&   ÚMODELSCOPE_CACHEz.cacheÚ
modelscoper   z¤Cannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z6We can not confirm the cached file is for revision: %sz
user-agent)r   ZCI_TESTzsnapshot-identifierÚhubz Downloading Model to directory: )r   r   ZRevisionÚSnapshotÚTrueÚcached_model_revisionTF)r   r   Ú	recursiveZuse_cookiesÚheaders)r&   r   r   r   r    r#   r"   Ú.zCreating symbolic link z -> zFailed to create symbolic link z: Údatasetsz"Downloading Dataset to directory: r   éd   ú/)Údataset_nameÚ	namespacer   Ú	root_pathr4   Úpage_numberÚ	page_sizeÚCodeéÈ   z=Get dataset: %s file list failed, request_id: %s, message: %sZ	RequestIdÚMessageÚDataÚFiles)Zrevision_info)(r   r   r   r   ÚosÚgetenvr   ÚhomeÚjoinpathÚlenÚcached_filesÚ
ValueErrorÚloggerÚwarningZget_root_locationr
   Zget_user_agentÚenvironÚstrÚuuidÚuuid4r	   Zget_cookiesÚpathÚjoinÚprintZget_valid_revision_detailr3   Zget_model_filesÚ_download_file_listsr   ÚinfoÚsymlinkÚabspathÚOSErrorr   r   r   Úlist_repo_treeZsave_model_version)r-   r&   r   r   r   r   r   r   r    r!   r"   r#   Útemporary_cache_dirÚcacheZsystem_cacher5   Ú_apiÚ
repo_filesÚ	directoryZrevision_detailZsnapshot_headerZmasked_directoryÚeÚgroup_or_ownerÚnamer=   r>   Zfiles_list_treer(   r(   r)   r'   ®   s  ÿ
ÿþÿÿ
þ

ÿÿÿþ
ÿÿûñ
ÿÿÿ€ÿø
ù
ÿÿÿñÝ%r'   Úpatternc                 C   s&   zt  | ¡ W dS  ty   Y dS w )NTF)ÚreÚcompileÚBaseException)rb   r(   r(   r)   Ú_is_valid_regexF  s   
ÿrf   Úpatternsc                 C   s*   t | tƒr| g} | d urdd„ | D ƒ} | S )Nc                 S   s"   g | ]}|  d ¡s|n|d ‘qS )r9   Ú*)Úendswith)Ú.0Úitemr(   r(   r)   Ú
<listcomp>R  s    ÿz'_normalize_patterns.<locals>.<listcomp>)Ú
isinstancerN   )rg   r(   r(   r)   Ú_normalize_patternsN  s   
ÿrn   c                 C   s0   | d urg }| D ]}t |ƒr| |¡ q|S d S )N)rf   Úappend)rg   Zregex_patternsrk   r(   r(   r)   Ú_get_valid_regex_patternX  s   
€rp   r]   r[   rZ   Úapira   r`   c                    s²  t |ƒ}t |ƒ}t |ƒ}t |ƒ}t|ƒ}| D ]À‰ ˆ d dkrqzU|r/t‡ fdd„|D ƒƒr/W q|r>t‡ fdd„|D ƒƒr>W q|rMt‡ fdd„|D ƒƒrMW q|d ur`|r`t‡ fdd„|D ƒƒs`W q|d urs|rst‡ fd	d„|D ƒƒssW qW n ty� } zt d
| ¡ W Y d }~nd }~ww | ˆ ¡r¥tj 	ˆ d ¡}t 
d|› d�¡ q|tkr³t|ˆ d |	d�}n|tkrÃ|jˆ d |||	d�}n
td|› dt› �ƒ‚t|ˆ ||||
ƒ qd S )NÚTypeÚtreec                    ó   g | ]
}t   ˆ d  |¡‘qS r   ©Úfnmatch©rj   rb   ©Ú	repo_filer(   r)   rl   €  ó    ÿÿz(_download_file_lists.<locals>.<listcomp>c                    rt   r   ru   rw   rx   r(   r)   rl   †  rz   c                    s    g | ]}t  |ˆ d  ¡du‘qS )ÚNameN)rc   Úsearchrw   rx   r(   r)   rl   Œ  s    ÿÿc                 3   ó    � | ]}t   ˆ d  |¡V  qdS ©r   Nru   rw   rx   r(   r)   Ú	<genexpr>“  ó
   € ÿ
ÿz'_download_file_lists.<locals>.<genexpr>c                 3   r}   r~   ru   rw   rx   r(   r)   r   ™  r€   z The file pattern is invalid : %sr{   zFile z$ already in cache, skip downloading!r   )r   Ú	file_pathr   )Ú	file_namer:   r;   r   zInvalid repo type: z, supported types: )rn   rp   ÚanyÚ	ExceptionrK   rL   ÚexistsrD   rQ   ÚbasenameÚdebugr   r   r   Zget_dataset_file_urlr   r   r   )r]   r[   rZ   r-   rq   ra   r`   r5   r&   r   r   r   r    r"   r#   Zignore_regex_patternr_   r‚   Úurlr(   rx   r)   rT   c  s|   þþþþþ€€ÿ

ÿýüÿÿÄrT   )-rv   rD   rc   rO   Úhttp.cookiejarr   Úpathlibr   Útypingr   r   r   r   Zmodelscope.hub.apir	   r
   Zmodelscope.hub.errorsr   Zmodelscope.hub.utils.cachingr   Zmodelscope.hub.utils.utilsr   r   Úmodelscope.utils.constantr   r   r   r   r   Úmodelscope.utils.loggerr   Úfile_downloadr   r   r   rK   rN   Úboolr*   r,   r'   rf   rn   rp   rT   r(   r(   r(   r)   Ú<module>   sB  õÿþýüûúùø	÷
öõ
ôKõÿþýüûúùø	÷
öõ
ôMóÿýüûúùø	÷
öõô
ó 
ñÿþýüûúù	÷
öõôóòñ