o
    2ôjq`  ã                   @   s@  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZmZmZ d dlZd dlmZ d dlmZ d d	lmZmZ d d
lmZmZmZmZmZm Z m!Z! d dl"m#Z#m$Z$m%Z%m&Z&m'Z' d dl(m)Z)m*Z* d dl+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6 e,ƒ Z7e$dddddfde8de8dee8 dee8 deee8df dee9 dee dee8 dee8 fdd„Z:e#dddddfde8de8dee8 dee8edf dee8 deeee8f  dee9 dee de8fdd „Z;de$dddddd!œd"e8de8d#e8dee8 dee8 deee8df dee9 dee dee8 dee8 fd$d%„Z<dde&fde8de8de8d#e8fd&d'„Z=de8de8de8fd(d)„Z>d*d+„ Z?		d8d,e8de8d-e8ded.eee8e8f  d/e@fd0d1„ZA	d9d,e8de8d-e8d/e@ded.eee8e8f  fd2d3„ZB	d9d,e8de8d-e8ded.eee8e8f  f
d4d5„ZCd6d7„ ZDdS ):é    N)ÚThreadPoolExecutor)Úpartial)Ú	CookieJar)ÚPath)ÚDictÚOptionalÚUnion)ÚRetry)Útqdm)ÚHubApiÚModelScopeConfig)ÚAPI_FILE_DOWNLOAD_CHUNK_SIZEÚAPI_FILE_DOWNLOAD_RETRY_TIMESÚAPI_FILE_DOWNLOAD_TIMEOUTÚ	FILE_HASHÚMODELSCOPE_DOWNLOAD_PARALLELSÚ)MODELSCOPE_PARALLEL_DOWNLOAD_THRESHOLD_MBÚTEMPORARY_FOLDER_NAME)ÚDEFAULT_DATASET_REVISIONÚDEFAULT_MODEL_REVISIONÚREPO_TYPE_DATASETÚREPO_TYPE_MODELÚREPO_TYPE_SUPPORT)Úget_dataset_cache_rootÚget_model_cache_root)Ú
get_loggeré   )ÚFileDownloadErrorÚInvalidParameterÚNotExistError)ÚModelFileSystemCache)Úfile_integrity_validationÚget_endpointÚmodel_id_to_group_owner_nameFÚmodel_idÚ	file_pathÚrevisionÚ	cache_dirÚ
user_agentÚlocal_files_onlyÚcookiesÚ	local_dirÚreturnc                 C   s   t | |t||||||d�	S )aF  Download from a given URL and cache it if it's not already present in the local cache.

    Given a URL, this function looks for the corresponding file in the local
    cache. If it's not there, download it. Then return the path to the cached
    file.

    Args:
        model_id (str): The model to whom the file to be downloaded belongs.
        file_path(str): Path of the file to be downloaded, relative to the root of model repo.
        revision(str, optional): revision of the model file to be downloaded.
            Can be any of a branch, tag or commit hash.
        cache_dir (str, Path, optional): Path to the folder where cached files are stored.
        user_agent (dict, str, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional):  If `True`, avoid downloading the file and return the path to the
            local cached file if it exists. if `False`, download the file anyway even it exists.
        cookies (CookieJar, optional): The cookie of download request.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.

    Returns:
        string: string of local file or if networking is off, last version of
        file cached on disk.

    Raises:
        NotExistError: The file is not exist.
        ValueError: The request parameter error.

    Note:
        Raises the following errors:

            - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
            if `use_auth_token=True` and the token cannot be found.
            - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError)
            if ETag cannot be determined.
            - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            if some parameter value is invalid
    ©Ú	repo_typer&   r'   r(   r)   r*   r+   )Ú_repo_file_downloadr   )r$   r%   r&   r'   r(   r)   r*   r+   © r0   úo/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/modelscope/hub/file_download.pyÚmodel_file_download'   ó   .÷r2   Ú
dataset_idc                 C   s   t | |t||||||d�	S )aã  Download raw files of a dataset.
    Downloads all files at the specified revision. This
    is useful when you want all files from a dataset, because you don't know which
    ones you will need a priori. All files are nested inside a folder in order
    to keep their actual filename relative to that folder.

    An alternative would be to just clone a dataset but this would require that the
    user always has git and git-lfs installed, and properly configured.

    Args:
        dataset_id (str): A user or an organization name and a dataset name separated by a `/`.
        file_path (str): The relative path of the file to download.
        revision (str, optional): An optional Git revision id which can be a branch name, a tag, or a
            commit hash. NOTE: currently only branch and tag name is supported
        cache_dir (str, Path, optional): Path to the folder where cached files are stored, dataset file will
            be save as cache_dir/dataset_id/THE_DATASET_FILES.
        local_dir (str, optional): Specific local directory path to which the file will be downloaded.
        user_agent (str, dict, optional): The user-agent info in the form of a dictionary or a string.
        local_files_only (bool, optional): If `True`, avoid downloading the file and return the path to the
            local cached file if it exists.
        cookies (CookieJar, optional): The cookie of the request, default None.
    Raises:
        ValueError: the value details.

    Returns:
        str: Local folder path (string) of repo snapshot

    Note:
        Raises the following errors:
        - [`EnvironmentError`](https://docs.python.org/3/library/exceptions.html#EnvironmentError)
        if `use_auth_token=True` and the token cannot be found.
        - [`OSError`](https://docs.python.org/3/library/exceptions.html#OSError) if
        ETag cannot be determined.
        - [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
        if some parameter value is invalid
    r-   )r/   r   )r4   r%   r&   r'   r+   r(   r)   r*   r0   r0   r1   Údataset_file_downloada   r3   r5   r-   Úrepo_idr.   c             	   C   s„  |st }|tvrtd|tf ƒ‚t| |||d�\}	}
|r0|
 |¡}|d ur,t d¡ |S tdƒ‚tƒ }dt	j
|d�i}|d u rCt	 ¡ }g }d }|t kr�|j| ||d�}|j| |d|d u r^d	n|d
�}|D ]*}|d dkrmqd|d |krŽ|
 |¡rŠt d|d › d�¡ |
 |¡  S |} nqdn€|tk�rt| ƒ\}}|sŸt}d}d}	 |j|||dd||d�}d|v rº|d dksÉtd| |d |d f ƒ d S |d d }d	}|D ],}|d dkrÜqÓ|d |krÿ|
 |¡rùt d|d › d�¡ |
 |¡  S |}d} nqÓt|ƒ|k �s
|�rn|d7 }q¤|d u �rtd|| f ƒ‚|t k�r)t| ||ƒ}n|tk�r9|j|d |||d�}t|||	|
||ƒS )Nz'Invalid repo type: %s, only support: %s)r+   r'   r.   z>File exists in local cache, but we're not sure it's up to datez¤Cannot find the requested files in the cached path and outgoing traffic has been disabled. To enable look-ups and downloads online, set 'local_files_only' to False.z
user-agent)r(   )r&   r*   TF)r$   r&   Ú	recursiveÚuse_cookiesÚTypeÚtreer   zFile ÚNamez$ already in cache, skip downloading!r   éd   ú/)Údataset_nameÚ	namespacer&   Ú	root_pathr7   Úpage_numberÚ	page_sizeÚCodeéÈ   z=Get dataset: %s file list failed, request_id: %s, message: %sÚ	RequestIdÚMessageÚDataÚFilesz"The file path: %s not exist in: %s)Ú	file_namer>   r?   r&   )r   r   r   Ú$create_temporary_directory_and_cacheÚget_file_by_pathÚloggerÚwarningÚ
ValueErrorr   r   Úget_user_agentÚget_cookiesÚget_valid_revisionÚget_model_filesÚexistsÚdebugÚget_file_by_infor   r#   r   Úlist_repo_treeÚprintÚlenr   Úget_file_download_urlÚget_dataset_file_urlÚdownload_file)r6   r%   r.   r&   r'   r(   r)   r*   r+   Útemporary_cache_dirÚcacheÚcached_file_pathÚ_apiÚheadersÚ
repo_filesZfile_to_download_metaÚ	repo_fileÚgroup_or_ownerÚnamerA   rB   Úfiles_list_treeZis_existÚurl_to_downloadr0   r0   r1   r/   ›   sÜ   ÿ
ÿ
ÿÿÿÿü
ÿø€
	ùÿÿÿ
ÿ÷
Þ
$ÿ

üÿr/   c           	      C   s¨   |t krtƒ }n|tkrtƒ }t| ƒ\}}|d ur%tj |t¡}t	|ƒ}n$|d u r+|}t
|tƒr4t|ƒ}tj |t||¡}| dd¡}t	|||ƒ}tj|dd� ||fS )NÚ.Ú___T©Úexist_ok)r   r   r   r   r#   ÚosÚpathÚjoinr   r    Ú
isinstancer   ÚstrÚreplaceÚmakedirs)	r$   r+   r'   r.   Zdefault_cache_rootrc   rd   r\   r]   r0   r0   r1   rJ     s&   


ÿrJ   c                 C   s0   t j |¡}t j |¡}d}|jtƒ | ||d�S )a¾  Format file download url according to `model_id`, `revision` and `file_path`.
    e.g., Given `model_id=john/bert`, `revision=master`, `file_path=README.md`,
    the resulted download url is: https://modelscope.cn/api/v1/models/john/bert/repo?Revision=master&FilePath=README.md

    Args:
        model_id (str): The model_id.
        file_path (str): File path
        revision (str): File revision.

    Returns:
        str: The file url.
    zQ{endpoint}/api/v1/models/{model_id}/repo?Revision={revision}&FilePath={file_path})Úendpointr$   r&   r%   )ÚurllibÚparseÚ
quote_plusÚformatr"   )r$   r%   r&   Zdownload_url_templater0   r0   r1   rY   5  s   ürY   c              
   C   sž  | \}}}}}}}}|d u ri nt  |¡}	tt ¡ jƒ|	d< ttddgd�}
|d||f  }	 zwd}tj	 
|¡rYt|dƒ�}| dtj¡}| |¡ W d   ƒ n1 sTw   Y  || }||krdW d S d	||f |	d
< t|dƒ�+}tj|d|	|td�}|jtd�D ]}|r’| |¡ | t|ƒ¡ q‚W d   ƒ W d S 1 sŸw   Y  W d S  tyÍ } z|
jd||d�}
t d||f ¡ |
 ¡  W Y d }~nd }~ww q/)NúX-Request-IDr   ÚGET©ÚtotalÚbackoff_factorÚallowed_methodsú_%s_%sTr   Úrbúbytes=%s-%sÚRangeúab+©Ústreamr`   r*   Útimeout©Ú
chunk_size©Úerrorz-Downloading: %s failed, reason: %s will retry)ÚcopyÚdeepcopyro   ÚuuidÚuuid4Úhexr	   r   rk   rl   rS   ÚopenÚseekÚioÚSEEK_ENDÚupdateÚrequestsÚgetr   Úiter_contentr   ÚwriterX   Ú	ExceptionÚ	incrementrL   rM   Úsleep)ÚparamsZmodel_file_pathÚprogressÚstartÚendÚurlrI   r*   r`   Úget_headersÚretryÚpart_file_nameÚpartial_lengthÚfZdownload_startÚrÚchunkÚer0   r0   r1   Údownload_part_with_retryM  sh   ýÿþû
ÿ
€ü
ùôÿ€üçr§   rž   rI   r`   Ú	file_sizec                 C   s´  t ddd|dd| d d�}d}g }tj ||¡}	tjtj |	¡dd	� tt|| ƒƒD ]}
|
| }|
d
 | d
 }| |	|||| |||f¡ q,|d
 |k r_| |	||d
 |d
 | |||f¡ t	dkret	nd}t
|dd��}t| t|¡ƒ W d   ƒ n1 s€w   Y  | ¡  ttj ||¡dƒ�=}|D ]1}|d d|d |d f  }t|dƒ�}| | ¡ ¡ W d   ƒ n1 s½w   Y  t |¡ q–W d   ƒ d S 1 sÓw   Y  d S )NÚBTé   r   úDownloading [Ú]©ÚunitÚ
unit_scaleÚunit_divisorrz   ÚinitialÚdesci   
ri   r   é   Údownload)Úmax_workersÚthread_name_prefixÚwbr}   é   é   r~   )r
   rk   rl   rm   rq   ÚdirnameÚrangeÚintÚappendr   r   ÚlistÚmapr§   ÚcloserŽ   r–   ÚreadÚremove)rž   r+   rI   r*   r`   r¨   r›   Z	PART_SIZEÚtasksr%   Úidxrœ   r�   Z	parallelsÚexecutorÚoutput_fileÚtaskr¡   Z	part_filer0   r0   r1   Úparallel_downloadw  sN   

úÿÿÿþÿü"ÿrÈ   c              
   C   s"  |du ri nt  |¡}tt ¡ jƒ|d< tj ||¡}tj	tj 
|¡dd� t d| |¡ ttddgd�}	 z±td	dd
|dkrA|nddd| d d�}	|dkrot|dƒ� |	 d¡ |	 ¡  	 W d  ƒ W n¡1 sjw   Y  d}
tj |¡r˜t|dƒ�}| dtj¡}
|	 |
¡ W d  ƒ n1 s“w   Y  |
|kržW njd|
|d f |d< t|dƒ�-}tj| d||td�}| ¡  |jtd�D ]}|rÒ|	 t|ƒ¡ | |¡ qÂW d  ƒ n1 sÝw   Y  |	 ¡  W n  t�y } z|j d| |d�}| !¡  W Y d}~nd}~ww q6t d| |¡ dS )a�  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        file_size(int):
            The file size.
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file

    Raises:
        FileDownloadError: File download failed.

    Nrw   Tri   údownloading %s to %sr   rx   ry   r©   rª   r   r«   r¬   r­   zw+r~   r   r€   r�   r‚   r…   r‡   ústoring %s in cache at %s)"r‰   rŠ   ro   r‹   rŒ   r�   rk   rl   rm   rq   rº   rL   rT   r	   r   r
   rŽ   r’   rÀ   rS   r�   r�   r‘   r“   r”   r   Úraise_for_statusr•   r   rX   r–   r—   r˜   r™   )rž   r+   rI   r¨   r*   r`   rŸ   Ztemp_file_pathr    r›   r¢   r£   r¤   r¥   r¦   r0   r0   r1   Úhttp_get_model_file¤  s„   ý
ú
 ýÿþ
ÿû
ÿ
€üø€þÖ.rÌ   c                 C   sØ  d}t tjd|dd�}|du ri nt |¡}tt ¡ jƒ|d< |ƒ �Œ}t	 
d| |j¡ ttdd	gd
�}		 zS| ¡ }
d|
 |d< tj| d||td�}| ¡  |j d¡}|dur[t|ƒnd}tddd||
d| d d�}|jtd�D ]}|r�| t|ƒ¡ | |¡ qq| ¡  W n ty¥ } z|	jd	| |d�}	|	 ¡  W Y d}~nd}~ww q4W d  ƒ n1 s±w   Y  t	 
d| |¡ tj  !|j¡}||krÞt "|j¡ d|||f }t	 #|¡ t$|ƒ‚t %|jtj  &||¡¡ dS )aj  Download remote file, will retry 5 times before giving up on errors.

    Args:
        url(str):
            actual download url of the file
        local_dir(str):
            local directory where the downloaded file stores
        file_name(str):
            name of the file stored in `local_dir`
        cookies(CookieJar):
            cookies used to authentication the user, which is used for downloading private repos
        headers(Dict[str, str], optional):
            http headers to carry necessary info when requesting the remote file

    Raises:
        FileDownloadError: File download failed.

    éÿÿÿÿr·   F)ÚmodeÚdirÚdeleteNrw   rÉ   r   rx   ry   Tz	bytes=%d-r€   r‚   zContent-Lengthr©   rª   r«   r¬   r­   r…   r‡   rÊ   z}File %s download incomplete, content_length: %s but the                     file downloaded length: %s, please download again)'r   ÚtempfileÚNamedTemporaryFiler‰   rŠ   ro   r‹   rŒ   r�   rL   rT   rd   r	   r   Útellr“   r”   r   rË   r`   r¼   r
   r•   r   r’   rX   r–   rÀ   r—   r˜   r™   rk   rl   ÚgetsizerÂ   rˆ   r   rp   rm   )rž   r+   rI   r*   r`   rz   Ztemp_file_managerrŸ   Ú	temp_filer    Zdownloaded_sizer¤   Úcontent_lengthr›   r¥   r¦   Zdownloaded_lengthÚmsgr0   r0   r1   Úhttp_get_fileû  sˆ   
ÿýûÿÿÿ
ú
ÿ
€€þãÝ(ÿÿ
rØ   c                 C   sš   t d d |d k r%tdkr%t| ||d ||d u rd n| ¡ |d d� nt| ||d |d ||d� tj ||d ¡}t|v rGt	||t ƒ | 
||¡S )Niè  ÚSizer   r   )r`   r*   r¨   )r¨   r`   r*   )r   r   rÈ   Úget_dictrÌ   rk   rl   rm   r   r!   Úput_file)rž   Z	file_metar\   r]   r`   r*   rÕ   r0   r0   r1   r[   M  s0   ÿúú	r[   )NN)N)Er‰   r�   rk   rÑ   rs   r‹   Úconcurrent.futuresr   Ú	functoolsr   Úhttp.cookiejarr   Úpathlibr   Útypingr   r   r   r“   Zrequests.adaptersr	   Ú	tqdm.autor
   Úmodelscope.hub.apir   r   Úmodelscope.hub.constantsr   r   r   r   r   r   r   Úmodelscope.utils.constantr   r   r   r   r   Úmodelscope.utils.file_utilsr   r   Úmodelscope.utils.loggerr   Úerrorsr   r   r   Zutils.cachingr    Zutils.utilsr!   r"   r#   rL   ro   Úboolr2   r5   r/   rJ   rY   r§   r¼   rÈ   rÌ   rØ   r[   r0   r0   r0   r1   Ú<module>   s8  $øÿþýüûúùø	
÷=øÿþýüûúùø	
÷>öÿþüûúùø	÷
ö
õ ýÿþ
ý/úÿþýüû
ú3úÿþýüû
ú\ûÿþýü
ûR