o
    2ôjÝ  ã                   @   s
  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZmZmZmZmZ d dlmZ d dlZd dlZd dlmZ d d	lmZmZ d d
lm Z  d dl!m"Z" d dl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3 d dl4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? d dl@mAZA d dlBmCZC d dlDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNmOZOmPZPmQZQmRZRmSZSmTZT d dlUmVZV ddlWmXZXmYZYmZZZm[Z[ eVƒ Z\G dd„ dƒZ]G dd„ dƒZ^dS )é    N)Údefaultdict)Ú
HTTPStatus)Ú	CookieJar)Ú
expanduser)ÚDictÚListÚOptionalÚTupleÚUnion)Ú	urlencode)ÚSession)ÚHTTPAdapterÚRetry)Úutils)Úio)ÚAPI_HTTP_CLIENT_MAX_RETRIESÚAPI_HTTP_CLIENT_TIMEOUTÚAPI_RESPONSE_FIELD_DATAÚAPI_RESPONSE_FIELD_EMAILÚ#API_RESPONSE_FIELD_GIT_ACCESS_TOKENÚAPI_RESPONSE_FIELD_MESSAGEÚAPI_RESPONSE_FIELD_USERNAMEÚDEFAULT_CREDENTIALS_PATHÚMODELSCOPE_CLOUD_ENVIRONMENTÚMODELSCOPE_CLOUD_USERNAMEÚMODELSCOPE_REQUEST_IDÚONE_YEAR_SECONDSÚREQUESTS_API_HTTP_METHODÚDatasetVisibilityÚLicensesÚModelVisibility)ÚInvalidParameterÚNotExistErrorÚNotLoginExceptionÚNoValidRevisionErrorÚRequestErrorÚdatahub_raise_on_errorÚhandle_http_post_errorÚhandle_http_responseÚis_okÚraise_for_http_statusÚraise_on_error)ÚGitCommandWrapper)Ú
Repository)ÚDEFAULT_DATASET_REVISIONÚDEFAULT_MODEL_REVISIONÚDEFAULT_REPOSITORY_REVISIONÚMASTER_MODEL_BRANCHÚMETA_FILES_FORMATÚREPO_TYPE_MODELÚConfigFieldsÚDatasetFormationsÚDatasetMetaFormatsÚDatasetVisibilityMapÚDownloadChannelÚDownloadModeÚ
FrameworksÚ	ModelFileÚTasksÚVirgoDatasetConfig)Ú
get_loggeré   )Úget_endpointÚget_readable_folder_sizeÚget_release_datetimeÚmodel_id_to_group_owner_namec                   @   sˆ  e Zd ZdZdeefdee fdd„Zdefdd„Z	e
jejdd	fd
edee dee dee dee defdd„Zd
efdd„Zd
efdd„Zefd
edee defdd„Zddœdedee defdd„Zedd„ ƒZe
jejdd dedddf	d
ed!edee dee dee d"ee d#ee dee dee d$eeee ef  d%eeeee f  fd&d'„Z	(	)d�d*ed+ee d,ee defd-d.„Z	/d�d0eeef defd1d2„Z		/d‘d
ed3ee d0eeef dee fd4d5„Z 		/d‘d
ed3ee d0eeef dee fd6d7„Z!d8d9„ Z"		d’d
ed:ee fd;d<„Z#		d’d
ed:ee fd=d>„Z$	/d�d
ed0eeef de%ee ee f fd?d@„Z&	/d�d
ed0eeef de%ee ee f fdAdB„Z'edd/d/i fd
edee dCee dDee d0eeef dEee dee fdFdG„Z(ddHœdedIedee fdJdK„Z)d	eje*jd	fdLedMedee dee dee dNee defdOdP„Z+dQdR„ Z,dLedMefdSdT„Z-	/	U	Vd“dWededXedYe.dDef
dZd[„Z/	\	(	Ud”dLedMeded]edDed+ed,efd^d_„Z0dLedMed`edefdadb„Z1edceddefdedf„ƒZ2dLedMededdedcedge3fdhdi„Z4edje5j6fdkdl„ƒZ7e8d/d\fdmedLedMedee dnee doee fdpdq„Z9e8fdmedLedMedee fdrds„Z:e8fdLedMedee fdtdu„Z;e8fdLedMedvedee fdwdx„Z<d•d`edyedefdzd{„Z=dLedMeded|efd}d~„Z>dd€„ Z?d�edLedMededef
d‚dƒ„Z@d�edLedMededef
d„d…„ZAd†d‡„ ZBd�dLedMedˆeddfd‰dŠ„ZCd‹dŒ„ ZDdMedLedefd�dŽ„ZEdS )–ÚHubApizModel hub api interface.
    NÚendpointc                 C   s’   |dur|nt ƒ | _dt ¡ i| _tƒ | _t|dddddd�}t|d�}| j 	d	|¡ | j 	d
|¡ t
D ]}t| j|tjt| j|ƒ|d�ƒ q4dS )u’   The ModelScope HubApiã€‚

        Args:
            endpoint (str, optional): The modelscope server http|https address. Defaults to None.
        Nú
user-agenté   r?   )iô  iö  i÷  iø  F)ÚtotalÚreadÚconnectÚbackoff_factorÚstatus_forcelistÚrespect_retry_after_header)Úmax_retrieszhttp://zhttps://)Útimeout)r@   rE   ÚModelScopeConfigÚget_user_agentÚheadersr   Úsessionr   r   Úmountr   ÚsetattrÚ	functoolsÚpartialÚgetattr)ÚselfrE   rO   rN   ÚretryÚadapterÚmethod© r]   úe/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/modelscope/hub/api.pyÚ__init__D   s.   	ú

þþÿzHubApi.__init__Úaccess_tokenc                 C   s”   | j › d�}| jj|d|i|  | j¡d�}t|ƒ | ¡ }t|ƒ |t t	 }|j
}t |¡ t |¡ t |t t |t t ¡ |t t	 |fS )a¡  Login with your SDK access token, which can be obtained from
           https://www.modelscope.cn user center.

        Args:
            access_token (str): user access token on modelscope.

        Returns:
            cookies: to authenticate yourself to ModelScope open-api
            git_token: token to access your git repository.

        Note:
            You only have to login once within 30 days.
        z/api/v1/loginZAccessToken)ÚjsonrR   )rE   rS   ÚpostÚbuilder_headersrR   r*   ra   r+   r   r   ÚcookiesrP   Ú
save_tokenÚsave_cookiesÚsave_user_infor   r   )rY   r`   ÚpathÚrÚdÚtokenrd   r]   r]   r^   Úloginc   s,   
ý



þÿÿzHubApi.loginÚ Úmodel_idÚ
visibilityÚlicenseÚchinese_nameÚoriginal_model_idÚreturnc              
   C   s¦   |du rt dƒ‚t ¡ }|du rtdƒ‚| j› d�}t|ƒ\}}	||	||||tj dd¡dœ}
| j	j
||
||  | j¡d�}t|||
ƒ t| ¡ ƒ tƒ › d	|› �}|S )
a�  Create model repo at ModelScope Hub.

        Args:
            model_id (str): The model id
            visibility (int, optional): visibility of the model(1-private, 5-public), default 5.
            license (str, optional): license of the model, default none.
            chinese_name (str, optional): chinese name of the model.
            original_model_id (str, optional): the base model id which this model is trained from

        Returns:
            Name of the model created

        Raises:
            InvalidParameter: If model_id is invalid.
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nzmodel_id is required!ú)Token does not exist, please login first.z/api/v1/modelsZMODELSCOPE_TRAIN_IDrm   )ÚPathÚNameÚChineseNameÚ
VisibilityÚLicenseZOriginalModelIdZTrainId)ra   rd   rR   ú/)r!   rP   Úget_cookiesÚ
ValueErrorrE   rC   ÚosÚenvironÚgetrS   rb   rc   rR   r'   r+   ra   r@   )rY   rn   ro   rp   rq   rr   rd   rh   Úowner_or_groupÚnameÚbodyri   Zmodel_repo_urlr]   r]   r^   Úcreate_modelŠ   s2   ù	
üzHubApi.create_modelc                 C   sZ   t  ¡ }|du rtdƒ‚| j› d|› �}| jj|||  | j¡d�}t|ƒ t	| 
¡ ƒ dS )zÎDelete model_id from ModelScope.

        Args:
            model_id (str): The model id.

        Raises:
            ValueError: If not login.

        Note:
            model_id = {owner}/{name}
        Nrt   ú/api/v1/models/©rd   rR   )rP   r{   r|   rE   rS   Údeleterc   rR   r*   r+   ra   )rY   rn   rd   rh   ri   r]   r]   r^   Údelete_model¾   s   
þzHubApi.delete_modelc                 C   s   | j › d|› d�S )Nr„   z.git©rE   )rY   rn   r]   r]   r^   Úget_model_urlÕ   s   zHubApi.get_model_urlÚrevisionc                 C   s´   t  ¡ }t|ƒ\}}|r| j› d|› d|› d|› �}n| j› d|› d|› �}| jj|||  | j¡d�}t|t	||ƒ |j
tjkrTt| ¡ ƒrL| ¡ t S t| ¡ t ƒ‚t|ƒ dS )an  Get model information at ModelScope

        Args:
            model_id (str): The model id.
            revision (str optional): revision of model.

        Returns:
            The model detail information.

        Raises:
            NotExistError: If the model is not exist, will throw NotExistError

        Note:
            model_id = {owner}/{name}
        r„   rz   z
?Revision=r…   N)rP   r{   rC   rE   rS   r   rc   rR   r(   ÚloggerÚstatus_coder   ÚOKr)   ra   r   r"   r   r*   )rY   rn   rŠ   rd   r€   r�   rh   ri   r]   r]   r^   Ú	get_modelØ   s   

ÿzHubApi.get_model)Ú	repo_typeÚrepo_idr�   c          	      C   sÊ   |dur|j tkrtd| ƒ‚|du s| d¡dkr td| ƒ‚t ¡ }t|ƒ\}}| j› d|› d|› �}| jj	|||  
| j¡d�}t|t||dƒ}|d	krPd
S |dkrVdS t d|› d�¡ td| ƒ‚)aÝ  
        Checks if a repository exists on ModelScope

        Args:
            repo_id (`str`):
                A namespace (user or an organization) and a repo name separated
                by a `/`.
            repo_type (`str`, *optional*):
                `None` or `"model"` if getting repository info from a model. Default is `None`.
                TODO: support dataset and studio

        Returns:
            True if the repository exists, False otherwise.
        NzNot support repo-type: %srz   r?   z5Invalid repo_id: %s, must be of format namespace/namer„   r…   FéÈ   Ti”  z%Check repo_exists return status code Ú.zOFailed to check existence of repo: %s, make sure you have access authorization.)Úlowerr3   Ú	ExceptionÚcountrP   r{   rC   rE   rS   r   rc   rR   r(   r‹   Úwarn)	rY   r�   r�   rd   r€   r�   rh   ri   Úcoder]   r]   r^   Úrepo_existsþ   s*   

ÿÿÿzHubApi.repo_existsc                 C   s4   t j | tj¡}tjtjtj	t
ji}t ||¡ d S ©N)r}   rh   Újoinr;   ÚCONFIGURATIONr4   Ú	frameworkr:   ÚtorchÚtaskr<   Úotherr   Údump)Ú	model_dirÚcfg_fileÚcfgr]   r]   r^   Ú_create_default_config(  s
   þzHubApi._create_default_configzupload modelr¡   Úcommit_messageÚtagÚignore_file_patternÚ
lfs_suffixc              
      s  |du rt dƒ‚|du rt dƒ‚tj |¡rtj |¡r t dƒ‚tj |tj¡}tj |¡s@t 	dtj› d|› d�¡ t
 |¡ t ¡ }|du rLtdƒ‚t |¡}t|ƒ}|
du r[g }
t|
tƒrc|
g}
|du sk|du rot d	ƒ‚|  |¡s…t d
| ¡ | j|||||	d� t ¡ }tƒ }t d|› d|› d�¡ t d|› d�¡ zÚzÆt||d�}| |¡}||vr¿t d| ¡ | ||¡ | ||¡ t |¡}|D ]"‰ ˆ d dkrîtj |ˆ ¡}tj |¡rçt |¡ qÌtj|dd� qÌ|D ]6‰ ˆ d dk�r't ‡ fdd„|
D ƒƒ�rqñtj |ˆ ¡}tj !|¡�r!t "|tj |ˆ ¡¡ qñt #||¡ qñ|�s9t$j$ %¡  &d¡}d||f }|du�rTt|tƒ�rG|gn|}|D ]}| '|¡ �qK|j(|||d� |du�rg| )||¡ W n	 t*�yq   ‚ w W tj|dd� dS tj|dd� w )a^
  Upload model from a given directory to given repository. A valid model directory
        must contain a configuration.json file.

        This function upload the files in given directory to given repository. If the
        given repository is not exists in remote, it will automatically create it with
        given visibility, license and chinese_name parameters. If the revision is also
        not exists in remote repository, it will create a new branch for it.

        This function must be called before calling HubApi's login with a valid token
        which can be obtained from ModelScope's website.

        If any error, please upload via git commands.

        Args:
            model_id (str):
                The model id to be uploaded, caller must have write permission for it.
            model_dir(str):
                The Absolute Path of the finetune result.
            visibility(int, optional):
                Visibility of the new created model(1-private, 5-public). If the model is
                not exists in ModelScope, this function will create a new model with this
                visibility and this parameter is required. You can ignore this parameter
                if you make sure the model's existence.
            license(`str`, defaults to `None`):
                License of the new created model(see License). If the model is not exists
                in ModelScope, this function will create a new model with this license
                and this parameter is required. You can ignore this parameter if you
                make sure the model's existence.
            chinese_name(`str`, *optional*, defaults to `None`):
                chinese name of the new created model.
            commit_message(`str`, *optional*, defaults to `None`):
                commit message of the push request.
            tag(`str`, *optional*, defaults to `None`):
                The tag on this commit
            revision (`str`, *optional*, default to DEFAULT_MODEL_REVISION):
                which branch to push. If the branch is not exists, It will create a new
                branch and push to it.
            original_model_id (str, optional): The base model id which this model is trained from
            ignore_file_pattern (`Union[List[str], str]`, optional): The file pattern to ignore uploading
            lfs_suffix (`List[str]`, optional): File types to use LFS to manage. examples: '*.safetensors'.

        Raises:
            InvalidParameter: Parameter invalid.
            NotLoginException: Not login
            ValueError: No configuration.json
            Exception: Create failed.
        Nzmodel_id cannot be empty!zmodel_dir cannot be empty!z$model_dir must be a valid directory.zNo z file found in z, creating a default one.zMust login before upload!z5Visibility and License cannot be empty for new model.zCreating new model [%s])rn   ro   rp   rq   rr   zPushing folder z
 as model r’   zTotal folder size z;, this may take a while depending on actual pushing size...)r¡   Z
clone_fromzCreating new branch %sr   T)Úignore_errorsc                    s   g | ]
}t  |ˆ ¡d u‘qS r™   )ÚreÚsearch)Ú.0Úpattern©Úfr]   r^   Ú
<listcomp>   s    z%HubApi.push_model.<locals>.<listcomp>z%Y_%m_%d_%H_%M_%Sz$[automsg] push model %s to hub at %s)r¥   Zlocal_branchZremote_branch)+r!   r}   rh   ÚexistsÚisfilerš   r;   r›   r‹   ÚwarningrD   r¤   rP   r{   r#   ÚlistdirrA   Ú
isinstanceÚstrr˜   Úinforƒ   ÚtempfileÚmkdtempr,   r-   Zget_remote_branchesZ
new_branchZcheckoutÚremoveÚshutilÚrmtreeÚanyÚisdirÚcopytreeÚcopyÚdatetimeÚnowÚstrftimeZadd_lfs_typeÚpushZtag_and_pushr”   )rY   rn   r¡   ro   rp   rq   r¥   r¦   rŠ   rr   r§   r¨   r¢   rd   Zfiles_to_saveZfolder_sizeÚtmp_dirZgit_wrapperÚrepoÚbranchesZfiles_in_repoÚsrcÚdateZlfs_suffix_listÚsuffixr]   r®   r^   Ú
push_model1  s¤   ;ÿ



û

€€ÿ
ý
€ÿ€"zHubApi.push_modelr?   é
   r€   Úpage_numberÚ	page_sizec                 C   sŒ   t  ¡ }| j› d�}| jj|d|||f ||  | j¡d�}t|t||ƒ |j	t
jkr@t| ¡ ƒr8| ¡ t }|S t| ¡ t ƒ‚t|ƒ dS )a£  List models in owner or group.

        Args:
            owner_or_group(str): owner or group.
            page_number(int, optional): The page number, default: 1
            page_size(int, optional): The page size, default: 10

        Raises:
            RequestError: The request error.

        Returns:
            dict: {"models": "list of models", "TotalCount": total_number_of_models_in_owner_or_group}
        r„   z.{"Path":"%s", "PageNumber":%s, "PageSize": %s})Údatard   rR   N)rP   r{   rE   rS   Úputrc   rR   r(   r‹   rŒ   r   r�   r)   ra   r   r%   r   r*   )rY   r€   rÍ   rÎ   rd   rh   ri   rÏ   r]   r]   r^   Úlist_modelsº  s$   ÿ
ûzHubApi.list_modelsFÚuse_cookiesc                 C   s6   d }t |tƒr|}|S |rt ¡ }|d u rtdƒ‚|S )Nrt   )rµ   r   rP   r{   r|   )rY   rÒ   rd   r]   r]   r^   Ú_check_cookieÞ  s   
üzHubApi._check_cookieÚcutoff_timestampc                 C   s.   | j |||d�}|rdd„ |D ƒ}|S g }|S )á3  Get model branch and tags.

        Args:
            model_id (str): The model id
            cutoff_timestamp (int): Tags created before the cutoff will be included.
                                    The timestamp is represented by the seconds elapsed from the epoch time.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        )rn   rÔ   rÒ   c                 S   ó   g | ]}|d  ‘qS ©ÚRevisionr]   ©r¬   Úxr]   r]   r^   r°   þ  ó    z/HubApi.list_model_revisions.<locals>.<listcomp>)Úlist_model_revisions_detail)rY   rn   rÔ   rÒ   Ztags_detailsÚtagsr]   r]   r^   Úlist_model_revisionsê  s   þÿÿÿzHubApi.list_model_revisionsc           	      C   sz   |   |¡}|du rtƒ }| j› d|› d�| }| jj|||  | j¡d�}t|t||ƒ | 	¡ }t
|ƒ |t }|d d S )rÕ   Nr„   z/revisions?EndTime=%sr…   ÚRevisionMapÚTags)rÓ   rB   rE   rS   r   rc   rR   r(   r‹   ra   r+   r   )	rY   rn   rÔ   rÒ   rd   rh   ri   rj   r·   r]   r]   r^   rÜ     s   


ÿz"HubApi.list_model_revisions_detailc                 C   s"   |D ]}|d |kr|  S qd S )NrØ   r]   )rY   Údetailsr�   Úitemr]   r]   r^   Úget_branch_tag_detail   s
   ÿzHubApi.get_branch_tag_detailrd   c                    s8  t ƒ ‰ tttj ¡  ¡ ƒƒ}| j||d u rdn|d�\}}|r&dd„ |D ƒng }|r1dd„ |D ƒng }ˆ |t kro|d u rFt}t	 
d| ¡ ||vrV||vrVtd||f ƒ‚|  ||¡}	|	d u rf|  ||¡}	t	 
d| ¡ |	S |d ur†||v r†|  ||¡}	t	 d	| ¡ |	S t|ƒd
kr§|d u s”|tkr—t}ntd||f ƒ‚|  ||¡}	|	S |d u rè|r¶‡ fdd„|D ƒng }
t|
ƒd
krÉ|
d
 d }|
d
 }	nt}|  ||¡}	dd |¡ }t	 d| ¡ t	 d| ¡ |	S ||v�r|tkrýt	 d¡ |  ||¡}	ndd |¡ }td|||f ƒ‚|  ||¡}	t	 
d| ¡ |	S )NF©rÒ   c                 S   rÖ   r×   r]   rÙ   r]   r]   r^   r°   1  rÛ   z4HubApi.get_valid_revision_detail.<locals>.<listcomp>c                 S   rÖ   r×   r]   rÙ   r]   r]   r^   r°   2  rÛ   zAModel revision not specified, use default: %s in development modez$The model: %s has no revision : %s .z!Development mode use revision: %sz9Using branch: %s as version is unstable, use with cautionr   z#The model: %s has no revision: %s !c                    s   g | ]
}|d  ˆ kr|‘qS )Z	CreatedAtr]   rÙ   ©Úrelease_timestampr]   r^   r°   N  s    ÿrØ   z[%s]ú,z5Model revision should be specified from revisions: %sz.Model revision not specified, use revision: %sz?Using the master branch is fragile, please use it with caution!z0The model: %s has no revision: %s valid are: %s!z%Use user-specified model revision: %s)rB   ÚintÚroundrÁ   rÂ   Ú	timestampÚ#get_model_branches_and_tags_detailsr   r1   r‹   r·   r"   rã   r³   Úlenrš   )rY   rn   rŠ   rd   Úcurrent_timestampZall_branches_detailZall_tags_detailZall_branchesZall_tagsÚrevision_detailZrevisions_detailÚvlr]   rå   r^   Úget_valid_revision_detail&  sv   
ÿÿÿ'Ûçÿÿÿ

õ
ÿz HubApi.get_valid_revision_detailc                 C   s   | j |||d�d S )N)rn   rŠ   rd   rØ   )rð   )rY   rn   rŠ   rd   r]   r]   r^   Úget_valid_revisionh  s   þþzHubApi.get_valid_revisionc                 C   st   |   |¡}| j› d|› d�}| jj|||  | j¡d�}t|t||ƒ | ¡ }t	|ƒ |t
 }|d d |d d fS )áq  Get model branch and tags.

        Args:
            model_id (str): The model id
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.

        Returns:
            Tuple[List[str], List[str]]: Return list of branch name and tags
        r„   z
/revisionsr…   rß   ZBranchesrà   )rÓ   rE   rS   r   rc   rR   r(   r‹   ra   r+   r   )rY   rn   rÒ   rd   rh   ri   rj   r·   r]   r]   r^   rë   p  s   


ÿz*HubApi.get_model_branches_and_tags_detailsc                 C   sF   | j ||d�\}}|rdd„ |D ƒng }|rdd„ |D ƒng }||fS )rò   )rn   rÒ   c                 S   rÖ   r×   r]   rÙ   r]   r]   r^   r°   ›  rÛ   z6HubApi.get_model_branches_and_tags.<locals>.<listcomp>c                 S   rÖ   r×   r]   rÙ   r]   r]   r^   r°   �  rÛ   )rë   )rY   rn   rÒ   Zbranches_detailZtags_detailrÇ   rÝ   r]   r]   r^   Úget_model_branches_and_tagsŠ  s   
ÿÿÿÿÿz"HubApi.get_model_branches_and_tagsÚrootÚ	recursiverR   c                 C   sÚ   |rd| j |||f }nd| j ||f }|  |¡}|dur$|d|› � }|du r+| jn|}tt ¡ jƒ|d< | jj|||d�}	t	|	t
||ƒ |	 ¡ }
t|
ƒ g }|
t d D ]}|d d	ksd|d d
kreqV| |¡ qV|S )a`  List the models files.

        Args:
            model_id (str): The model id
            revision (Optional[str], optional): The branch or tag name.
            root (Optional[str], optional): The root path. Defaults to None.
            recursive (Optional[str], optional): Is recursive list files. Defaults to False.
            use_cookies (Union[bool, CookieJar], optional): If is cookieJar, we will use this cookie, if True,
                        will load cookie from local. Defaults to False.
            headers: request headers

        Returns:
            List[dict]: Model file list.
        z7%s/api/v1/models/%s/repo/files?Revision=%s&Recursive=%sz+%s/api/v1/models/%s/repo/files?Recursive=%sNz&Root=zX-Request-IDr…   ÚFilesrv   z
.gitignorez.gitattributes)rE   rÓ   rR   r¶   ÚuuidÚuuid4ÚhexrS   r   r(   r‹   ra   r+   r   Úappend)rY   rn   rŠ   rô   rõ   rÒ   rR   rh   rd   ri   rj   ÚfilesÚfiler]   r]   r^   Úget_model_files¡  s0   
ÿÿ
ÿzHubApi.get_model_files©rŠ   Úfilenamec                C   s$   | j ||d�}dd„ |D ƒ}||v S )a  Get if the specified file exists

        Args:
            repo_id (`str`): The repo id to use
            filename (`str`): The queried filename
            revision (`Optional[str]`): The repo revision
        Returns:
            The query result in bool value
        rþ   c                 S   rÖ   ©rv   r]   )r¬   rü   r]   r]   r^   r°   á  rÛ   z&HubApi.file_exists.<locals>.<listcomp>)rý   )rY   r�   rÿ   rŠ   rû   r]   r]   r^   Úfile_existsÐ  s   zHubApi.file_existsÚdataset_nameÚ	namespaceÚdescriptionc                 C   sÄ   |d u s|d u rt dƒ‚t ¡ }|d u rtdƒ‚| j› d�}d |fd |fd |fd |fd |fd |fdœ}	| jj||	||  | j¡d�}
t	|
||	ƒ t
|
 ¡ ƒ | j› d|› d|› �}t d|› �¡ |S )	Nz(dataset_name and namespace are required!rt   ú/api/v1/datasets)rv   rw   ZOwnerry   rx   ÚDescription)rû   rd   rR   z
/datasets/rz   zCreate dataset success: )r!   rP   r{   r|   rE   rS   rb   rc   rR   r'   r+   ra   r‹   r·   )rY   r  r  rq   rp   ro   r  rd   rh   rû   ri   Zdataset_repo_urlr]   r]   r^   Úcreate_datasetä  s0   ú	
üzHubApi.create_datasetc                 C   sL   | j › d�}i }| jj|||  | j¡d�}t|ƒ | ¡ t }dd„ |D ƒS )Nr  )ÚparamsrR   c                 S   rÖ   r   r]   rÙ   r]   r]   r^   r°     rÛ   z(HubApi.list_datasets.<locals>.<listcomp>)rE   rS   r   rc   rR   r*   ra   r   )rY   rh   r  ri   Zdataset_listr]   r]   r^   Úlist_datasets
  s   

ÿzHubApi.list_datasetsc           	      C   sb   | j › d|› d|› �}t ¡ }| jj||d�}| ¡ }t|||ƒ |d d }|d d }||fS )z Get the dataset id and type. ú/api/v1/datasets/rz   ©rd   ÚDataÚIdÚType©rE   rP   r{   rS   r   ra   r&   )	rY   r  r  Údatahub_urlrd   ri   ÚrespÚ
dataset_idÚdataset_typer]   r]   r^   Úget_dataset_id_and_type  s   zHubApi.get_dataset_id_and_typeéd   ÚTrueÚdataset_hub_idÚfiles_metadatarO   c                 C   s^   | j › d|› d�}|d|dœ}t ¡ }|rd|d< | jj||||d�}	|	 ¡ }
t||
|	ƒ |
S )z$
        Get dataset infos.
        r
  ú
/repo/treeN)rØ   ÚRootÚ	RecursiveTÚblobs)r  rd   rO   r  )rY   r  rŠ   r  rO   rõ   r  r  rd   ri   r  r]   r]   r^   Úget_dataset_infos  s   	zHubApi.get_dataset_infosTÚ	root_pathc                 C   s‚   | j ||d�\}}	|rdnd}| j› d|› d�}
|r|nd|r!|nd|||dœ}t ¡ }| jj|
||d	�}| ¡ }t|
||ƒ |S )
N)r  r  r  ÚFalser
  r  Úmasterrz   )rØ   r  r  Ú
PageNumberZPageSize)r  rd   )r  rE   rP   r{   rS   r   ra   r&   )rY   r  r  rŠ   r  rõ   rÍ   rÎ   r  r  r  r  rd   ri   r  r]   r]   r^   Úlist_repo_tree2  s   	
ÿ
þzHubApi.list_repo_treer  c           
      C   s„   | j › d|› d|› �}t ¡ }| jj|||  | j¡d�}| ¡ }t|||ƒ |d }	|	du r<t	d|› d|› d|› d	�ƒ‚|	d
 }	|	S )z( Get the meta file-list of the dataset. r
  z/repo/tree?Revision=r…   r  Nz'The modelscope dataset [dataset_name = z, namespace = z, version = z] dose not existrö   )
rE   rP   r{   rS   r   rc   rR   ra   r&   r"   )
rY   r  r  r  rŠ   r  rd   ri   r  Ú	file_listr]   r]   r^   Úget_dataset_meta_file_listK  s"   
þÿÿz!HubApi.get_dataset_meta_file_listr  Úmeta_cache_dirc                 C   sX   t j |t| ƒ› tjj› �¡}t|dƒ�}| d¡ W d  ƒ dS 1 s%w   Y  dS )zè
        Dump the data_type as a local file, in order to get the dataset
         formation without calling the datahub.
        More details, please refer to the class
        `modelscope.utils.constant.DatasetFormations`.
        Úwz3*** Automatically-generated file, do not modify ***N)	r}   rh   rš   r¶   r5   Úformation_mark_extÚvalueÚopenÚwrite)r  r%  Zdataset_type_file_pathÚfpr]   r]   r^   Údump_datatype_file]  s   ÿ"ÿzHubApi.dump_datatype_filer#  c              
   C   s   t tƒ}t|ƒ}t| }	t ¡ }
tj||d� |D ]r}|d }tj	 
|¡d }||	v r‹| j› d|› d|› d|› d|› �	}| jj||
d�}t|ƒ tj	 ||¡}tj	 |¡rit d	|› d
|› d�¡ ||  |¡ qt|dƒ�}| |j¡ W d   ƒ n1 sw   Y  ||  |¡ q||fS )N)r  r%  ru   éÿÿÿÿr
  rz   ú/repo?Revision=ú
&FilePath=r  zReusing dataset z's python file (ú)Úwb)r   Úlistr5   r6   rP   r{   rD   r,  r}   rh   ÚsplitextrE   rS   r   r*   rš   r±   r‹   r³   rú   r)  r*  Úcontent)rY   r  r  rŠ   r%  r  r#  Zlocal_pathsZdataset_formationZdataset_meta_formatrd   Ú	file_infoÚ	file_pathÚ	extensionr  ri   Ú
local_pathr¯   r]   r]   r^   Ú"get_dataset_meta_files_local_pathsj  s:   ÿÿÿÿ€z)HubApi.get_dataset_meta_files_local_pathsi   c                    s”  ddl }ddlm} ddl}tj || | jdd�¡ 	¡ ¡}|t
jkr.tj |¡r.t |¡ tj |¡r>t d|› �¡ |S t ¡ }t d¡ tj| |dd	�}t|j d
d¡ƒ}	||	dd�}
‡ fdd„}d}t|dƒ�Q}||ƒD ]D}|
 t|ƒ¡ |  d¡r¨dd„ |D ƒ}t|ƒdkrŒqp|dkr“d}nd}| |¡}|j|d|dd� |d7 }qp|D ]	}| |d ¡ qªqpW d  ƒ n1 s¿w   Y  |
 ¡  |S )zO
        Fetch the meta-data files from the url, e.g. csv/jsonl files.
        r   N)ÚtqdmzUTF-8©ÚencodingzReusing cached meta-data file: zLoading meta-data file ...T)rd   Ústreamzcontent-length)rH   Údynamic_ncolsc                 3   sH   � g }|   ¡ D ]}| d¡}| |¡ t|ƒˆ kr|V  g }q|V  d S )Núutf-8)Ú
iter_linesÚdecoderú   rì   )r  Z
chunk_datarÏ   ©Ú
chunk_sizer]   r^   Ú	get_chunk¢  s   €

€
z3HubApi.fetch_meta_files_from_url.<locals>.get_chunkÚaÚjsonlc                 S   s   g | ]}|  ¡ rt |¡‘qS r]   )Ústripra   Úloads)r¬   Úliner]   r]   r^   r°   ±  s    z4HubApi.fetch_meta_files_from_url.<locals>.<listcomp>Fú\)ÚindexÚheaderÚ
escapecharr?   Ú
)ÚhashlibÚ	tqdm.autor:  Úpandasr}   rh   rš   Úmd5ÚencodeÚ	hexdigestr9   ÚFORCE_REDOWNLOADr±   rº   r‹   r·   rP   r{   Úrequestsr   rè   rR   r)  Úupdaterì   ÚendswithÚ	DataFrameÚto_csvr*  Úclose)ÚurlZout_pathrC  ÚmoderO  r:  Úpdrd   ÚresponseÚ
total_sizeÚprogressrD  Ziter_numr¯   ÚchunkZwith_headerZchunk_dfrI  r]   rB  r^   Úfetch_meta_files_from_url‹  sH    





ÿñÿz HubApi.fetch_meta_files_from_urlÚ	file_nameÚviewÚextension_filterc           	      C   sJ   |r|r|s
t dƒ‚d|||dœ}t|ƒ}| j› d|› d|› d|› �}|S )Nz:Args (file_name, dataset_name, namespace) cannot be empty!ZSDK)ÚSourcerØ   ÚFilePathZViewr
  rz   ú/repo?)r|   r   rE   )	rY   rd  r  r  rŠ   re  rf  r  Zfile_urlr]   r]   r^   Úget_dataset_file_urlÃ  s   	zHubApi.get_dataset_file_urlc              	   C   s>   |rt j |¡d tv r| j› d|› d|› d|› d|› �	}|S )Nr-  r
  rz   r.  r/  )r}   rh   r3  r2   rE   )rY   rd  r  r  rŠ   r]   r]   r^   Úget_dataset_file_url_originà  s   ÿÿz"HubApi.get_dataset_file_url_originc                 C   s&   | j › d|› d|› d|› �}|  |¡S )Nr
  rz   ú/ststoken?Revision=)rE   Údatahub_remote_call)rY   r  r  rŠ   r  r]   r]   r^   Úget_dataset_access_configë  s   ÿ
z HubApi.get_dataset_access_configÚcheck_cookiec           	      C   sh   | j › d|› d|› d|› �}|r| jdd�}nt ¡ }| jj|||  | j¡d�}| ¡ }t	|ƒ |d S )Nr
  rz   rl  Trä   ©r\  rd   rR   r  )
rE   rÓ   rP   r{   rS   r   rc   rR   ra   r+   )	rY   r  r  ro  rŠ   r  rd   ri   r  r]   r]   r^   Ú!get_dataset_access_config_sessionô  s   ÿ
ýz(HubApi.get_dataset_access_config_sessionÚversionc           
      C   sœ   t j tjd¡}|stdtj› �ƒ‚|› d�}tj t	 
¡ ¡}t||d�}t|d�}| jj||||  | j¡dd�}| ¡ }	|	d d	krJtd
|	› �ƒ‚|	d S )z.
        Get virgo dataset meta info.
        rm   z"Virgo endpoint is not set in env: z/data/set/download)Z	dataSetIdZdataSetVersion)rÏ   i„  )r\  ra   rd   rR   rO   r—   r   zFailed to get virgo dataset: rÏ   )r}   r~   r   r=   Úenv_virgo_endpointÚRuntimeErrorrV  r   Údict_from_cookiejarrP   r{   ÚdictrS   rb   rc   rR   ra   )
rY   r  rr  Zvirgo_endpointZvirgo_dataset_urlrd   Údataset_inforÏ   ri   r  r]   r]   r^   Úget_virgo_meta
  s,   
þÿ
üzHubApi.get_virgo_metaÚzip_file_namec                 C   sÊ   | j › d|› d|› �}t ¡ }| jj|||  | j¡d�}| ¡ }t|ƒ |d }	t	 |	d ¡}
|› d|› �}| jj|||  | j¡d�}| ¡ }t|ƒ |d }|
d d | d | d | }||d	< |S )
Nr
  rz   rp  r  rx   rl  z	-unzippedÚ_ZDir)
rE   rP   r{   rS   r   rc   rR   ra   r+   r7   )rY   r  r  rŠ   ry  r  rd   ri   r  rÏ   ro   Zdatahub_sts_urlZr_stsZresp_stsZdata_stsZfile_dirr]   r]   r^   Ú&get_dataset_access_config_for_unzipped'  s&   

ÿ

ÿ z-HubApi.get_dataset_access_config_for_unzippedc           
      C   sd   | j › d|› d|› d|› d|› d|› d|› �}t ¡ }| jj||dd�}	|	 ¡ }	t|	ƒ |	d	 }	|	S )
Nr
  rz   z/oss/tree/?MaxLimit=ú
&Revision=z&Recursive=z&FilterDir=i  )r\  rd   rO   r  )rE   rP   r{   rS   r   ra   r+   )
rY   r  r  Z	max_limitZis_recursiveZis_filter_dirrŠ   r\  rd   r  r]   r]   r^   Úlist_oss_dataset_objects@  s   ÿÿÿÿzHubApi.list_oss_dataset_objectsÚobject_namec              	   C   ón   |r|r|r|st dƒ‚| j› d|› d|› d|› d|› �	}t ¡ }| jj||d�}| ¡ }t|ƒ |d }|S )NúArgs cannot be empty!r
  rz   z
/oss?Path=r|  ©r\  rd   ÚMessage©r|   rE   rP   r{   rS   r†   ra   r+   ©rY   r~  r  r  rŠ   r\  rd   r  r]   r]   r^   Údelete_oss_dataset_objectL  s   "z HubApi.delete_oss_dataset_objectc              	   C   r  )Nr€  r
  rz   z/oss/prefix?Prefix=z/&Revision=r�  r‚  rƒ  r„  r]   r]   r^   Údelete_oss_dataset_dirZ  s   ÿzHubApi.delete_oss_dataset_dirc                 C   s>   t  ¡ }| jj||dt  ¡ id�}| ¡ }t|||ƒ |d S )NrF   r…   r  )rP   r{   rS   r   rQ   ra   r&   )rY   r\  rd   ri   r  r]   r]   r^   rm  i  s   
ýzHubApi.datahub_remote_callÚuse_streamingc              
   C   s(  t  d¡dk}|rŒ|rŽ|s�|s’zdt ¡ }| j› d|› d|› d�}| jj|||  | j¡d�}t	|ƒ t
jj}d}	tt jv rAt jt }tt jv rKt jt }	| j› d|› d|› d|› d	|	› �	}
| jj|
||  | j¡d�}| ¡ }t|ƒ W d S  ty‹ } zt |¡ W Y d }~d S d }~ww d S d S d S d S )
NÚCI_TESTr  r
  rz   z/download/increaser…   rm   z/download/uv/z?user=)r}   ÚgetenvrP   r{   rE   rS   rb   rc   rR   r*   r8   ÚLOCALr(  r   r~   r   ra   r+   r”   r‹   Úerror)rY   r  r  r‡  Z
is_ci_testrd   Zdownload_count_urlZdownload_count_respÚchannelÚ	user_nameZdownload_uv_urlZdownload_uv_respÚer]   r]   r^   Údataset_download_statisticss  s<   

ÿ



ÿÿ

ÿ€ÿèz"HubApi.dataset_download_statisticsc                 C   s   t tt ¡ jƒi|¥S r™   )r   r¶   r÷   rø   rù   )rY   rR   r]   r]   r^   rc   �  s   ÿzHubApi.builder_headersc                 C   s   | j › d|› d|› d�S )Nr
  rz   ri  rˆ   )rY   r  r  r]   r]   r^   Úget_file_base_path”  s   zHubApi.get_file_base_path)r?   rÌ   )F)NF©NN)Fr  r  )Tr?   r  )r?   )FÚ__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r¶   r_   rl   r    ZPUBLICr   Z	APACHE_V2rè   rƒ   r‡   r‰   r/   rŽ   Úboolr˜   Ústaticmethodr¤   r0   r
   r   rË   rv  rÑ   r   rÓ   rÞ   rÜ   rã   rð   rñ   r	   rë   ró   rý   r  r   r  r	  r  Úfloatr  r"  r$  r,  r2  r9  r9   ÚREUSE_DATASET_IF_EXISTSrc  r.   rj  rk  rn  rq  rx  r{  r}  r…  r†  rm  r�  rc   r�  r]   r]   r]   r^   rD   A   s†   ý
ÿ
þ)ûÿþýüû
û4ýþý
ü*üþü
û*
õÿþýüûúùø	÷
ö
õ ýÿþý
ý&þÿÿ
þüþý
ü
üüþý
ü
üýÿ
ýDýÿ
ýýþ
ý
üýþ
ý
üúÿþýü
ûú
ú4ûþý
ûúÿþýüûú
ú&	ûÿþýü
ûùÿþýüûú
ùÿþýý
ý!<ùþýüûú
ù"ûþýü
ûüþý
üûþýü
ûÿþý
ü
ÿÿ
ÿ
ÿÿ
ÿ
rD   c                   @   sØ   e Zd ZeeƒZdZdZdZdZ	dZ
edd„ ƒZedefdd	„ƒZed
d„ ƒZedd„ ƒZedefdd„ƒZededefdd„ƒZedeeef fdd„ƒZedee fdd„ƒZeddeeedf defdd„ƒZdS )rP   rd   Z	git_tokenÚuserrS   Fc                   C   s   t jtjdd� d S )NT)Úexist_ok)r}   ÚmakedirsrP   Úpath_credentialr]   r]   r]   r^   Úmake_sure_credential_path_exist¡  s   z0ModelScopeConfig.make_sure_credential_path_existc                 C   sR   t  ¡  ttj t jt j¡dƒ�}t 	| |¡ W d   ƒ d S 1 s"w   Y  d S )Nzwb+)
rP   rž  r)  r}   rh   rš   r�  ÚCOOKIES_FILE_NAMEÚpickler    )rd   r¯   r]   r]   r^   rf   ¥  s   
ÿþ"ýzModelScopeConfig.save_cookiesc                  C   s”   t j tjtj¡} t j | ¡rHt| dƒ�,}t 	|¡}|D ]}| 
¡ r6tjs6dt_t d¡  W d   ƒ d S q|W  d   ƒ S 1 sCw   Y  d S )NÚrbTzŒAuthentication has expired, please re-login with modelscope login --token "YOUR_SDK_TOKEN" if you need to access private models or datasets.)r}   rh   rš   rP   r�  rŸ  r±   r)  r   ÚloadÚ
is_expiredÚcookie_expired_warningr‹   r³   )Zcookies_pathr¯   rd   Úcookier]   r]   r^   r{   ­  s$   
ÿ
ÿ÷€
 özModelScopeConfig.get_cookiesc                  C   sÈ   t j tjtj¡} d}t j | ¡r2t| dƒ�}t| 	¡  
¡ dd�}|W  d   ƒ S 1 s-w   Y  |dks<t|ƒdkrbtt ¡ jƒ}t ¡  t| dƒ�}| |¡ W d   ƒ |S 1 s]w   Y  |S )Nrm   r¡  r?  r;  é    úw+)r}   rh   rš   rP   r�  ÚUSER_SESSION_ID_FILE_NAMEr±   r)  r¶   ÚreadlinerG  rì   r÷   rø   rù   rž  r*  )Zsession_pathZ
session_idr¯   Zwfr]   r]   r^   Úget_user_session_id¿  s$   
ÿ þ
ÿýz$ModelScopeConfig.get_user_session_idrk   c                 C   sP   t  ¡  ttj t jt j¡dƒ�}| | ¡ W d   ƒ d S 1 s!w   Y  d S )Nr§  )	rP   rž  r)  r}   rh   rš   r�  ÚGIT_TOKEN_FILE_NAMEr*  ©rk   r¯   r]   r]   r^   re   Ð  s   
ÿþ"ýzModelScopeConfig.save_tokenr�  Ú
user_emailc                 C   sX   t  ¡  ttj t jt j¡dƒ�}| d| |f ¡ W d   ƒ d S 1 s%w   Y  d S )Nr§  z%s:%s)	rP   rž  r)  r}   rh   rš   r�  ÚUSER_INFO_FILE_NAMEr*  )r�  r­  r¯   r]   r]   r^   rg   Ø  s   
ÿþ"ýzModelScopeConfig.save_user_infors   c                  C   s~   z4t tj tjtj¡ddd��} |  ¡ }| d¡d | d¡d fW  d   ƒ W S 1 s-w   Y  W dS  t	y>   Y dS w )Nri   r?  r;  ú:r   r?   r‘  )
r)  r}   rh   rš   rP   r�  r®  rI   ÚsplitÚFileNotFoundError)r¯   r·   r]   r]   r^   Úget_user_infoà  s"   
ÿü$ú	þþzModelScopeConfig.get_user_infoc                  C   sh   d} z't tj tjtj¡ddd��}| ¡ } W d  ƒ W | S 1 s"w   Y  W | S  ty3   Y | S w )z‹
        Get token or None if not existent.

        Returns:
            `str` or `None`: The token, `None` if it doesn't exist.

        Nri   r?  r;  )	r)  r}   rh   rš   rP   r�  r«  rI   r±  r¬  r]   r]   r^   Ú	get_tokenî  s&   	
ÿü
ûøþþzModelScopeConfig.get_tokenNÚ
user_agentc                 C   sª   d}t tjv rtjt  }d}ttjv rtjt }ddlm} d|t ¡ t 	¡ t ¡ t 
¡ ||f }t| tƒrH|dd dd„ |  ¡ D ƒ¡ 7 }|S t| tƒrS|d|  7 }|S )	a  Formats a user-agent string with basic info about a request.

        Args:
            user_agent (`str`, `dict`, *optional*):
                The user agent info in the form of a dictionary or a single string.

        Returns:
            The formatted user-agent string.
        ÚcustomÚunknownr   )Ú__version__zSmodelscope/%s; python/%s; session_id/%s; platform/%s; processor/%s; env/%s; user/%sz; c                 s   s"   � | ]\}}|› d |› �V  qdS )rz   Nr]   )r¬   ÚkÚvr]   r]   r^   Ú	<genexpr>#  s   €  z2ModelScopeConfig.get_user_agent.<locals>.<genexpr>)r   r}   r~   r   Ú
modelscoper·  ÚplatformÚpython_versionrP   rª  Ú	processorrµ   rv  rš   Úitemsr¶   )r´  Úenvr�  r·  Úuar]   r]   r^   rQ     s,   



ù
	 
þzModelScopeConfig.get_user_agentr™   )r’  r“  r”  r   r   r�  rŸ  r«  r®  r¨  r¤  r—  rž  r   rf   r{   rª  r¶   re   rg   r	   r²  r   r³  r
   r   rQ   r]   r]   r]   r^   rP   ™  s2    


$rP   )_rÁ   rV   r}   r   r¼  rª   r»   r¸   r÷   Úcollectionsr   Úhttpr   Úhttp.cookiejarr   Úos.pathr   Útypingr   r   r   r	   r
   Úurllib.parser   ra   rV  r   Zrequests.adaptersr   r   r»  r   Zmodelscope.fileior   Zmodelscope.hub.constantsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    Úmodelscope.hub.errorsr!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   Zmodelscope.hub.gitr,   Zmodelscope.hub.repositoryr-   Úmodelscope.utils.constantr.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   Úmodelscope.utils.loggerr>   Zutils.utilsr@   rA   rB   rC   r‹   rD   rP   r]   r]   r]   r^   Ú<module>   sN   H4H	        `