o
    1ôjøu  ã                   @   s  d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlZd dl	m	Z	 d dl
mZ d dlmZ d dlmZ d dlZd dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZ d dlm Z m!Z! d d	l"m#Z# d d
l$m%Z% ee&ƒZ'dZ(e' )¡ j*d Z+e#ƒ Z,dZ-ddgZ.g d¢Z/dZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8dZ9dZ:dZ;dZ<dZ=dZ>d Z?d!Z@d"ZAd#ZBd$d%„ ZCeCƒ ZDG d&d'„ d'eEƒZFG d(d)„ d)eEƒZGeGƒ ZHd*eIdeeJef d+dfd,d-„ZKd?d/d0„ZLd@d1d2„ZMd3d4„ ZNd5d6„ ZOdd.e,e0fd7d8„ZPdAd9d:„ZQdBd;d<„ZReSd=k�rePdd>�ZTeUeTƒ dS dS )Cé    N)Údatetime)Úreduce)ÚPath)ÚUnion)Úversion)ÚCustomDatasetsÚHeadsÚHooksÚLR_SchedulersÚMetricsÚModelsÚ
OptimizersÚ	PipelinesÚPreprocessorsÚ
TaskModelsÚTrainers)ÚFieldsÚTasks)Úget_modelscope_cache_dir)Údefault_groupTé   Úregister_moduleÚ
modelscopeÚ.)ÚmodelsÚmetricsÚ	pipelinesÚpreprocessorsÚtrainersÚ
msdatasetsÚ	exportersZast_indexerZ
decoratorsÚexpressZfrom_importsÚimportsÚfilepathZmodelscope_pathr   Úmd5ÚindexÚfiles_mtimeÚrequirementsÚmoduleÚ
class_nameZ	group_keyÚmodule_nameÚ
module_clsÚTEMPLATE_PATHzast_index_file.pyc                  C   s   t  d¡} |  t j¡ | S )Nzmodelscope.ast)ÚloggingÚ	getLoggerÚsetLevelÚINFO)Z
ast_logger© r1   úm/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/modelscope/utils/ast_utils.pyÚget_ast_logger;   s   
r3   c                	   @   s&  e Zd Zd)dd„Zdedefdd„Zdejdefdd	„Z	de
ejd
f defdd„Zd*dejdedefdd„Zd*dejdedefdd„Zdd„ Zde
ejdef fdd„Z		d+de
ejdef dededefdd„Zdejdefdd„Zdd„ Zd ed!edefd"d#„Zd$edefd%d&„Zd'd(„ ZdS ),ÚAstScanningÚreturnNc                 C   ó    t ƒ | _t ƒ | _g | _g | _d S ©N)ÚdictÚresult_importÚresult_from_importÚresult_decoratorr!   ©Úselfr1   r1   r2   Ú__init__F   ó   
zAstScanning.__init__Únodec                 C   s   t |tjƒot |tjƒ S r7   )Ú
isinstanceÚastÚASTÚexpr_context©r=   r@   r1   r1   r2   Ú_is_sub_nodeL   s
   ÿÿzAstScanning._is_sub_nodec                 C   sT   |j D ]$}t||ƒ}|  |¡r dS t|ttfƒr'|D ]}|  |¡r&  dS qqdS )NFT)Ú_fieldsÚgetattrrF   rA   ÚlistÚtuple)r=   r@   ÚfieldÚattrÚvalr1   r1   r2   Ú_is_leafP   s   



ÿ€zAstScanning._is_leafÚstrc                 C   s"   t rt|ƒjdks|dkrdS dS )NÚFunctionDefTF)ÚSKIP_FUNCTION_SCANNINGÚtypeÚ__name__rE   r1   r1   r2   Ú_skip_function\   s   zAstScanning._skip_functionTÚnÚshow_offsetsc                 C   s   |r|j |j S |jS r7   )Ú_attributesrG   )r=   rU   rV   r1   r1   r2   rG   b   s   zAstScanning._fieldsc                 C   s^   t ƒ }t|tjƒr-t ƒ }| j||d�D ]}| jt||ƒ|d�}|||< q||t|ƒj< |S |S )N©rV   )	r8   rA   rB   rC   rG   Ú_leafrH   rR   rS   )r=   r@   rV   ÚoutputZ
local_dictrK   Zfield_outputr1   r1   r2   rY   h   s   
ÿ
zAstScanning._leafc                 C   r6   r7   )r8   r9   r:   r;   Úresult_expressr<   r1   r1   r2   Ú_refreshu   r?   zAstScanning._refreshc                 C   s   |   ¡  | j|ddd� d S )Nz  F)ÚindentrV   )Z_setup_globalÚscan_importrE   r1   r1   r2   Úscan_ast{   s   zAstScanning.scan_astÚ Úparent_node_namec                    sš  |d u r|S ˆ   |¡rˆ j|ˆd�S 	ddttjd tf dtdtf‡ ‡fdd„}tƒ }t|ƒjdkrat	|d	ƒ}|d
krad 
dg| ¡}t|d	dƒ t	|dƒ}|d u rYt|d|ƒ nt|d|| ƒ ˆ j|ˆd�D �]U}	t	||	ƒ}
|
g kryg ||	< n|ˆ  |¡rqht|
tƒr¤t|
ƒd
kr¤t|
d tjƒr¤ˆ   |
d ¡r¤||
d ƒ}|||	< nQt|
tƒrâtƒ }|
D ].}||t|ƒjƒ}t|ƒj}|dksÍ|dksÍ|dksÍ|dkrÜ||vrÕg ||< ||  |¡ q®|||	< nt|
tjƒrñ||
ƒ}|||	< n|
||	< t|ƒjdk�st|ƒjdk�r‰t|ƒjdk�rX|	dk�rtƒ ˆ j||	 < |	dk�rXt||	 d tƒ�rHg }||	 d D ]}|d d }| |¡ �q1|ˆ j|d < n||	 d d }|gˆ j|d < t|ƒjdk�r‰||	 d }t|tƒ�r~|D ]}|d ˆ j|d d < �qnn|ˆ j||	 d d < d|	k�r¦|
g k�r¦|
D ]
}t|t|jƒ �q•ˆ j |
¡ |
g k�r¾t|
ƒjdk�r¾|dk�r¾ˆ j |
¡ qhtˆ jtˆ jtˆ jtˆ jiS )NrX   r`   Úelra   r5   c                    s   ˆ j | ˆ|d�S )N)rV   ra   )r^   )rb   ra   ©r=   rV   r1   r2   Ú_scan_import‹   s
   ýz-AstScanning.scan_import.<locals>._scan_importÚ
ImportFromÚlevelr   r   r   r(   ÚImportÚnamesÚaliasÚnameÚdecorator_listÚCallÚExpr)r`   )rN   rY   r   rB   rC   rO   r8   rR   rS   rH   ÚjoinÚsetattrrG   rT   rA   rI   ÚlenÚappendr:   r9   Ú
CLASS_NAMErj   r;   Úextendr[   Ú
IMPORT_KEYÚFROM_IMPORT_KEYÚDECORATOR_KEYÚEXPRESS_KEY)r=   r@   rV   ra   rd   Úoutputsrf   Z
path_levelr*   rK   rL   Z	local_outZel_dictrb   rj   rZ   Ú	item_nameÚitemZ
local_nameZ
final_dictr1   rc   r2   r^      sÄ   
ÿÿÿ




ÿþ


€



ÿÿÿ
ÿÿÿÿÿÿþ
€üzAstScanning.scan_importc                    sj   dt jdtfdd„‰ dtdtf‡ fdd„}dt jdtf‡ fdd	„}ˆ |jƒ}||jƒ}||jƒ}|||fS )
Nr@   r5   c                 S   sV   d\}}}t | ƒjdkrt| dƒ}t|dƒ}t| dƒ}t | ƒjdkr't| dƒ}||fS )N)NNNÚ	AttributeÚvalueÚidrL   ÚName)rR   rS   rH   )r@   r|   r}   rL   r1   r1   r2   Ú_get_attribute_itemë   s   




z9AstScanning._parse_decorator.<locals>._get_attribute_itemÚnodesc                    s`   g }| D ])}t |ƒjdkr| |jd f¡ qt |ƒjdkr&| |jd f¡ q| ˆ |ƒ¡ q|S )NÚStrÚConstant)rR   rS   rq   Úsr|   )r€   Úresultr@   ©r   r1   r2   Ú_get_args_nameõ   s   z4AstScanning._parse_decorator.<locals>._get_args_namec                    s”   g }| D ]C}t |ƒjdkrGt|dƒ}t |ƒjdkr&| t|dƒ|jd f¡ qt |ƒjdkr:| t|dƒ|jd f¡ q| t|dƒfˆ |ƒ ¡ q|S )NÚkeywordr|   r�   Úargr‚   )rR   rS   rH   rq   rƒ   r|   )r€   r„   r@   Zattribute_noder…   r1   r2   Ú_get_keyword_name   s&   
ÿÿÿÿ€z7AstScanning._parse_decorator.<locals>._get_keyword_name)rB   rC   rJ   rI   ÚfuncÚargsÚkeywords)r=   r@   r†   r‰   Ú	functionsÚ	args_listÚkeyword_listr1   r…   r2   Ú_parse_decoratoré   s   




zAstScanning._parse_decoratorc                 C   sH   |d u rd S |dkrt S | d¡}t|ƒdkr|S tt|d ƒ|d ƒS )Nr   r   r   r   )r   Úsplitrp   rH   Úeval)r=   Zkey_itemZ
split_listr1   r1   r2   Ú_get_registry_value  s   
zAstScanning._get_registry_valueÚparsed_inputr)   c                 C   sŒ  |\}}}t |d krdS |d g}t|ƒdkr#t|ƒdkr#| t¡ t|ƒdkr4t|ƒdkr4| |¡ t|ƒdkrbt|ƒdkrbd}|D ]}|\}	}
}|	tkrX| |
|f¡ |}qD|durb| |¡ t|ƒdkrm| t¡ |D ]}|\}	}
}|	tkr}|
}qo| |
|f¡ qo|D ]-}|du r“| d¡ q‡|d du r¡| |d ¡ q‡t|tƒr¬| |¡ q‡| d 	|¡¡ q‡|d |  
|d ¡|  
|d ¡fS )z’format registry information to a tuple indexer

        Return:
            tuple: (MODELS, Tasks.text-classification, Models.structbert)
        r   Nr   r   é   )ÚREGISTER_MODULErp   rq   r   Ú	GROUP_KEYÚremoveÚ
MODULE_CLSrA   rO   rn   r“   )r=   r”   r)   r�   rŽ   r�   rZ   Zremove_group_itemrz   Úkeyrj   rL   r1   r1   r2   Ú_registry_indexer!  sH   




€



ÿzAstScanning._registry_indexerr€   c                 C   sr   g }|D ]2}t |ƒjdkrqt|tdƒ}t|dƒ}t|ddƒtkr"q|  |¡}|  ||¡}d|ur6| |¡ q|S )zÇparse the AST nodes of decorators object to registry indexer

        Args:
            nodes (list): list of AST decorator nodes

        Returns:
            list: list of registry indexer
        rl   NrŠ   rL   )rR   rS   rH   rr   r–   r�   r›   rq   )r=   r€   Úresultsr@   r)   rŠ   Zparse_outputr%   r1   r1   r2   Úparse_decoratorsU  s   	


€zAstScanning.parse_decoratorsc                 C   sš   |   ¡  t|ddd��}| ¡ }W d   ƒ n1 sw   Y  d |¡}t |¡}| j|dd�}|  |t ¡|t< |  |t	 ¡|t	< |t  
|t	 ¡ |S )NÚrÚutf8)Úencodingr`   FrX   )r\   ÚopenÚ	readlinesrn   rB   Úparser^   r�   rv   rw   rs   )r=   ÚfileÚcodeÚdatar@   rZ   r1   r1   r2   Úgenerate_astn  s   
ÿ

zAstScanning.generate_ast©r5   N)T)Tr`   )rS   Ú
__module__Ú__qualname__r>   ÚobjectÚboolrF   rB   rC   rN   r   rT   rJ   rG   rO   rY   r\   r_   r^   r�   r“   r›   rI   r�   r§   r1   r1   r1   r2   r4   D   s2    
üþýü
ûj,4r4   c                   @   sž   e Zd Zddd„Z	d dededefdd„Zd	d
„ Zdedefdd„Z	d!dd„Z
d"dd„Zdd„ Zdd„ Zdd„ Zdd„ Zdeefdd„Zeedfdd„ZdS )#ÚFilesAstScanningr5   Nc                 C   s   t ƒ | _g | _g | _d S r7   )r4   Ú	astScanerÚ	file_dirsÚrequirement_dirsr<   r1   r1   r2   r>   }  s   
zFilesAstScanning.__init__Úimport_packageÚcurrent_pathc           	      C   s°   |  td ¡rtd d | d¡dd… ¡ d S |  td ¡rV| d¡}| d¡}d}t|ƒD ]\}}|dkr<|} nq0|d| … }| ||d… ¡ dd |¡ d S |S )zŒ
        Args:
            import_package (str): relative import or abs import
            current_path (str): path/to/current/file
        r   ú/r   r   Nú.pyr`   )Ú
startswithÚIGNORED_PACKAGESÚMODELSCOPE_PATHrn   r‘   Ú	enumeraters   )	r=   r±   r²   Zcurrent_path_listZimport_package_listrf   r%   rz   Zabs_path_listr1   r1   r2   Ú_parse_import_path‚  s&   
ÿÿ

þz#FilesAstScanning._parse_import_pathc                 C   s   d S r7   r1   )r=   Zimport_abs_pathr1   r1   r2   Ú_traversal_importœ  s   z"FilesAstScanning._traversal_importÚscan_resultc                 C   s²   g }|  t|t  ¡ ƒ¡ |  t|t  ¡ ƒ¡ t|ƒD ]\}}d| d¡d kr.d||< q| d¡d ||< qtƒ }|D ]}tD ]}| 	|¡rM| 
|¡ qAq=tt|ƒt|ƒ ƒS )a  parse import and from import dicts to a third party package list

        Args:
            scan_result (dict): including the import and from import result

        Returns:
            list: a list of package ignored 'modelscope' and relative path import
        r`   r   r   )rs   rI   rt   Úkeysru   r¸   r‘   Úsetr¶   rµ   Úadd)r=   r»   rZ   r%   rz   ÚignoredZignored_packager1   r1   r2   Úparse_import¢  s   	


€þzFilesAstScanning.parse_importFc                 C   sb   g | _ |d u st|ƒdkr| j||d� d S |D ]}tj ||¡}tj |¡r.| j||d� qd S )Nr   ©Úinclude_init)r¯   rp   Ú_traversal_filesÚosÚpathrn   Úisdir)r=   rÅ   Zcheck_sub_dirrÂ   rz   Zsub_dirr1   r1   r2   Útraversal_files½  s   €ýz FilesAstScanning.traversal_filesc                 C   s¶   t  |¡}|D ]Q}|jdkr|sq|j d¡r|jdks(|j d¡s(|j d¡r)q| ¡ r6| j|j|d� q| ¡ rH|j d¡rH| j	 
|j¡ q| ¡ rXd|jv rX| j 
|j¡ qd S )Nz__init__.pyÚ__z.jsonz.mdrÁ   r´   Zrequirement)rÄ   Úscandirrj   rµ   ÚendswithÚis_dirrÃ   rÅ   Úis_filer¯   rq   r°   )r=   rÅ   rÂ   Zdir_listrz   r1   r1   r2   rÃ   Ç  s&   
ÿ
ÿ€ôz!FilesAstScanning._traversal_filesc                 C   s˜   z| j  |¡}W n8 ty@ } z,t |j¡}td|› d|d j› d|d j› d|d j› dt	|ƒj
› d|› d|› d	�ƒ‚d }~ww |  |¡}|t |fS )
NzDuring ast indexing the file z', a related error excepted in the file éÿÿÿÿz
 at line: z: "z" with error msg: "z: z'", please double check the origin file z- to see whether the file is correctly edited.)r®   r§   Ú	ExceptionÚ	tracebackÚ
extract_tbÚ__traceback__ÚfilenameÚlinenoÚlinerR   rS   rÀ   rv   )r=   r¤   rZ   ÚeÚdetailÚimport_listr1   r1   r2   Ú_get_single_file_scan_result×  s.   ÿþþýýýÿ€þ
	z-FilesAstScanning._get_single_file_scan_resultc              	   C   sJ   t ƒ }|D ]}|| t D ]}t|t|| t t|| t i||< qq|S r7   )r8   rv   ÚFILE_NAME_KEYrt   Ú
MODULE_KEY)r=   Úforward_indexÚinverted_indexr%   rz   r1   r1   r2   Ú_inverted_indexæ  s   
ýÿz FilesAstScanning._inverted_indexc                 C   s,   t ƒ }| ¡ D ]\}}|t ||t < q|S r7   )r8   Úitemsrt   rÚ   )r=   rÛ   Úmodule_importr%   Z
value_dictr1   r1   r2   Ú_module_importñ  s   zFilesAstScanning._module_importc                 C   s    d|v r|d= d|v r|d= |S )N)Z
OPTIMIZERSÚdefaultrj   )ZLR_SCHEDULERrá   rj   r1   )r=   rÜ   r1   r1   r2   Ú_ignore_useless_keys÷  s
   z%FilesAstScanning._ignore_useless_keysc                 C   sè   t   ¡ }|dur|| _n|  ||¡ t d|› d|› �¡ tƒ }| jD ]'}|| d¡d… }| tj	d¡ dd¡}|  
|¡\}	}
t|	t|
t|i||< q#|  |¡}|  |¡}|  |¡}t|t|i}t dt|ƒ› d	t   ¡ | › d
�¡ |S )aÎ  the entry method of the ast scan method

        Args:
            target_file_list can override the dir and folders combine
            target_dir (str, optional): the absolute path of the target directory to be scanned. Defaults to None.
            target_folder (list, optional): the list of
            sub-folders to be scanned in the target folder.
            Defaults to SCAN_SUB_FOLDERS.

        Returns:
            dict: indexer of registry
        NzAST-Scanning the path "z!" with the following sub folders r   r   r´   r`   zScanning done! A number of z. components indexed or updated! Time consumed rƒ   )Útimer¯   rÇ   ÚloggerÚinfor8   ÚrfindÚreplaceÚospÚseprØ   rv   rt   rÚ   rÝ   râ   rà   Ú	INDEX_KEYÚREQUIREMENT_KEYrp   )r=   Ztarget_file_listZ
target_dirZtarget_foldersÚstartr„   r¤   r#   r*   rk   r×   Zinverted_index_with_resultsrß   r%   r1   r1   r2   Úget_files_scan_resultsþ  s@   ÿ
ÿ
ý
ÿ
þ
ÿÿz'FilesAstScanning.get_files_scan_resultsc           
      C   s†   g | _ |rt|tƒr|| _ n|  ||¡ g }tƒ }| j D ]}tj |¡}| |¡ |||< qt	dd„ |dƒ}t
 | ¡ ¡}	|	 ¡ |fS )Nc                 S   s   t | ƒt |ƒ S r7   ©rO   )ÚxÚyr1   r1   r2   Ú<lambda>?  s    z2FilesAstScanning.files_mtime_md5.<locals>.<lambda>r`   )r¯   rA   rI   rÇ   r8   rÄ   rÅ   Úgetmtimerq   r   Úhashlibr$   ÚencodeÚ	hexdigest)
r=   Útarget_pathZtarget_subfolderÚ	file_listr&   Zfiles_mtime_dictrz   ÚmtimeZ
result_strr$   r1   r1   r2   Úfiles_mtime_md50  s   


z FilesAstScanning.files_mtime_md5r¨   r7   ©NF©F)rS   r©   rª   r>   rO   r¹   rº   r8   rI   rÀ   rÇ   rÃ   rØ   rÝ   rà   râ   r·   ÚSCAN_SUB_FOLDERSrí   rù   r1   r1   r1   r2   r­   {  s4    
þÿþ
þ



ý3ýr­   Úobjr5   c                 C   sd   t j |¡}|rt j |¡st j|dd� t|dƒ�}| | ¡ W d  ƒ dS 1 s+w   Y  dS )a  Write data to a given ``filepath`` with 'wb' mode.

    Note:
        ``write`` will create a directory if the directory of ``filepath``
        does not exist.

    Args:
        obj (bytes): Data to be written.
        filepath (str or Path): Path to write data.
    T)Úexist_okÚwbN)rÄ   rÅ   ÚdirnameÚexistsÚmakedirsr¡   Úwrite)rý   r#   r   Úfr1   r1   r2   Úensure_writeG  s   "ÿr  Fc                 C   sœ   dd„ | t   ¡ D ƒ| t < ddlm} || t< tj|d�\| t< | t< t	 
¡ | t< t | ¡}|r8| t	 
¡ t¡}t| ¡ |ƒ dd„ | t   ¡ D ƒ| t < d S )Nc                 S   s   i | ]	\}}t |ƒ|“qS r1   rî   ©Ú.0ÚkÚvr1   r1   r2   Ú
<dictcomp>\  s    z_save_index.<locals>.<dictcomp>r   ©Ú__version__©r÷   c                 S   ó   i | ]
\}}t  |¡|“qS r1   ©rB   Úliteral_evalr  r1   r1   r2   r
  g  ó    
ÿÿ)rê   rÞ   Úmodelscope.versionr  ÚVERSION_KEYÚfile_scannerrù   ÚMD5_KEYÚFILES_MTIME_KEYr·   Úas_posixÚMODELSCOPE_PATH_KEYÚjsonÚdumpsrç   r,   r  rô   )r%   Ú	file_pathr÷   Úwith_templater  Z
json_indexr1   r1   r2   Ú_save_indexZ  s    ÿ

ÿ
þr  c                 C   sr   t | dƒ�}| ¡ }W d   ƒ n1 sw   Y  |r%| ¡  tt ¡ ¡}t |¡}dd„ |t	  
¡ D ƒ|t	< |S )NÚrbc                 S   r  r1   r  r  r1   r1   r2   r
  u  r  z_load_index.<locals>.<dictcomp>)r¡   ÚreadÚdecoderç   r,   r·   r  r  Úloadsrê   rÞ   )r  r  r  Zbytes_indexÚwrapped_indexr1   r1   r2   Ú_load_indexm  s   
ÿ
ÿ


þr#  c                 C   s0  | t  }tt|ƒt|ƒ ƒ}tt|ƒt|ƒ ƒ}g }|D ]}||vr/|| || kr/| |¡ q| |¡ | |¡ t|ƒdkrg }g }| t D ]}	| t |	 t |v rd| |	¡ | | t |	 t ¡ qH|D ]}	| t |	= qg|D ]}	|	| t	 v r~| t	 |	= qqt
 |¡}
| t  |
t ¡ | t	  |
t	 ¡ d S )Nr   )r  rI   r½   rq   rs   rp   rê   rÙ   rÚ   rë   r  rí   Úupdate)r%   r&   Zorigin_files_mtimeZ	new_filesZremoved_filesZupdated_filesr¤   Zremove_index_keysZremove_requirement_keysrš   Zupdated_indexr1   r1   r2   Ú_update_index|  s<   
€


ÿ€
€
r%  c                  C   sF   t tt tjd¡ ¡ ƒƒ} d}t tt ¡  ¡ ƒƒ}| || kr!dS dS )Nz%Y-%m-%d %H:%M:%Si€3áTF)ÚintÚroundr   Ústrptimer   Ú__release_datetime__Ú	timestampÚnow)Zrelease_timestampZSECONDS_PER_YEARZcurrent_timestampr1   r1   r2   Ú__is_develop_modelž  s   ÿþÿr,  c              
   C   s:  t  d|¡}t  d|¡}t j ||¡}d}|r#t d¡ t | ¡}|S tƒ r˜t d|› �¡ t j 	|¡rkt
|ƒ}tj| d�\}	}
|}ddlm} |t |krT|t |	ksZ|t |krjt d	¡ t||
ƒ t||| ƒ nt d
|› d�¡ t | ¡}t||| ƒ t d|t › d|t › dt|t ƒ› d�¡ |S tƒ }|S )a–  get the index from scan results or cache

    Args:
        file_list: load indexer only from the file lists if provided, default as None
        force_rebuild: If set true, rebuild and load index, default as False,
        indexer_file_dir: The dir where the indexer file saved, default as INDEXER_FILE_DIR
        indexer_file: The indexer file name, default as INDEXER_FILE
    Returns:
        dict: the index information for all registered modules, including key:
        index, requirements, files last modified time, modelscope home path,
        version and md5, the detail is shown below example: {
            'index': {
                ('MODELS', 'nlp', 'bert'):{
                    'filepath' : 'path/to/the/registered/model', 'imports':
                    ['os', 'torch', 'typing'] 'module':
                    'modelscope.models.nlp.bert'
                },
                ...
            }, 'requirements': {
                'modelscope.models.nlp.bert': ['os', 'torch', 'typing'],
                'modelscope.models.nlp.structbert': ['os', 'torch', 'typing'],
                ...
            }, 'files_mtime' : {
                '/User/Path/To/Your/Modelscope/modelscope/preprocessors/nlp/text_generation_preprocessor.py':
                16554565445, ...
            },'version': '0.2.3', 'md5': '8616924970fe6bc119d1562832625612',
            'modelscope_path': '/User/Path/To/Your/Modelscope'
        }
    ZMODELSCOPE_CACHEZMODELSCOPE_INDEX_FILENz4Force rebuilding ast index from scanning every file!zLoading ast index from r  r   r  z}Updating the files for the changes of local files, first time updating will take longer time! Please wait till updating done!zNo valid ast index found from z$, generating ast index from scratch!z,Loading done! Current index file version is z, with md5 z and a total number of z components indexed)rÄ   ÚgetenvrÅ   rn   rä   rå   r  rí   r,  r  r#  rù   r  r  r  r  r%  r  rp   rê   Úload_from_prebuilt)r÷   Úforce_rebuildZindexer_file_dirZindexer_fileZ	cache_dirZ
index_filer  r%   r"  r$   r&   r  r1   r1   r2   Ú
load_index«  sV   $


ÿÿ
€
ÿÿÿ
þÿþr0  c                 C   sH   | d u rt  ¡ jd }tj |t¡} tj | ¡r t| dd�}|S d }|S )Nr   T©r  )	ÚpÚresolveÚparentsrÄ   rÅ   rn   ÚTEMPLATE_FILEr  r#  )r  Ú
local_pathr%   r1   r1   r2   r.  ü  s   ÿr.  c                 C   sT   t |d�}| d u rt ¡ jd }tj |t¡} t|| dd� tj 	| ¡s(t
dƒ‚|S )N©r/  r   Tr1  z;The index file is not create correctly, please double check)r0  r2  r3  r4  rÄ   rÅ   rn   r5  r  r  rÎ   )r  r/  r%   r6  r1   r1   r2   Úgenerate_ast_template  s   
ÿr8  Ú__main__r7  rú   rû   r7   )NT)VrB   ró   r-   rÄ   Úos.pathrÅ   rè   rã   rÏ   r   Ú	functoolsr   Úpathlibr   Útypingr   r  r   r   Zmodelscope.metainfor   r   r	   r
   r   r   r   r   r   r   r   Zmodelscope.utils.constantr   r   Zmodelscope.utils.file_utilsr   Zmodelscope.utils.registryr   Ú__file__r2  rQ   r3  r4  r·   ZINDEXER_FILE_DIRr–   r¶   rü   ZINDEXER_FILErv   rw   ru   rt   rÙ   r  r  r  rê   r  rë   rÚ   rr   r—   ZMODULE_NAMEr™   r,   r5  r3   rä   r«   r4   r­   r  ÚbytesrO   r  r  r#  r%  r,  r0  r.  r8  rS   r%   Úprintr1   r1   r1   r2   Ú<module>   s†   4  9 J

"
ü
Q


þ