o
    óôj	1  ã                   @   s¼   d dl mZ d dlmZ d dlZd dlZd dlZd dlZd dl	Z	d dlm
Z
 d dlm  mZ d dlZd dlZd dlZd dlmZmZ d dlmZmZmZmZmZmZmZ G dd„ dƒZdS )	é    )Úpartial)Ú	GeneratorN)ÚCallable)ÚloggingÚload_wav)Úcontains_chineseÚreplace_blankÚreplace_corner_markÚremove_bracketÚspell_out_numberÚsplit_paragraphÚis_only_punctuationc                   @   s–   e Zd Z		d%dedededededefd	d
„Zdd„ Zdd„ Zdd„ Zdd„ Z	dd„ Z
d&dd„Zdd„ Zdd„ Zdd„ Zdd„ Zd d!„ Zd"d#„ Zd$S )'ÚCosyVoiceFrontEndÚ ÚallÚget_tokenizerÚfeat_extractorÚcampplus_modelÚspeech_tokenizer_modelÚspk2infoÚallowed_specialc              	   C   s€  |ƒ | _ || _t tj ¡ rdnd¡| _t ¡ }tjj	|_
d|_tj||dgd�| _tj||tj ¡ r5dndgd�| _tj |¡rLtj|| jdd�| _ni | _|| _t ¡ | _z3d	d l}| ¡ | _tj tj t¡¡}	| j d
 |	¡¡du szJ dƒ‚| j  d¡ d| _!t" #d¡ W d S    z"d	dl$m%}
 d	dl$m%} |
dd�| _&|ƒ | _'d| _!t" #d¡ W Y d S    d| _!t" #d¡ Y Y d S )NÚcudaÚcpué   ÚCPUExecutionProvider)Úsess_optionsÚ	providersÚCUDAExecutionProviderT)Úmap_locationÚweights_onlyr   z4{}/../../pretrained_models/CosyVoice-ttsfrd/resourcez$failed to initialize ttsfrd resourceZpinyinvgÚttsfrdzuse ttsfrd frontend)Ú
NormalizerF)Zremove_erhuaÚwetextzuse wetext frontendr   zno frontend is avaliable)(Ú	tokenizerr   ÚtorchÚdevicer   Úis_availableÚonnxruntimeÚSessionOptionsÚGraphOptimizationLevelÚORT_ENABLE_ALLÚgraph_optimization_levelÚintra_op_num_threadsÚInferenceSessionÚcampplus_sessionÚspeech_tokenizer_sessionÚosÚpathÚexistsÚloadr   r   ÚinflectÚengineÚinflect_parserr    ZTtsFrontendEngineÚfrdÚdirnameÚabspathÚ__file__Ú
initializeÚformatZset_lang_typeÚtext_frontendr   Úinfor"   r!   Úzh_tn_modelÚen_tn_model)Úselfr   r   r   r   r   r   Úoptionr    ZROOT_DIRZZhNormalizerZEnNormalizer© rC   úL/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/cli/frontend.pyÚ__init__    sL   
ÿÿ

ÿzCosyVoiceFrontEnd.__init__c                 C   sŒ   t |tƒrt d¡ |  |¡tjdgtjd� | j	¡fS | j
j|| jd�}tj|gtjd� | j	¡}tj|jd gtjd� | j	¡}||fS )NzBget tts_text generator, will return _extract_text_token_generator!r   ©Údtype©r   r   )Ú
isinstancer   r   r>   Ú_extract_text_token_generatorr$   ÚtensorÚint32Útor%   r#   Úencoder   Úshape)rA   ÚtextÚ
text_tokenZtext_token_lenrC   rC   rD   Ú_extract_text_tokenN   s   

$ z%CosyVoiceFrontEnd._extract_text_tokenc                 c   sL   � |D ] }|   |¡\}}t|jd ƒD ]}|d d …||d …f V  qqd S )Nr   )rR   ÚrangerO   )rA   Ztext_generatorrP   rQ   Ú_ÚirC   rC   rD   rJ   Y   s   €ÿþz/CosyVoiceFrontEnd._extract_text_token_generatorc              
   C   sÎ   t |dƒ}|jd d dksJ dƒ‚tj|dd�}| j d | j ¡ d j| ¡  	¡  
¡ | j ¡ d jtj|jd gtjd	�i¡d  ¡  ¡ }tj|gtjd	� | j¡}tj|jd gtjd	� | j¡}||fS )
Né€>  r   é   z=do not support extract speech token for audio longer than 30sé€   )Ún_melsr   é   rF   )r   rO   ÚwhisperZlog_mel_spectrogramr/   ÚrunÚ
get_inputsÚnameÚdetachr   ÚnumpyÚnpÚarrayrL   ÚflattenÚtolistr$   rK   rM   r%   )rA   Ú
prompt_wavÚspeechÚfeatÚspeech_tokenÚspeech_token_lenrC   rC   rD   Ú_extract_speech_token_   s"   
ýÿüü z'CosyVoiceFrontEnd._extract_speech_tokenc                 C   s€   t |dƒ}tj|dddd�}||jddd� }| j d | j ¡ d j|jdd� 	¡  
¡ i¡d  ¡  ¡ }t |g¡ | j¡}|S )NrV   éP   r   )Únum_mel_binsÚditherÚsample_frequencyT)ÚdimÚkeepdim©ro   )r   ÚkaldiÚfbankÚmeanr.   r\   r]   r^   Ú	unsqueezer   r`   rc   rd   r$   rK   rM   r%   )rA   re   rf   rg   Ú	embeddingrC   rC   rD   Ú_extract_spk_embeddingl   s    
ý"ÿÿÿz(CosyVoiceFrontEnd._extract_spk_embeddingc                 C   s`   t |dƒ}|  |¡jdd� dd¡ | j¡}|jdd�}tj|j	d gtj
d� | j¡}||fS )NéÀ]  r   rq   r   rF   )r   r   ÚsqueezeÚ	transposerM   r%   ru   r$   rK   rO   rL   )rA   re   rf   Úspeech_featÚspeech_feat_lenrC   rC   rD   Ú_extract_speech_featx   s
   
" z&CosyVoiceFrontEnd._extract_speech_featTc              
   C   sˆ  t |tƒrt d¡ |gS d|v rd|v rd}|du s|dkr(|du r&|gS |S | ¡ }| jdkrGdd	„ t | j 	|¡¡d
 D ƒ}d 
|¡}nnt|ƒr�| jdkrV| j |¡}| dd¡}t|ƒ}t|ƒ}| dd¡}| dd¡}t|ƒ}t dd|¡}tt|t| jj| jd�dddddd�ƒ}n%| jdkr›| j |¡}t|| jƒ}tt|t| jj| jd�dddddd�ƒ}dd	„ |D ƒ}|du rÂ|S |S )Nz1get tts_text generator, will skip text_normalize!z<|z|>Fr   Tr    c                 S   s   g | ]}|d  ‘qS )rP   rC   ©Ú.0rU   rC   rC   rD   Ú
<listcomp>Š   s    z4CosyVoiceFrontEnd.text_normalize.<locals>.<listcomp>Ú	sentencesr"   Ú
Ú.u   ã€‚z - u   ï¼Œu   [ï¼Œ,ã€�]+$rH   Úzhrk   é<   é   )Ztoken_max_nZtoken_min_nZ	merge_lenZcomma_splitÚenc                 S   s   g | ]}t |ƒs|‘qS rC   )r   r~   rC   rC   rD   r€   Ÿ   s    )rI   r   r   r>   Ústripr=   ÚjsonÚloadsr7   Zdo_voicegen_frdÚjoinr   r?   Ú	normalizeÚreplacer   r	   r
   ÚreÚsubÚlistr   r   r#   rN   r   r@   r   r6   )rA   rP   Úsplitr=   ÚtextsrC   rC   rD   Útext_normalize   s@   


 

ÿ
ÿz CosyVoiceFrontEnd.text_normalizec                 C   s.   |   |¡\}}| j| d }||||dœ}|S )Nrv   )rP   Útext_lenÚllm_embeddingÚflow_embedding)rR   r   )rA   Útts_textÚspk_idÚtts_text_tokenÚtts_text_token_lenrv   Úmodel_inputrC   rC   rD   Úfrontend_sft¢   s   zCosyVoiceFrontEnd.frontend_sftc                 C   sú   |   |¡\}}|dkrl|   |¡\}}	|  |¡\}
}|  |¡\}}|dkrYtt|
jd d ƒ|jd ƒ}|
d d …d d| …f d| }
|d d …< |d d …d |…f |}|d d …< |  |¡}||	|||||
|||dœ
}ni | j| ¥}||d< ||d< |S )Nr   rx   r   rZ   )
Úprompt_textÚprompt_text_lenÚllm_prompt_speech_tokenÚllm_prompt_speech_token_lenÚflow_prompt_speech_tokenÚflow_prompt_speech_token_lenÚprompt_speech_featÚprompt_speech_feat_lenr•   r–   rP   r”   )rR   r}   rj   ÚminÚintrO   rw   r   )rA   r—   r�   re   Úresample_rateÚzero_shot_spk_idr™   rš   Zprompt_text_tokenZprompt_text_token_lenr{   r|   rh   ri   Z	token_lenrv   r›   rC   rC   rD   Úfrontend_zero_shot¨   s(   *"
üz$CosyVoiceFrontEnd.frontend_zero_shotc                 C   s.   |   |d|||¡}|d= |d= |d= |d= |S )Nr   r�   rž   rŸ   r    ©r©   )rA   r—   re   r§   r¨   r›   rC   rC   rD   Úfrontend_cross_lingual¿   s   z(CosyVoiceFrontEnd.frontend_cross_lingualc                 C   s4   |   ||¡}|d= |  |¡\}}||d< ||d< |S )Nr•   r�   rž   )rœ   rR   )rA   r—   r˜   Úinstruct_textr›   Zinstruct_text_tokenZinstruct_text_token_lenrC   rC   rD   Úfrontend_instructÈ   s   z#CosyVoiceFrontEnd.frontend_instructc                 C   s"   |   |||||¡}|d= |d= |S )NrŸ   r    rª   )rA   r—   r¬   re   r§   r¨   r›   rC   rC   rD   Úfrontend_instruct2Ñ   s   z$CosyVoiceFrontEnd.frontend_instruct2c                 C   sL   |   |¡\}}|  |¡\}}|  |¡}|   |¡\}	}
|	|
|||||dœ}|S )N)Úsource_speech_tokenÚsource_speech_token_lenr¡   r¢   r£   r¤   r–   )rj   r}   rw   )rA   Zsource_speech_16kre   r§   Zprompt_speech_tokenZprompt_speech_token_lenr£   r¤   rv   r¯   r°   r›   rC   rC   rD   Úfrontend_vc×   s   
ýzCosyVoiceFrontEnd.frontend_vcN)r   r   )TT)Ú__name__Ú
__module__Ú__qualname__r   ÚstrrE   rR   rJ   rj   rw   r}   r“   rœ   r©   r«   r­   r®   r±   rC   rC   rC   rD   r      s8    úÿþýüû
ú.
#		r   )Ú	functoolsr   Útypingr   r‰   r'   r$   r`   ra   r[   r   Ztorchaudio.compliance.kaldiÚ
compliancerr   r0   rŽ   r4   Úcosyvoice.utils.file_utilsr   r   Zcosyvoice.utils.frontend_utilsr   r   r	   r
   r   r   r   r   rC   rC   rC   rD   Ú<module>   s   $