o
    óôjƒŒ  ã                   @   s*  d dl Z d dlZd dlZd dlZd dlZd dlmZmZmZm	Z	m
Z
 d dlZd dlZd dlmZ d dlm  mZ d dlmZ d dlmZmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dl m!Z!m"Z"m#Z# G dd„ dejj$ƒZ%G dd„ dejj$ƒZ&G dd„ de%ƒZ'G dd„ de'ƒZ(dS )é    N)ÚDictÚOptionalÚCallableÚListÚ	Generator)Únn)ÚQwen2ForCausalLM)Úpad_sequenceÚunpad_sequence)Ú	IGNORE_ID)ÚLabelSmoothingLoss)Úth_accuracy)Úlogging)Úmake_pad_mask)ÚSpeechTokenExtractorÚonline_featureÚ	onnx_pathc                       s$  e Zd Z			d1dededededed	ejjd
ejjdedede	def‡ fdd„Z
dejdejfdd„Zdd„ Zdedejdeeeej f fdd„Z	d2dejdededefdd „Ze ¡ 	!	"	#	$d3dejd%ejd&ejd'ejd(ejd)ejd*ejded+e	d,e	d-edeejd.d.f fd/d0„ƒZ‡  ZS )4ÚTransformerLMTç        éÀ   Útext_encoder_input_sizeÚllm_input_sizeÚllm_output_sizeÚtext_token_sizeÚspeech_token_sizeÚtext_encoderÚllmÚsamplingÚlength_normalized_lossÚ
lsm_weightÚspk_embed_dimc                    s¼   t ƒ  ¡  || _|| _tj ||¡| _|| _t 	| j 
¡ |¡| _d| _d| _| j| _tj d|¡| _|| _t 	||d ¡| _t|d t|
|	d�| _tj ||¡| _tj 	||¡| _|| _d S )Nr   é   é   ©ÚsizeÚpadding_idxÚ	smoothingZnormalize_length)ÚsuperÚ__init__r   r   Útorchr   Ú	EmbeddingÚtext_embeddingr   ÚLinearÚoutput_sizeÚtext_encoder_affine_layerÚsosÚtask_idÚ	eos_tokenÚllm_embeddingr   Úllm_decoderr   r   Úcriterion_ceÚspeech_embeddingÚspk_embed_affine_layerr   )Úselfr   r   r   r   r   r   r   r   r   r   r    ©Ú	__class__© úG/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/llm/llm.pyr(   #   s0   
þü
zTransformerLM.__init__ÚtextÚtext_lengthsc                 C   s8   | j ||ddd�\}}| d¡ d¡}|  |¡}||fS )Nr!   éÿÿÿÿ)Zdecoding_chunk_sizeZnum_decoding_left_chunks)r   ÚsqueezeÚsumr.   )r7   r<   r=   Zencoder_outZencoder_maskZencoder_out_lensr:   r:   r;   ÚencodeQ   s   
zTransformerLM.encodec           
         sv   t ˆ| ¡ dd�‰t ˆ| ¡ dd�‰‡ ‡‡‡‡fdd„ttˆƒƒD ƒ}tjdd„ |D ƒtjd�}	t|dtd�}||	fS )NT©Úbatch_firstc              	      s@   g | ]}t jˆjd d�ˆ | ˆ| ˆjd d�ˆ| gd d�‘qS )r   ©Údim)r)   Úconcatr?   ©Ú.0Úi©Ú	embeddingÚsos_embÚspeech_tokenÚtask_id_embÚ
text_tokenr:   r;   Ú
<listcomp>^   s    8ÿz4TransformerLM.pad_unpad_sequence.<locals>.<listcomp>c                 S   ó   g | ]}|  d ¡‘qS ©r   ©r$   rG   r:   r:   r;   rP   `   ó    ©Údtype©rC   Úpadding_value)	r
   ÚcpuÚrangeÚlenr)   ÚtensorÚint32r	   r   )
r7   rL   rK   rO   Útext_token_lenrN   rM   Úspeech_token_lenÚlm_inputÚlm_input_lenr:   rJ   r;   Úpad_unpad_sequence[   s   
ÿz TransformerLM.pad_unpad_sequenceÚbatchÚdeviceÚreturnc              	      s^  |d   |¡}|d   |¡‰|d   |¡‰|d   |¡‰|d   |¡}‡ ‡‡‡fdd„t| d¡ƒD ƒ}t|d	td
�  |¡}ˆ  |¡}ˆ  |ˆ¡\}‰tj|dd�}ˆ  	|¡}| 
d¡}ˆ jjˆ j  ddd¡}ˆ jjˆ j  ddd¡}ˆ  ˆ¡‰ˆ  |||ˆ|ˆˆ¡\}}	ˆ  ||	  |¡¡\}
}ˆ  |
¡}ˆ  ||¡}t| dˆ jd ¡|td�}||dœS )ú™
        Args:
            text: (B, L, D)
            text_lengths: (B,)
            audio: (B, T, N) or (B, T)
            audio_lengths: (B,)
        rO   r^   rM   r_   rK   c              
      sD   g | ]}t  tgd ˆ|   ˆ|dˆ| …f  ¡  ˆ jg ¡‘qS )r"   N)r)   r\   r   Útolistr   rG   ©r7   rM   r_   r^   r:   r;   rP   w   s
    ,ÿ
ÿz)TransformerLM.forward.<locals>.<listcomp>r   TrW   r!   rD   r>   ©Úignore_label©ÚlossÚacc)ÚtorZ   r$   r	   r   r+   rA   ÚFÚ	normalizer6   Ú	unsqueezer2   Úweightr/   Úreshaper0   r5   rb   r   r3   r4   r   Úviewr   )r7   rc   rd   rO   rK   Ú	lm_targetrL   rN   r`   ra   Ú	lm_outputÚlm_output_maskÚlogitsrl   rm   r:   rh   r;   Úforwardd   s2   ÿ



ÿ

zTransformerLM.forwardÚweighted_scoresÚdecoded_tokensÚ
ignore_eosc                 C   s*   |du rt dƒ || j< |  |||¡}|S )NTÚinf)Úfloatr   r   )r7   rz   r{   r   r|   Útop_idsr:   r:   r;   Úsampling_ids–   s   zTransformerLM.sampling_idsé   é   r"   Ú Útext_lenÚprompt_textÚprompt_text_lenÚprompt_speech_tokenÚprompt_speech_token_lenrK   Úmax_token_text_ratioÚmin_token_text_ratioÚuuidNc                 c   s<  � |j }tj||gdd�}||7 }|  |¡}|  ||¡\}}|jd dkr8tj|dd�}|  |¡}|j	dd�}ntj
dd| j|jd� |¡ |j¡}| jj| j  ddd¡}| jj| j  ddd¡}|dkrl|  |¡}ntj
dd| j|jd� |¡}tj|||||gdd�}t|| |
 ƒ}t|| |	 ƒ}g }d}tj
d|j d�tj
d|j d�}}t|ƒD ]l}| jj||d||t tjd|jd |jd f|j d�¡ tj¡d�\}}}|  |d d …df ¡jdd�}| j|jdd�||||k rôd	nd
d�}|| jk�r d S |V  |  |¡ || !d¡7 }| jj|  ddd¡}q¯d S )Nr!   rD   r   rU   r>   )r   r   r   r   ©rd   )ÚoffsetZrequired_cache_sizeÚ	att_cacheÚ	cnn_cacheZatt_maskTF©r|   )"rd   r)   rF   r+   rA   Úshapero   rp   r6   rq   Úzerosr   rV   rn   r2   rr   r/   rs   r0   r5   ÚintrZ   r   Zforward_chunkÚtrilÚonesÚboolr3   Úlog_softmaxr€   r?   r1   Úappendr$   )r7   r<   r„   r…   r†   r‡   rˆ   rK   r   r‰   rŠ   r‹   rd   rL   rN   Úprompt_speech_token_embr`   Úmin_lenÚmax_lenÚ
out_tokensr�   rŽ   r�   rI   Úy_predÚlogpr   r:   r:   r;   Ú	inference¢   sN   €

$"ÿý&
ózTransformerLM.inference)Tr   r   )T©r�   r‚   r"   rƒ   )Ú__name__Ú
__module__Ú__qualname__r“   r)   r   ÚModuler   r–   r~   r(   ÚTensorrA   rb   Údictrd   r   Ústrr   ry   r   r€   Úinference_moder   rŸ   Ú__classcell__r:   r:   r8   r;   r   "   s     ôþýüûúùø	÷
öõô.þ
ý
	þý
ü7ûþýü
û
ôþýüûúùø	÷
öõôór   c                       s<   e Zd Z‡ fdd„Zdejdejfdd„Zd
dd	„Z‡  ZS )ÚQwen2Encoderc                    s   t ƒ  ¡  t |¡| _d S ©N)r'   r(   r   Úfrom_pretrainedÚmodel)r7   Zpretrain_pathr8   r:   r;   r(   ã   s   
zQwen2Encoder.__init__ÚxsÚxs_lensc                 C   s<   |  d¡}t||ƒ }| j||ddd�}|jd | d¡fS )Nr!   T)Úinputs_embedsÚattention_maskÚoutput_hidden_statesÚreturn_dictr>   )r$   r   r­   Úhidden_statesrq   )r7   r®   r¯   ÚTÚmasksÚoutsr:   r:   r;   ry   ç   s   
üzQwen2Encoder.forwardNc                 C   sD   |d d …dd d …f }| j ||ddd|d�}|jd }|j}||fS )Nr>   T)r°   r±   r²   r³   Ú	use_cacheÚpast_key_values)r­   r´   r¹   )r7   r®   r¶   ÚcacheZinput_masksr·   Ú	new_cacher:   r:   r;   Úforward_one_stepò   s   ú
zQwen2Encoder.forward_one_stepr«   )	r¡   r¢   r£   r(   r)   r¥   ry   r¼   r©   r:   r:   r8   r;   rª   â   s    rª   c                   @   sn  e Zd Zddddgfdedededejjd	ed
ede	de
e fdd„Zd-dd„Zdedejdeeeej f fdd„Zdedejdeeeej f fdd„Ze ¡ 				d.dejdejdejd ejd!ejd"ejd#ejd	ed$e	d%e	d&edeejddf fd'd(„ƒZe ¡ d)d*„ ƒZe ¡ 			d/dedejd ejd!ejd"ejd#ejd	ed$e	d%e	deejddf fd+d,„ƒZdS )0ÚQwen2LMTr   é   é   r   r   r   r   r   r   r   Ú	mix_ratioc	           	         sÞ   t jj | ¡ || _|| _ˆ | _d| _d| _ˆ | _	ˆ d | _
t j d|¡| _|| _t |ˆ d ¡| _tˆ d t||d�| _t j ˆ d |¡| _|| _|| _‡ fdd„tdƒD ƒ| _i | _tdu rmttj td	¡d
�| _d S d S )Nr   r!   r"   é   r#   c                    ó   g | ]}ˆ | ‘qS r:   r:   rG   ©r   r:   r;   rP   )  ó    z$Qwen2LM.__init__.<locals>.<listcomp>Tzspeech_tokenizer_v2.batch.onnx©Ú
model_path) r)   r   r¤   r(   r   r   r   r/   r0   r1   Ú
fill_tokenr*   r2   r   r,   r3   r   r   r4   r5   r   rÀ   rZ   Ústop_token_idsÚvllm_output_queuer   r   ÚosÚpathÚjoinr   Úspeech_token_extractor©	r7   r   r   r   r   r   r   r   rÀ   r:   rÃ   r;   r(     s2   
üÿzQwen2LM.__init__Nc              	   C   s  g g }}t || ¡ dd�}t || ¡ dd�}t || ¡ dd�}t || ¡ dd�}|	d urH|
d urH|d urHt |	| ¡ dd�}	t |
| ¡ dd�}
n)t d¡ |d ¡gt|ƒ }	t dd¡ |d ¡gt|ƒ }
t t|ƒ¡ |¡}tt|ƒƒD �]i}t ¡ dk �r£|| ||  | j	d | j	d  k�r£t
g|jdd�g}}|t
g||  7 }| |
| ¡ t|| d | j	d   ¡  ¡  ¡ ƒD ]Ò}|| || j	d  |d | j	d  …  ¡ }|| || j	d  |d | j	d  …  ¡ }t|ƒ| j	d k�rKt|ƒ| j	d k�sJ ‚|t
g| j	d d  7 }||7 }| | j¡ | || || j	d  |d | j	d  … ¡ | || || j	d  |d | j	d  … ¡ qÂ|dgt|ƒ 7 }||| || j	d  d …  ¡ 7 }| | j¡ | || || j	d  d … ¡ | |jdd�¡ | || || j	d  d … ¡ qÂt |¡tj|dd�}}n4t t
gd||  ||   ||  ¡  | jg ¡}tj|jdd�|
| || |jdd�|| gdd�}| |¡ | |¡ qwtjd	d
„ |D ƒtjd�}t|dt
d�}t|dt
d�}|||fS )NTrB   r   i€  g      à?r!   rD   r>   c                 S   rQ   rR   rS   rG   r:   r:   r;   rP   Z  rT   z3Qwen2LM.prepare_lm_input_target.<locals>.<listcomp>rU   rW   )r
   rY   r)   Úemptyrn   r[   r’   rZ   ÚrandomrÀ   r   r?   r˜   Úceilr“   Úitemrg   rÇ   r1   r\   rF   r]   r	   )r7   rL   rO   Útext_token_embr^   rN   rM   Úspeech_token_embr_   Úinstruct_tokenÚinstruct_token_embÚinstruct_token_lenru   r`   rI   Zthis_lm_targetZthis_lm_inputÚjZthis_text_tokenZthis_speech_tokenra   r:   r:   r;   Úprepare_lm_input_target.  sR   
 4*,,.0" "44

zQwen2LM.prepare_lm_input_targetrc   rd   re   c                 C   sæ  |d   |¡}|d   |¡}| jjj |¡}d|vr)| j |d |d |¡\}}n|d   |¡}|d   |¡}|  |¡}| jjdkr[| jj	| j
  ddd	¡}	| jj	| j  ddd	¡}
n!| jjd
krz| jj	| j
  ddd	¡}	| jj	| j  ddd	¡}
nt‚| jjdkr«|d   |¡}|d   |¡}| jjj |¡}|  |	||||
||||||¡\}}}n| jjd
krÁ|  |	||||
|||¡\}}}nt‚|  |¡}|  ||  |¡¡\}}|  |¡}|  ||  |¡¡}t| d	| jj¡|td�}||dœS )rf   rO   r^   rM   Zwhisper_featZwhisper_feat_lenr_   ÚCosyVoice3LMr!   r>   r½   rÕ   r×   ri   rk   )rn   r   r­   Úembed_tokensrÍ   rŸ   r5   r9   r¡   rr   r/   rs   r0   r2   Ú
ValueErrorrÙ   r3   r4   r   rt   Úout_featuresr   )r7   rc   rd   rO   r^   rÓ   rM   r_   rÔ   rL   rN   rÕ   r×   rÖ   ru   r`   ra   rv   rw   rx   rl   rm   r:   r:   r;   ry   _  sB   
ÿÿ


zQwen2LM.forwardc              
   C   sv  |d   |¡}|d   |¡}|d   |¡}|d   |¡}|d   |¡}|d   |¡}| jjj |¡}	| jj| j  ddd¡}
| jj| j  ddd¡}t	|| 
¡ d	d
�}t	|| 
¡ d	d
�}|| }t|d	dd�}tj||gdd�}|  |¡}|  |
| dd¡|	 ddd¡| d¡||||¡\}}}|  |¡}|  ||  |¡¡\}}|  |¡}|d |jd … }||jd d … }|d |jd … }||jd d … }|  ||  |¡¡}t| d| jd ¡|td�}|tk}|tk}tj|jdd�d| |d¡jdd�d�jdd�}tj|jdd�d| |d¡jdd�d�jdd�}|| jdd�|jdd� }|| jdd�|jdd� }||||dœS )NrO   r^   rM   r_   Úreject_speech_tokenÚreject_speech_token_lenr!   r>   TrB   r   rW   rD   r"   rÁ   ri   )rE   Úindex)rl   rm   Úchosen_logpsÚrejected_logps)rn   r   r­   rÛ   r2   rr   r/   rs   r0   r
   rY   r	   r)   rF   r5   rÙ   Úrepeatr3   r‘   r4   r   rt   r   r   Úgatherr—   Úmasked_fillrq   r?   r@   )r7   rc   rd   rO   r^   rM   r_   rÞ   rß   rÓ   rL   rN   Zspeech_token_combinedZspeech_token_combined_lenZspeech_token_combined_embru   r`   ra   rv   rw   rx   Zchosen_logitsZrejected_logitsZchosen_lm_targetZrejected_lm_targetrl   rm   Zchosen_lm_maskZrejected_lm_maskrá   râ   r:   r:   r;   Úforward_dpo—  sD   
$
ÿ

00zQwen2LM.forward_dpor�   r‚   r"   rƒ   r<   r„   r…   r†   r‡   rˆ   rK   r‰   rŠ   r‹   c                 c   sV  � |j }tj||gdd�}||7 }| jjj |¡}| jjdkr'd|v s'J dƒ‚| jjdkrF| jj	| j
  ddd¡}| jj	| j  ddd¡}n!| jjdkre| jj	| j
  ddd¡}| jj	| j  ddd¡}nt‚|dkrq|  |¡}ntjdd| j|jd	� |¡}tj||||gdd�}t|| |
 ƒ}t|| |	 ƒ}|  |||||¡D ]}|V  q£d S )
Nr!   rD   rÚ   é^P zQ<|endofprompt|> not detected in CosyVoice3 text or prompt_text, check your input!r>   r½   r   rU   )rd   r)   rF   r   r­   rÛ   r9   r¡   r5   rr   r/   rs   r0   r2   rÜ   r’   r   rV   rn   r“   Úinference_wrapper)r7   r<   r„   r…   r†   r‡   rˆ   rK   r   r‰   rŠ   r‹   rd   Ztext_embrL   rN   r™   r`   rš   r›   Útokenr:   r:   r;   rŸ   Ê  s.   €ÿzQwen2LM.inferencec              
   c   sl  � t | dƒrÉddlm}m} ||| j||d�}| j�$ | j |d| d¡ t	j
¡ |j¡i|¡ t ¡ | j|< W d   ƒ n1 sAw   Y  g }		 | j�. | j|  ¡ du rs| j ¡ }
|
D ]}t|jd jƒd }| j|j  |¡ q]W d   ƒ n1 s}w   Y  | j|  ¡ du r§| j|  ¡ }|| jv r˜n|V  |	 |¡ t|	ƒ|kr§nt d	¡ qI| j� | j |¡ W d   ƒ d S 1 sÂw   Y  d S g }	d }t|ƒD ]b}| jj|t	 t	j d
|j!d
 |j!d
 f|jd�¡ t	j"¡|d�\}}|  #|d d …df ¡j$dd�}| j%|jdd�|	|||k �rdndd�}|| jv �r  d S |V  |	 |¡ | j&j'|  (d
d
d¡}qÑd S )NÚvllmr   )ÚSamplingParamsÚRequestOutput)Útop_krÈ   Z
min_tokensZ
max_tokensZprompt_embedsTr>   Fgü©ñÒMbP?r!   rŒ   ©r¶   rº   rD   r�   ))Úhasattrrê   rë   rì   rÈ   ÚlockÚadd_requestr?   rn   r)   Úbfloat16rd   ÚqueueÚQueuerÉ   rÏ   ÚstepÚlistÚoutputsÚ	token_idsÚ
request_idÚputÚgetr˜   r[   ÚtimeÚsleepÚpoprZ   r   r¼   r”   r•   r‘   r–   r3   r—   r€   r5   rr   rs   )r7   r`   r   rš   r›   r‹   rë   rì   Zsampling_paramsrœ   Zrequest_outputsZrequest_outputr   rº   rI   r�   rž   r:   r:   r;   rè   ø  sd   €
ý*þ
€û


ð"ÿ.
þ(
õzQwen2LM.inference_wrapperc
              	   c   sú  � |j }
| jjdkr#| jj| j  ddd¡}| jj| j  ddd¡}n!| jjdkrB| jj| j  ddd¡}| jj| j  ddd¡}nt	‚|dkrN|  |¡}nt
jdd| j|jd� |
¡}t
j|gdd�}g }d }| jjdkr¤d|v svJ d	ƒ‚| ¡  ¡  d¡}t
j|| jjj |d d …d |d …f ¡gdd�}|d d …|d d …f }| jjj |¡}t|jd | jd  ƒd | jd  |jd  }|D �]¬}t
j|| jjj |¡gdd�}| d¡dk�rH| d¡| jd k�r;|d d …d | jd …f |d d …d | jd …f }}t d
 | d¡| d¡¡¡ t
j|||gdd�}|d d …| jd d …f |d d …| jd d …f }}nt d¡ n| d¡dksà| d¡dk�rst|ƒdk�r_|d | jk�snt|ƒdk�rÅ| d¡dk�rÅt d¡ | d¡| jd k�r¿|d d …d | jd …f }t d | d¡¡¡ t|ƒdk�r¨|d | jk�r¨|}n	t
j||gdd�}|d d …| jd d …f }nt d¡ qÆ	 |d u �rÐ|jd n|jd |d d  d¡ }| jj|t
  t
j!d||f|j d�¡ t
j"¡|d�\}}|  #|d d …df ¡j$dd�}|dk�r!t|ƒ|k�r!| j}|| jd d 7 }n| j%|j&dd�||dd�}|| jk�rJt|ƒ| jd  d }t d t|ƒ|¡¡ | '|¡ || j(k�rc|| jk�r\nt	d |¡ƒ‚|V  | jj|  ddd¡}�qÆqÆt
j|||gdd�}t d¡ 	 |d u �rŽ|jd n|jd |d d  d¡ }| jj|t
  t
j!d||f|j d�¡ t
j"¡|d�\}}|  #|d d …df ¡j$dd�}| j%|j&dd�||dd�}| '|¡ || j(k�rí|| j)k�ræd S t	d |¡ƒ‚|V  | jj|  ddd¡}�q„)NrÚ   r!   r>   r½   r   rU   rD   rç   zI<|endofprompt|> not detected in CosyVoice3 prompt_text, check your input!z$append {} text token {} speech tokenz.not enough text token to decode, wait for morez.get fill token, need to append more text tokenzappend {} text tokenTr"   rŒ   rî   r�   z,fill_token index {} next fill_token index {}zshould not get token {}z(no more text token, decode until met eosF)*rd   r9   r¡   r5   rr   r/   rs   r0   r2   rÜ   r)   r’   r   rV   rn   rF   Úflattenrg   rà   r   r­   rÛ   r“   r‘   rÀ   r$   r   ÚinfoÚformatr[   rÇ   r¼   r”   r•   r–   r3   r—   r€   r?   r˜   r   r1   )r7   r<   r…   r†   r‡   rˆ   rK   r   r‰   rŠ   rd   rL   rN   r™   r`   rœ   rº   Z	eop_indexZ
text_cacheZnext_fill_indexZ	this_textZlm_input_textZlm_input_speechÚseq_lenr�   rž   r   r:   r:   r;   Úinference_bistream'  s¬   €20
68
ø
<

0"
þ
ë€
0"
þ
ñzQwen2LM.inference_bistream)NNNr    )r�   r‚   r"   )r¡   r¢   r£   r“   r)   r   r¤   r   r–   r~   r   r(   rÙ   r¦   rd   r   r§   r   r¥   ry   ræ   r¨   r   rŸ   rè   r  r:   r:   r:   r;   r½     sº    ÷þýüûúùø	
÷
,1þý
ü8þý
ü3
ôþýüûúùø	÷
öõôó-
.	öþýüûúùø	÷
öõr½   c                   @   sJ   e Zd Zddddgfdedededejjd	ed
ede	de
e fdd„ZdS )rÚ   Tr   r¾   r¿   r   r   r   r   r   r   r   rÀ   c	           	         sÞ   t jj | ¡ || _|| _ˆ | _ˆ d | _ˆ d | _ˆ d | _	ˆ d | _
|| _tj|ˆ d dd�| _tˆ d t||d�| _t j ˆ d |¡| _|| _|| _‡ fd	d
„tdƒD ƒ| _i | _tdu rmttj td¡d�| _d S d S )Nr   r!   r"   rÁ   éÈ   F)Úbiasr#   c                    rÂ   r:   r:   rG   rÃ   r:   r;   rP   ¿  rÄ   z)CosyVoice3LM.__init__.<locals>.<listcomp>Tzspeech_tokenizer_v3.batch.onnxrÅ   )r)   r   r¤   r(   r   r   r   r/   r1   r0   rÇ   r   r,   r3   r   r   r4   r*   r5   r   rÀ   rZ   rÈ   rÉ   r   r   rÊ   rË   rÌ   r   rÍ   rÎ   r:   rÃ   r;   r(   ™  s0   



üÿzCosyVoice3LM.__init__N)r¡   r¢   r£   r“   r)   r   r¤   r   r–   r~   r   r(   r:   r:   r:   r;   rÚ   ˜  s*    ÷þýüûúùø	÷rÚ   ))rÊ   ró   rÐ   rü   Ú	threadingÚtypingr   r   r   r   r   ÚnumpyÚnpr)   r   Ztorch.nn.functionalÚ
functionalro   Útransformersr   Ztorch.nn.utils.rnnr	   r
   Úcosyvoice.utils.commonr   Z*cosyvoice.transformer.label_smoothing_lossr   r   Úcosyvoice.utils.file_utilsr   Zcosyvoice.utils.maskr   Zcosyvoice.utils.onnxr   r   r   r¤   r   rª   r½   rÚ   r:   r:   r:   r;   Ú<module>   s2    A   