o
    óôjM  ã                   @   s  d dl Z d dlZd dlZd dlmZmZ d dlZd dlmZ d dlm	Z
 d dlmZ d dlmZ d dlmZmZmZ G dd„ dejjƒZG d	d
„ d
ejjƒZG dd„ dejjƒZedk�rˆdejj_dejj_d dlmZ eddƒ�Zeedddœd�Z W d  ƒ n1 s†w   Y  e d Z!ej" #¡ r–dndZ$e! %e$¡ e! &¡  de!j'j(j) Z*e!j'j(j)Z+e!j,j-Z.ej/d dde*fd� %e$¡Z0e 1e*g¡ %e$¡Z2ej/d dde+fd� %e$¡Z3e 1e+g¡ %e$¡Z4e 5de+d d¡ %e$¡Z6e 1e+d g¡ %e$¡Z7e 5dd¡ %e$¡Z8e!j9e0e2e3e4e6e7e8ddd�	\Z:Z;e<d e*e+ƒD ]vZ=e=e+ e. e*k�r dndZ>e!j9e0dd…de=e+ e. …f e 1e0dd…de=e+ e. …f j?d g¡ %e$¡e3e4e6e7e8de>d�	\Z@Z;e@dd…dd…e=e!jA d…f Z@eBe:dd…dd…e=e!jA e=e!jA e@j?d  …f e@  C¡  D¡  E¡ ƒ �qdS dS ) é    N)ÚDictÚOptional)Ú
functional)Ú
DictConfig)Úmake_pad_mask)ÚSpeechTokenExtractorÚonline_featureÚ	onnx_pathc                       sØ   e Zd Zddddddddddd	ddd
edddddddœƒddgdddddddœdœfdededededed ed!ed"ejj	d#ejj	d$ejj	d%e
f‡ fd&d'„Zd(ed)ejd*e
eeej f fd+d,„Ze ¡ d-d.„ ƒZ‡  ZS )/ÚMaskedDiffWithXvecé   éP   éÀ   Úmelé   é2   TNéð   é   ç�íµ ÷Æ°>ÚeulerÚcosineçš™™™™™É?çffffffæ?Úl1©Z	sigma_minÚsolverZt_schedulerZtraining_cfg_rateZinference_cfg_rateZreg_loss_typeé   ç        é@   é   é   é   Úgelu©ÚchannelsÚdropoutZattention_head_dimÚn_blocksZnum_mid_blocksÚ	num_headsÚact_fn©Úin_channelsZout_channelZspk_emb_dimZn_spksZ
cfm_paramsZdecoder_paramsÚ
input_sizeÚoutput_sizeÚspk_embed_dimÚoutput_typeÚ
vocab_sizeÚinput_frame_rateÚonly_mask_lossÚencoderÚlength_regulatorÚdecoderÚdecoder_confc                    s�   t ƒ  ¡  || _|| _|| _|| _|| _|| _t 	d| j› �¡ t
 ||¡| _tj
 ||¡| _|| _tj
 | j ¡ |¡| _|
| _|	| _|| _d S )Núinput frame rate=)ÚsuperÚ__init__r*   r+   r4   r.   r-   r/   ÚloggingÚinfoÚnnÚ	EmbeddingÚinput_embeddingÚtorchÚLinearÚspk_embed_affine_layerr1   Úencoder_projr3   r2   r0   )Úselfr*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   ©Ú	__class__© úI/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/flow/flow.pyr7      s   

zMaskedDiffWithXvec.__init__ÚbatchÚdeviceÚreturnc                 C   sx  |d   |¡}|d   |¡}|d   |¡}|d   |¡}|d   |¡}tj|dd�}|  |¡}t|ƒ  ¡  d¡  |¡}|  tj	|d	d
�¡| }|  
||¡\}	}
|  |	¡}	|  |	|¡\}	}
tj|j|jd�}t|ƒD ]#\}}t ¡ dk ruqjt d	td| ƒ¡}||d |…f ||d |…f< qj| dd¡}t|ƒ   |	¡}| jj| dd¡ ¡ | d¡|	 dd¡ ¡ ||d�\}}d|iS )NÚspeech_tokenÚspeech_token_lenÚspeech_featÚspeech_feat_lenÚ	embeddingr   ©Údiméÿÿÿÿr   ©Úmin©rG   ç      à?ç333333Ó?é   )ÚcondÚloss)ÚtoÚFÚ	normalizer?   r   ÚfloatÚ	unsqueezer<   r=   Úclampr1   r@   r2   ÚzerosÚshaperG   Ú	enumerateÚrandomÚrandintÚintÚ	transposer3   Úcompute_lossÚ
contiguous)rA   rF   rG   ÚtokenÚ	token_lenÚfeatÚfeat_lenrM   ÚmaskÚhÚ	h_lengthsÚcondsÚiÚjÚindexrX   Ú_rD   rD   rE   Úforward:   s8   


ûzMaskedDiffWithXvec.forwardc	              	   C   sÆ  |j d dks	J ‚tj|dd�}|  |¡}|j d |j d }	}
tj||gdd�|| }}t|ƒ  d¡ |¡}|  	tj
|dd�¡| }|  ||¡\}}|  |¡}|j d t|
| j d d ƒ}}| j |d d …d |	…f |d d …|	d …f ||| j¡\}}tjd|| | jg|jd� |j¡}||d d …d |…f< | dd	¡}tt || g¡ƒ  |¡}| j| dd	¡ ¡ | d¡||d
||d�\}}|d d …d d …|d …f }|j d	 |ksÝJ ‚| ¡ |fS )Nr   r   rN   rP   rQ   i"V  r   rS   rV   é
   )Úmurl   ÚspksrW   Ún_timestepsZ
prompt_lenÚcache)r`   rZ   r[   r?   r=   Úconcatr   r]   rY   r<   r^   r1   r@   rd   r/   r2   Ú	inferencer_   r+   rG   Údtypere   Útensorr3   rg   r\   )rA   rh   ri   Úprompt_tokenÚprompt_token_lenÚprompt_featÚprompt_feat_lenrM   Ú
flow_cacheZ
token_len1Z
token_len2rl   rm   rn   Úmel_len1Úmel_len2ro   rj   rD   rD   rE   r{   f   s6   


":$
ù	zMaskedDiffWithXvec.inference©Ú__name__Ú
__module__Ú__qualname__r   rd   ÚstrÚboolr=   r:   ÚModuler   r7   ÚdictrG   r   ÚTensorrt   Úinference_moder{   Ú__classcell__rD   rD   rB   rE   r
      sf    ÿ
ÿýõÿþýüûúùø	÷
öõ þý
ü,r
   c                       óÚ   e Zd Zddddddddd	d
d
ddddedddddddœƒddgdddddddœdœfdededed ed!ed"ed#ed$ed%ed&ejj	d'ejj	d(e
f‡ fd)d*„Zd+ed,ejd-e
eeej f fd.d/„Ze ¡ d0d1„ ƒZ‡  ZS )2ÚCausalMaskedDiffWithXvecr   r   r   r   r   r   TrV   é   Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r(   r*   r+   r,   r-   r.   r/   r0   Útoken_mel_ratioÚpre_lookahead_lenr1   r3   r4   c                    s¸   t ƒ  ¡  || _|| _|| _|| _|| _|| _t 	d| j› �¡ t
 ||¡| _tj
 ||¡| _|
| _tj
 | j ¡ |¡| _|| _|| _|| _|	| _tdu rZttj td¡d�| _d S d S )Nr5   Tzspeech_tokenizer_v2.batch.onnx©Ú
model_path)r6   r7   r*   r+   r4   r.   r-   r/   r8   r9   r:   r;   r<   r=   r>   r?   r1   r@   r3   r0   r“   r”   r   r   ÚosÚpathÚjoinr	   Úspeech_token_extractor)rA   r*   r+   r,   r-   r.   r/   r0   r“   r”   r1   r3   r4   rB   rD   rE   r7   –   s&   
ÿz!CausalMaskedDiffWithXvec.__init__rF   rG   rH   c                 C   s¸  d|vr| j  |d |d |¡\}}n|d  |¡}|d  |¡}|d  |¡}|d  |¡}|d  |¡}t ¡ dk r>d	nd
}tj|dd�}|  |¡}t|ƒ  ¡  	d¡ |¡}	|  
tj|dd�¡|	 }| j|||d�\}
}|  |
¡}
tj|j|jd�}t|ƒD ]#\}}t ¡ dk rŒq�t dtd| ƒ¡}||d |…f ||d |…f< q�| dd¡}t|jdd�jdd�ƒ  |
¡}	| jj| dd¡ ¡ |	 	d¡|
 dd¡ ¡ |||d�\}}d|iS )NrI   Úwhisper_featÚwhisper_feat_lenrJ   rK   rL   rM   rT   TFr   rN   rP   r   rQ   ©Ú	streamingrS   rU   rV   ©rW   rž   rX   )rš   r{   rY   rb   rZ   r[   r?   r   r\   r]   r<   r=   r^   r1   r@   r_   r`   rG   ra   rc   rd   re   ÚsumÚsqueezer3   rf   rg   )rA   rF   rG   rh   ri   rj   rk   rM   rž   rl   rm   rn   ro   rp   rq   rr   rX   rs   rD   rD   rE   rt   º   s>   

 
úz CausalMaskedDiffWithXvec.forwardc
                 C   sÈ  |j d dks	J ‚tj|dd�}|  |¡}tj||gdd�|| }}t|ƒ  d¡ |¡}
|  	tj
|dd�¡|
 }|	du rI| j|||d�\}}n$|d d …d | j …f |d d …| j d …f }}| j||||d�\}}|j d |j d |j d  }}|  |¡}tjd|| | jg|jd	� |j¡}||d d …d |…f< | dd
¡}tt || g¡ƒ  |¡}
| j| dd
¡ ¡ |
 d¡||d|d�\}}|d d …d d …|d …f }|j d
 |ksÞJ ‚| ¡ d fS )Nr   r   rN   rP   rQ   Tr�   )Úcontextrž   rS   rV   ru   ©rv   rl   rw   rW   rx   rž   )r`   rZ   r[   r?   r=   rz   r   r]   rY   r<   r^   r1   r”   r@   r_   r+   rG   r|   re   r}   r3   rg   r\   )rA   rh   ri   r~   r   r€   r�   rM   rž   Úfinalizerl   rm   rn   r¢   rƒ   r„   ro   rj   rs   rD   rD   rE   r{   ë   s6   
2 
$
úz"CausalMaskedDiffWithXvec.inferencer…   rD   rD   rB   rE   r‘   •   sl    ÿ
ÿýôÿþýüûúùø	÷
öõô$þý
ü1r‘   c                       r�   )2ÚCausalMaskedDiffWithDiTr   r   r   r   r   r   TrV   r’   Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r(   r*   r+   r,   r-   r.   r/   r0   r“   r”   Úpre_lookahead_layerr3   r4   c                    s¢   t ƒ  ¡  || _|| _|| _|| _|| _|| _t 	d| j› �¡ t
 ||¡| _tj
 ||¡| _|	| _|
| _|| _|| _|| _tdu rOttj td¡d�| _d S d S )Nr5   Tzspeech_tokenizer_v3.batch.onnxr•   )r6   r7   r*   r+   r4   r.   r-   r/   r8   r9   r:   r;   r<   r=   r>   r?   r”   r¦   r3   r0   r“   r   r   r—   r˜   r™   r	   rš   )rA   r*   r+   r,   r-   r.   r/   r0   r“   r”   r¦   r3   r4   rB   rD   rE   r7     s$   
ÿz CausalMaskedDiffWithDiT.__init__rF   rG   rH   c                 C   s¬  d|vr| j  |d |d |¡\}}n|d  |¡}|d  |¡}|d  |¡}|d  |¡}|d  |¡}t ¡ dk r>d	nd
}tj|dd�}|  |¡}t|ƒ  ¡  	d¡ |¡}	|  
tj|dd�¡|	 }|  |¡}
|
j| jdd�}
|	j| jdd�jdd�}	tj|j|jd�}t|ƒD ]#\}}t ¡ dk r–q‹t dtd| ƒ¡}||d |…f ||d |…f< q‹| dd¡}| jj| dd¡ ¡ |	 	d¡|
 dd¡ ¡ |||d�\}}d|iS )NrI   r›   rœ   rJ   rK   rL   rM   rT   TFr   rN   rP   r   rQ   rS   rU   rV   rŸ   rX   )rš   r{   rY   rb   rZ   r[   r?   r   r\   r]   r<   r=   r^   r¦   Úrepeat_interleaver“   r¡   r_   r`   rG   ra   rc   rd   re   r3   rf   rg   )rA   rF   rG   rh   ri   rj   rk   rM   rž   rl   rm   ro   rp   rq   rr   rX   rs   rD   rD   rE   rt   @  s>   


úzCausalMaskedDiffWithDiT.forwardc
                 C   s²  |j d dks	J ‚tj|dd�}|  |¡}tj||gdd�|| }}t|ƒ  d¡ |¡}
|  	tj
|dd�¡|
 }|	du rD|  |¡}n| j|d d …d | j …f |d d …| j d …f d�}|j| jdd�}|j d |j d |j d  }}tjd|| | jg|jd� |j¡}||d d …d |…f< | dd	¡}tt || g¡ƒ  |¡}
| j| dd	¡ ¡ |
 d¡||d
|d�\}}|d d …d d …|d …f }|j d	 |ksÓJ ‚| ¡ d fS )Nr   r   rN   rP   rQ   T)r¢   rS   rV   ru   r£   )r`   rZ   r[   r?   r=   rz   r   r]   rY   r<   r^   r¦   r”   r§   r“   r_   r+   rG   r|   re   r}   r3   rg   r\   )rA   rh   ri   r~   r   r€   r�   rM   rž   r¤   rl   rm   rƒ   r„   ro   rj   rs   rD   rD   rE   r{   q  s4   
6 $
úz!CausalMaskedDiffWithDiT.inferencer…   rD   rD   rB   rE   r¥     sl    ÿ
ÿýôÿþýüûúùø	÷
öõô#þý
ü1r¥   Ú__main__TF)Úload_hyperpyyamlz7./pretrained_models/Fun-CosyVoice3-0.5B/cosyvoice3.yamlÚr)ÚllmÚhift)Ú	overridesÚflowÚcudaÚcpuru   i¡  r   )ÚsizerV   r   r   )rž   r¤   )Fr—   r8   rb   Útypingr   r   r=   Útorch.nnr:   r   rZ   Ú	omegaconfr   Úcosyvoice.utils.maskr   Úcosyvoice.utils.onnxr   r   r	   r‹   r
   r‘   r¥   r†   ÚbackendsÚcudnnÚdeterministicÚ	benchmarkÚhyperpyyamlr©   ÚopenÚfÚconfigsÚmodelr¯   Úis_availablerG   rY   Úevalr3   Ú	estimatorÚstatic_chunk_sizeÚmax_lenÚ
chunk_sizer¦   r”   Zcontext_sizerc   rh   r}   ri   r~   r   Úrandr€   r�   Zprompt_embeddingr{   Zpred_gtrs   Úrangerp   r¤   r`   Z
pred_chunkr“   ÚprintÚabsÚmaxÚitemrD   rD   rD   rE   Ú<module>   sZ   |  


ÿ

 L
ÿ Hæ