o
    óôj•7  ã                   @   sÀ   d Z ddlmZ ddlZddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZ dd	lmZmZmZmZ dd
lmZ ddlmZ G dd„ dejƒZG dd„ dejƒZG dd„ dejjƒZdS )zEncoder definition.é    )ÚTupleN)Únn)Ú
functional)ÚConvolutionModule)ÚConformerEncoderLayer)ÚPositionwiseFeedForward)ÚCOSYVOICE_EMB_CLASSESÚCOSYVOICE_SUBSAMPLE_CLASSESÚCOSYVOICE_ATTENTION_CLASSESÚCOSYVOICE_ACTIVATION_CLASSES)Úmake_pad_mask)Úadd_optional_chunk_maskc                       sV   e Zd ZdZddededef‡ fdd„Zdejd	ejd
eejejf fdd„Z	‡  Z
S )Ú
Upsample1DaÂ  A 1D upsampling layer with an optional convolution.

    Parameters:
        channels (`int`):
            number of channels in the inputs and outputs.
        use_conv (`bool`, default `False`):
            option to use a convolution.
        use_conv_transpose (`bool`, default `False`):
            option to use a convolution transpose.
        out_channels (`int`, optional):
            number of output channels. Defaults to `channels`.
    é   ÚchannelsÚout_channelsÚstridec                    sB   t ƒ  ¡  || _|| _|| _tj| j| j|d d ddd�| _d S )Nr   é   r   )r   Úpadding)ÚsuperÚ__init__r   r   r   r   ÚConv1dÚconv)Úselfr   r   r   ©Ú	__class__© ú\/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/transformer/upsample_encoder.pyr   3   s
   
&zUpsample1D.__init__ÚinputsÚinput_lengthsÚreturnc                 C   sH   t j|t| jƒdd�}t j|| jd dfdd�}|  |¡}||| j fS )NÚnearest)Úscale_factorÚmoder   r   ç        )Úvalue)ÚFÚinterpolateÚfloatr   Úpadr   )r   r   r   Úoutputsr   r   r   Úforward;   s   
zUpsample1D.forward)r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   ÚtorchÚTensorr   r+   Ú__classcell__r   r   r   r   r   %   s    .r   c                       sV   e Zd Zddededef‡ fdd„Ze ddd¡fdejd	ejd
ejfdd„Z‡  Z	S )ÚPreLookaheadLayerr   Úin_channelsr   Úpre_lookahead_lenc                    sP   t ƒ  ¡  || _|| _|| _tj|||d ddd�| _tj||dddd�| _d S )Nr   r   )Úkernel_sizer   r   é   )	r   r   r5   r   r6   r   r   Úconv1Úconv2)r   r5   r   r6   r   r   r   r   C   s   
ýþzPreLookaheadLayer.__init__r   r   Úcontextr    c                 C   sö   |  dd¡ ¡ }|  dd¡ ¡ }| d¡dkr$tj|d| jfddd�}n+| jdu s-J dƒ‚| d¡| jks7J ‚tjtj||gdd	�d| j| d¡ fddd�}t 	|  
|¡¡}tj|| jjd d dfddd�}|  |¡}|  dd¡ ¡ }|| }|S )
z9
        inputs: (batch_size, seq_len, channels)
        r   r   r   Úconstantr$   )r#   r%   FúFyou have passed context, make sure that you are running inference mode)Údim)Ú	transposeÚ
contiguousÚsizer&   r)   r6   Útrainingr1   ÚconcatÚ
leaky_relur9   r:   r7   )r   r   r;   r*   r   r   r   r+   R   s   0"
zPreLookaheadLayer.forward)r   )
r,   r-   r.   r0   r   r1   Úzerosr2   r+   r3   r   r   r   r   r4   B   s    2r4   c                3       sb  e Zd Z												
																d@dedededededededededededededejj	d ed!ed"ed#ed$ed%ed&ed'ed(ed)ed*ef2‡ fd+d,„Z
d-efd.d/„Ze d
d
d
¡d
d0dfd1ejd2ejd3ejd4ed5ed6ed-eejejf fd7d8„Zd1ejd9ejd:ejd;ejd-ejf
d<d=„Zd1ejd9ejd:ejd;ejd-ejf
d>d?„Z‡  ZS )AÚUpsampleConformerEncoderé   é   é   é   çš™™™™™¹?r$   Úconv2dÚrel_posTr   FNr   Úrel_selfattnÚswishé   Ú
batch_normÚ
input_sizeÚoutput_sizeÚattention_headsÚlinear_unitsÚ
num_blocksÚdropout_rateÚpositional_dropout_rateÚattention_dropout_rateÚinput_layerÚpos_enc_layer_typeÚnormalize_beforeÚstatic_chunk_sizeÚuse_dynamic_chunkÚglobal_cmvnÚuse_dynamic_left_chunkÚpositionwise_conv_kernel_sizeÚmacaron_styleÚselfattention_layer_typeÚactivation_typeÚuse_cnn_moduleÚcnn_module_kernelÚcausalÚcnn_module_normÚkey_biasÚgradient_checkpointingc                    s6  t ƒ  ¡  ˆ| _|| _t|	 |ˆˆt|
 ˆ|ƒƒ| _ˆ| _tj	j
ˆdd�| _|| _|| _|| _|| _t| ƒ }|ˆ||f‰ˆ|ˆ|f‰ˆ||||f‰ tdddd�| _tj	 ‡ ‡‡‡‡‡‡‡‡f	dd„t|ƒD ƒ¡| _tdddd	�| _t|	 |ˆˆt|
 ˆ|ƒƒ| _tj	 ‡ ‡‡‡‡‡‡‡‡f	d
d„tdƒD ƒ¡| _dS )aÕ  
        Args:
            input_size (int): input dim
            output_size (int): dimension of attention
            attention_heads (int): the number of heads of multi head attention
            linear_units (int): the hidden units number of position-wise feed
                forward
            num_blocks (int): the number of decoder blocks
            dropout_rate (float): dropout rate
            attention_dropout_rate (float): dropout rate in attention
            positional_dropout_rate (float): dropout rate after adding
                positional encoding
            input_layer (str): input layer type.
                optional [linear, conv2d, conv2d6, conv2d8]
            pos_enc_layer_type (str): Encoder positional encoding layer type.
                opitonal [abs_pos, scaled_abs_pos, rel_pos, no_pos]
            normalize_before (bool):
                True: use layer_norm before each sub-block of a layer.
                False: use layer_norm after each sub-block of a layer.
            static_chunk_size (int): chunk size for static chunk training and
                decoding
            use_dynamic_chunk (bool): whether use dynamic chunk size for
                training or not, You can only use fixed chunk(chunk_size > 0)
                or dyanmic chunk size(use_dynamic_chunk = True)
            global_cmvn (Optional[torch.nn.Module]): Optional GlobalCMVN module
            use_dynamic_left_chunk (bool): whether use dynamic left chunk in
                dynamic chunk training
            key_bias: whether use bias in attention.linear_k, False for whisper models.
            gradient_checkpointing: rerunning a forward-pass segment for each
                checkpointed segment during backward.
        gñhãˆµøä>)Úepsi   r8   )r5   r   r6   c              
      óD   g | ]}t ˆtˆ ˆŽ tˆŽ ˆrtˆŽ nd ˆrtˆ Ž nd ˆˆƒ‘qS ©N©r   r
   r   r   ©Ú.0Ú_©	Zconvolution_layer_argsrW   Zencoder_selfattn_layer_argsrb   r\   rS   Zpositionwise_layer_argsrc   re   r   r   Ú
<listcomp>Ì   ó,    õÿÿÿÿÿöÿz5UpsampleConformerEncoder.__init__.<locals>.<listcomp>r   )r   r   r   c              
      rl   rm   rn   ro   rr   r   r   rs   â   rt   rH   N)r   r   Ú_output_sizer_   r	   r   Úembedr\   r1   r   Ú	LayerNormÚ
after_normr]   r^   r`   rj   r   r4   Úpre_lookahead_layerÚ
ModuleListÚrangeÚencodersr   Úup_layerÚup_embedÚup_encoders)r   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   Ú
activationr   rr   r   r   l   s^   
;ÿü
üüÿ 
ôÿü ôz!UpsampleConformerEncoder.__init__r    c                 C   s   | j S rm   )ru   )r   r   r   r   rS   ñ   s   z$UpsampleConformerEncoder.output_sizeéÿÿÿÿÚxsÚxs_lensr;   Údecoding_chunk_sizeÚnum_decoding_left_chunksÚ	streamingc                 C   s�  |  d¡}t||ƒ d¡ }| jdur|  |¡}|  ||¡\}}	}|  d¡dkrL| jdu s1J dƒ‚t dd|  d¡¡ |¡}
| j||
|  d¡d�\}}}|}t	||ddd|du r[| j
nddƒ}| j||d	�}|  |||	|¡}| dd
¡ ¡ }|  ||¡\}}| dd
¡ ¡ }|  d¡}t||ƒ d¡ }|  ||¡\}}	}|}t	||ddd|du r°| j
| jj nddƒ}|  |||	|¡}| jrÄ|  |¡}||fS )a�  Embed positions in tensor.

        Args:
            xs: padded input tensor (B, T, D)
            xs_lens: input length (B)
            decoding_chunk_size: decoding chunk size for dynamic chunk
                0: default for training, use random dynamic chunk.
                <0: for decoding, use full chunk.
                >0: for decoding, use fixed chunk size as set.
            num_decoding_left_chunks: number of left chunks, this is for decoding,
            the chunk size is decoding_chunk_size.
                >=0: use num_decoding_left_chunks
                <0: use all left chunks
        Returns:
            encoder output tensor xs, and subsampled masks
            xs: padded output tensor (B, T' ~= T/subsample_rate, D)
            masks: torch.Tensor batch padding mask after subsample
                (B, 1, T' ~= T/subsample_rate)
        NOTE(xcsong):
            We pass the `__call__` method of the modules instead of `forward` to the
            checkpointing API because `__call__` attaches all the hooks of the module.
            https://discuss.pytorch.org/t/any-different-between-model-input-and-model-forward-input/3690/2
        r   Nr   Fr=   )ÚoffsetTr�   )r;   r   )rA   r   Ú	unsqueezer_   rv   rB   r1   ÚonesÚtor   r]   ry   Úforward_layersr?   r@   r}   r~   r   Úforward_up_layersr\   rx   )r   r‚   rƒ   r;   r„   r…   r†   ÚTÚmasksÚpos_embZcontext_masksrq   Úmask_padÚchunk_masksr   r   r   r+   ô   s2   
 

"
*
z UpsampleConformerEncoder.forwardr‘   r�   r�   c                 C   ó&   | j D ]}|||||ƒ\}}}}q|S rm   )r|   ©r   r‚   r‘   r�   r�   Úlayerrq   r   r   r   r‹   5  ó   
z'UpsampleConformerEncoder.forward_layersc                 C   r’   rm   )r   r“   r   r   r   rŒ   <  r•   z*UpsampleConformerEncoder.forward_up_layers)rG   rH   rI   rJ   rK   rK   r$   rL   rM   Tr   FNFr   TrN   rO   TrP   FrQ   TF)r,   r-   r.   r0   r(   ÚstrÚboolr1   r   ÚModuler   rS   rE   r2   r   r+   r‹   rŒ   r3   r   r   r   r   rF   j   sÞ    æþýüûúùø	÷
öõôóòñðïîíìëêéèçæ ùþýüûúù
øAÿþ
þÿþþrF   )r/   Útypingr   r1   r   Útorch.nnr   r&   Ú!cosyvoice.transformer.convolutionr   Z#cosyvoice.transformer.encoder_layerr   Z/cosyvoice.transformer.positionwise_feed_forwardr   Úcosyvoice.utils.class_utilsr   r	   r
   r   Úcosyvoice.utils.maskr   r   r˜   r   r4   rF   r   r   r   r   Ú<module>   s   (