o
    óôj‚%  ã                   @   sP   d Z ddlmZmZ ddlZddlmZ G dd„ dejƒZG dd„ dejƒZdS )	z(Encoder self-attention layer definition.é    )ÚOptionalÚTupleN)Únnc                       s¨   e Zd ZdZ	ddedejjdejjdede	f
‡ fdd	„Z
ejd
ej	d�e d¡e d¡fdejdejdejdejdejdejdeejejejejf fdd„Z‡  ZS )ÚTransformerEncoderLayera\  Encoder layer module.

    Args:
        size (int): Input dimension.
        self_attn (torch.nn.Module): Self-attention module instance.
            `MultiHeadedAttention` or `RelPositionMultiHeadedAttention`
            instance can be used as the argument.
        feed_forward (torch.nn.Module): Feed-forward module instance.
            `PositionwiseFeedForward`, instance can be used as the argument.
        dropout_rate (float): Dropout rate.
        normalize_before (bool):
            True: use layer_norm before each sub-block.
            False: to use layer_norm after each sub-block.
    TÚsizeÚ	self_attnÚfeed_forwardÚdropout_rateÚnormalize_beforec                    sR   t ƒ  ¡  || _|| _tj|dd�| _tj|dd�| _t |¡| _	|| _
|| _dS )ú!Construct an EncoderLayer object.çê-�™—q=©ÚepsN)ÚsuperÚ__init__r   r   r   Ú	LayerNormÚnorm1Únorm2ÚDropoutÚdropoutr   r
   )Úselfr   r   r   r	   r
   ©Ú	__class__© úY/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/transformer/encoder_layer.pyr   (   s   
	
z TransformerEncoderLayer.__init__©r   r   r   ©Údtype©r   r   r   r   ÚxÚmaskÚpos_embÚmask_padÚ	att_cacheÚ	cnn_cacheÚreturnc                 C   s¤   |}| j r
|  |¡}| j||||||d�\}}	||  |¡ }| j s&|  |¡}|}| j r0|  |¡}||  |  |¡¡ }| j sB|  |¡}tjd|j|j	d�}
|||	|
fS )uh  Compute encoded features.

        Args:
            x (torch.Tensor): (#batch, time, size)
            mask (torch.Tensor): Mask tensor for the input (#batch, timeï¼Œtime),
                (0, 0, 0) means fake mask.
            pos_emb (torch.Tensor): just for interface compatibility
                to ConformerEncoderLayer
            mask_pad (torch.Tensor): does not used in transformer layer,
                just for unified api with conformer.
            att_cache (torch.Tensor): Cache tensor of the KEY & VALUE
                (#batch=1, head, cache_t1, d_k * 2), head * d_k == size.
            cnn_cache (torch.Tensor): Convolution cache in conformer layer
                (#batch=1, size, cache_t2), not used here, it's for interface
                compatibility to ConformerEncoderLayer.
        Returns:
            torch.Tensor: Output tensor (#batch, time, size).
            torch.Tensor: Mask tensor (#batch, time, time).
            torch.Tensor: att_cache tensor,
                (#batch=1, head, cache_t1 + time, d_k * 2).
            torch.Tensor: cnn_cahce tensor (#batch=1, size, cache_t2).

        )r!   Úcacher   ©r   Údevice)
r
   r   r   r   r   r   ÚtorchÚzerosr   r(   )r   r   r    r!   r"   r#   r$   ÚresidualÚx_attÚnew_att_cacheZfake_cnn_cacher   r   r   Úforward:   s    



zTransformerEncoderLayer.forward)T)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr)   r   ÚModuleÚfloatÚboolr   Úonesr*   ÚTensorr   r.   Ú__classcell__r   r   r   r   r      s@    úþýüûúùþýüûúùør   c                       sÆ   e Zd ZdZ					ddedejjdeej deej d	eej d
e	de
f‡ fdd„Zejdej
d�e d¡e d¡fdejdejdejdejdejdejdeejejejejf fdd„Z‡  ZS )ÚConformerEncoderLayera’  Encoder layer module.
    Args:
        size (int): Input dimension.
        self_attn (torch.nn.Module): Self-attention module instance.
            `MultiHeadedAttention` or `RelPositionMultiHeadedAttention`
            instance can be used as the argument.
        feed_forward (torch.nn.Module): Feed-forward module instance.
            `PositionwiseFeedForward` instance can be used as the argument.
        feed_forward_macaron (torch.nn.Module): Additional feed-forward module
             instance.
            `PositionwiseFeedForward` instance can be used as the argument.
        conv_module (torch.nn.Module): Convolution module instance.
            `ConvlutionModule` instance can be used as the argument.
        dropout_rate (float): Dropout rate.
        normalize_before (bool):
            True: use layer_norm before each sub-block.
            False: use layer_norm after each sub-block.
    Nçš™™™™™¹?Tr   r   r   Úfeed_forward_macaronÚconv_moduler	   r
   c                    s®   t ƒ  ¡  || _|| _|| _|| _tj|dd�| _tj|dd�| _	|dur1tj|dd�| _
d| _nd| _| jdurItj|dd�| _tj|dd�| _t |¡| _|| _|| _dS )r   r   r   Ng      à?g      ð?)r   r   r   r   r<   r=   r   r   Únorm_ffÚnorm_mhaÚnorm_ff_macaronÚff_scaleÚ	norm_convÚ
norm_finalr   r   r   r
   )r   r   r   r   r<   r=   r	   r
   r   r   r   r   �   s&   

ÿ
zConformerEncoderLayer.__init__r   r   r   r   r    r!   r"   r#   r$   r%   c                 C   sR  | j dur$|}| jr|  |¡}|| j|  |   |¡¡  }| js$|  |¡}|}| jr.|  |¡}|  ||||||¡\}}	||  |¡ }| jsI|  |¡}tjd|j	|j
d�}
| jdurz|}| jrb|  |¡}|  |||¡\}}
||  |¡ }| jsz|  |¡}|}| jr„|  |¡}|| j|  |  |¡¡  }| js™|  |¡}| jdur£|  |¡}|||	|
fS )u   Compute encoded features.

        Args:
            x (torch.Tensor): (#batch, time, size)
            mask (torch.Tensor): Mask tensor for the input (#batch, timeï¼Œtime),
                (0, 0, 0) means fake mask.
            pos_emb (torch.Tensor): positional encoding, must not be None
                for ConformerEncoderLayer.
            mask_pad (torch.Tensor): batch padding mask used for conv module.
                (#batch, 1ï¼Œtime), (0, 0, 0) means fake mask.
            att_cache (torch.Tensor): Cache tensor of the KEY & VALUE
                (#batch=1, head, cache_t1, d_k * 2), head * d_k == size.
            cnn_cache (torch.Tensor): Convolution cache in conformer layer
                (#batch=1, size, cache_t2)
        Returns:
            torch.Tensor: Output tensor (#batch, time, size).
            torch.Tensor: Mask tensor (#batch, time, time).
            torch.Tensor: att_cache tensor,
                (#batch=1, head, cache_t1 + time, d_k * 2).
            torch.Tensor: cnn_cahce tensor (#batch, size, cache_t2).
        Nr   r'   )r<   r
   r@   rA   r   r?   r   r)   r*   r   r(   r=   rB   r>   r   rC   )r   r   r    r!   r"   r#   r$   r+   r,   r-   Znew_cnn_cacher   r   r   r.       sH   
 

ÿ

ÿ







zConformerEncoderLayer.forward)NNNr;   T)r/   r0   r1   r2   r3   r)   r   r4   r   r5   r6   r   r7   r*   r8   r   r.   r9   r   r   r   r   r:   m   sP    øþýüûúùø$ùþýüûúùør:   )	r2   Útypingr   r   r)   r   r4   r   r:   r   r   r   r   Ú<module>   s   U