o
    óôj58  ã                   @   sR   d Z ddlZddlmZ ddlZddlmZ G dd„ dejƒZG dd„ deƒZdS )	z&Multi-Head Attention layer definition.é    N)ÚTuple)Únnc                       sö   e Zd ZdZ	ddedededef‡ fdd„Zd	ej	d
ej	dej	de
ej	ej	ej	f fdd„Zejdejd�fdej	dej	dej	dej	fdd„Zejdejd�e d¡e d¡fd	ej	d
ej	dej	dej	dej	dej	de
ej	ej	f fdd„Z‡  ZS )ÚMultiHeadedAttentionz±Multi-Head Attention layer.

    Args:
        n_head (int): The number of heads.
        n_feat (int): The number of features.
        dropout_rate (float): Dropout rate.

    TÚn_headÚn_featÚdropout_rateÚkey_biasc                    sx   t ƒ  ¡  || dksJ ‚|| | _|| _t ||¡| _tj|||d�| _t ||¡| _t ||¡| _	tj
|d�| _dS )z)Construct an MultiHeadedAttention object.r   ©Úbias)ÚpN)ÚsuperÚ__init__Úd_kÚhr   ÚLinearÚlinear_qÚlinear_kÚlinear_vÚ
linear_outÚDropoutÚdropout©Úselfr   r   r   r   ©Ú	__class__© úU/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/transformer/attention.pyr   $   s   

zMultiHeadedAttention.__init__ÚqueryÚkeyÚvalueÚreturnc                 C   s†   |  d¡}|  |¡ |d| j| j¡}|  |¡ |d| j| j¡}|  |¡ |d| j| j¡}| dd¡}| dd¡}| dd¡}|||fS )aK  Transform query, key and value.

        Args:
            query (torch.Tensor): Query tensor (#batch, time1, size).
            key (torch.Tensor): Key tensor (#batch, time2, size).
            value (torch.Tensor): Value tensor (#batch, time2, size).

        Returns:
            torch.Tensor: Transformed query tensor, size
                (#batch, n_head, time1, d_k).
            torch.Tensor: Transformed key tensor, size
                (#batch, n_head, time2, d_k).
            torch.Tensor: Transformed value tensor, size
                (#batch, n_head, time2, d_k).

        r   éÿÿÿÿé   é   )Úsizer   Úviewr   r   r   r   Ú	transpose)r   r   r   r   Ún_batchÚqÚkÚvr   r   r   Úforward_qkv5   s   

z MultiHeadedAttention.forward_qkv©r   r   r   ©ÚdtypeÚscoresÚmaskc                 C   sÈ   |  d¡}|  d¡dkr<| d¡ d¡}|dd…dd…dd…d|  d¡…f }| |tdƒ ¡}tj|dd� |d¡}ntj|dd�}|  |¡}t ||¡}| 	dd¡ 
¡  |d| j| j ¡}|  |¡S )	a/  Compute attention context vector.

        Args:
            value (torch.Tensor): Transformed value, size
                (#batch, n_head, time2, d_k).
            scores (torch.Tensor): Attention score, size
                (#batch, n_head, time1, time2).
            mask (torch.Tensor): Mask, size (#batch, 1, time2) or
                (#batch, time1, time2), (0, 0, 0) means fake mask.

        Returns:
            torch.Tensor: Transformed value (#batch, time1, d_model)
                weighted by the attention score (#batch, time1, time2).

        r   r#   r"   Nr!   Úinf©Údimg        )r$   Ú	unsqueezeÚeqÚmasked_fillÚfloatÚtorchÚsoftmaxr   Úmatmulr&   Ú
contiguousr%   r   r   r   )r   r   r/   r0   r'   ÚattnZp_attnÚxr   r   r   Úforward_attentionR   s   
&ÿ

ÿ
z&MultiHeadedAttention.forward_attentionr   ©r   r   r   r   Úpos_embÚcachec                 C   s¨   |   |||¡\}}}	| d¡dkr2tj|| d¡d dd�\}
}tj|
|gdd�}tj||	gdd�}	tj||	fdd�}t || dd¡¡t | j	¡ }|  
|	||¡|fS )aÂ  Compute scaled dot product attention.

        Args:
            query (torch.Tensor): Query tensor (#batch, time1, size).
            key (torch.Tensor): Key tensor (#batch, time2, size).
            value (torch.Tensor): Value tensor (#batch, time2, size).
            mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
                (#batch, time1, time2).
                1.When applying cross attention between decoder and encoder,
                the batch padding mask for input is in (#batch, 1, T) shape.
                2.When applying self attention of encoder,
                the mask is in (#batch, T, T)  shape.
                3.When applying self attention of decoder,
                the mask is in (#batch, L, L)  shape.
                4.If the different position in decoder see different block
                of the encoder, such as Mocha, the passed in mask could be
                in (#batch, L, T) shape. But there is no such case in current
                CosyVoice.
            cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2),
                where `cache_t == chunk_size * num_decoding_left_chunks`
                and `head * d_k == size`


        Returns:
            torch.Tensor: Output tensor (#batch, time1, d_model).
            torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2)
                where `cache_t == chunk_size * num_decoding_left_chunks`
                and `head * d_k == size`

        r   r!   r#   r2   éþÿÿÿ)r+   r$   r8   ÚsplitÚcatr:   r&   ÚmathÚsqrtr   r>   )r   r   r   r   r0   r@   rA   r(   r)   r*   Ú	key_cacheÚvalue_cacheÚ	new_cacher/   r   r   r   Úforward�   s   '
þ zMultiHeadedAttention.forward©T)Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr7   Úboolr   r8   ÚTensorr   r+   Úonesr>   ÚemptyÚzerosrJ   Ú__classcell__r   r   r   r   r      sb    üÿþýüÿÿÿ
þ!üþýü
û4ùþýüûúùør   c                       sª   e Zd ZdZ	ddedededef‡ fdd„Zd	ej	d
ej	fdd„Z
ejdejd�e d¡e d¡fdej	dej	dej	dej	dej	dej	d
eej	ej	f fdd„Z‡  ZS )ÚRelPositionMultiHeadedAttentionzûMulti-Head Attention layer with relative position encoding.
    Paper: https://arxiv.org/abs/1901.02860
    Args:
        n_head (int): The number of heads.
        n_feat (int): The number of features.
        dropout_rate (float): Dropout rate.
    Tr   r   r   r   c                    sx   t ƒ  ||||¡ tj||dd�| _t t | j| j	¡¡| _
t t | j| j	¡¡| _tjj | j
¡ tjj | j¡ dS )z4Construct an RelPositionMultiHeadedAttention object.Fr	   N)r   r   r   r   Ú
linear_posÚ	Parameterr8   rR   r   r   Ú
pos_bias_uÚ
pos_bias_vÚinitÚxavier_uniform_r   r   r   r   r   Ñ   s   z(RelPositionMultiHeadedAttention.__init__r=   r    c                 C   sÄ   t j| ¡ d | ¡ d | ¡ d df|j|jd�}t j||gdd�}| | ¡ d | ¡ d | d¡d | d¡¡}|dd…dd…dd…f  |¡dd…dd…dd…d| d¡d d …f }|S )	zùCompute relative positional encoding.

        Args:
            x (torch.Tensor): Input tensor (batch, head, time1, 2*time1-1).
            time1 means the length of query vector.

        Returns:
            torch.Tensor: Output tensor.

        r   r"   r#   )Údevicer.   r!   r2   é   N)r8   rU   r$   r^   r.   rD   r%   Úview_as)r   r=   Zzero_padZx_paddedr   r   r   Ú	rel_shiftá   s   &þ
þ(ÿz)RelPositionMultiHeadedAttention.rel_shiftr,   r-   r   r?   r   r   r   r0   r@   rA   c                 C   s:  |   |||¡\}}}	| dd¡}| d¡dkr8tj|| d¡d dd�\}
}tj|
|gdd�}tj||	gdd�}	tj||	fdd�}| d¡}|  |¡ |d| j| j	¡}| dd¡}|| j
  dd¡}|| j  dd¡}t || dd¡¡}t || dd¡¡}|j|jkrŠ|  |¡}|| t | j	¡ }|  |	||¡|fS )aå  Compute 'Scaled Dot Product Attention' with rel. positional encoding.
        Args:
            query (torch.Tensor): Query tensor (#batch, time1, size).
            key (torch.Tensor): Key tensor (#batch, time2, size).
            value (torch.Tensor): Value tensor (#batch, time2, size).
            mask (torch.Tensor): Mask tensor (#batch, 1, time2) or
                (#batch, time1, time2), (0, 0, 0) means fake mask.
            pos_emb (torch.Tensor): Positional embedding tensor
                (#batch, time2, size).
            cache (torch.Tensor): Cache tensor (1, head, cache_t, d_k * 2),
                where `cache_t == chunk_size * num_decoding_left_chunks`
                and `head * d_k == size`
        Returns:
            torch.Tensor: Output tensor (#batch, time1, d_model).
            torch.Tensor: Cache tensor (1, head, cache_t + time1, d_k * 2)
                where `cache_t == chunk_size * num_decoding_left_chunks`
                and `head * d_k == size`
        r"   r#   r   r!   r2   rB   )r+   r&   r$   r8   rC   rD   rX   r%   r   r   rZ   r[   r:   Úshapera   rE   rF   r>   )r   r   r   r   r0   r@   rA   r(   r)   r*   rG   rH   rI   Zn_batch_posr   Zq_with_bias_uZq_with_bias_vZ	matrix_acZ	matrix_bdr/   r   r   r   rJ   ù   s.   
þ


ÿz'RelPositionMultiHeadedAttention.forwardrK   )rL   rM   rN   rO   rP   r7   rQ   r   r8   rR   ra   rS   rT   rU   r   rJ   rV   r   r   r   r   rW   È   s>    üÿþýüùþýüûúùørW   )	rO   rE   Útypingr   r8   r   ÚModuler   rW   r   r   r   r   Ú<module>   s    /