o
    ²ƒðj;  ã                   @   s”   d dl Z d dlZd dlZd dlmZ d dlmZ ddlmZ ddl	m
Z
 G dd„ dejƒZG d	d
„ d
ejƒZG dd„ dejƒZG dd„ dejƒZdS )é    N)Únn)Ú
functionalé   )Úsubsequent_mask)Ú	LayerNormc                       sJ   e Zd Z			ddededededed	ed
ef‡ fdd„Zdd„ Z‡  ZS )ÚEncoderr   ç        é   Úhidden_channelsÚfilter_channelsÚn_headsÚn_layersÚkernel_sizeÚ	p_dropoutÚwindow_sizec           
   
      sÐ   t ƒ  ¡  || _|| _|| _|| _|| _|| _|| _t	 
|¡| _t	 ¡ | _t	 ¡ | _t	 ¡ | _t	 ¡ | _t| jƒD ],}	| j t|||||d�¡ | j t|ƒ¡ | j t|||||d�¡ | j t|ƒ¡ q9d S )N)r   r   ©r   )ÚsuperÚ__init__r
   r   r   r   r   r   r   r   ÚDropoutÚdropÚ
ModuleListÚattn_layersÚnorm_layers_1Ú
ffn_layersÚnorm_layers_2ÚrangeÚappendÚMultiHeadAttentionr   ÚFFN)
Úselfr
   r   r   r   r   r   r   ÚkwargsÚi©Ú	__class__© ú�/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/piper/train/vits/attentions.pyr      sF   




ûÿ	ûÿ	ìzEncoder.__init__c           	      C   sŽ   |  d¡|  d¡ }|| }t| j| j| j| jƒD ]'\}}}}||||ƒ}|  |¡}||| ƒ}|||ƒ}|  |¡}||| ƒ}q|| }|S )Né   éÿÿÿÿ)Ú	unsqueezeÚzipr   r   r   r   r   )	r   ÚxÚx_maskÚ	attn_maskÚ
attn_layerÚnorm_layer_1Ú	ffn_layerÚnorm_layer_2Úyr$   r$   r%   Úforward<   s   ÿ


zEncoder.forward)r   r   r	   )Ú__name__Ú
__module__Ú__qualname__ÚintÚfloatr   r2   Ú__classcell__r$   r$   r"   r%   r      s(    øþýüûúùø/r   c                       sP   e Zd Z				ddedededed	ed
ededef‡ fdd„Zdd„ Z‡  ZS )ÚDecoderr   r   FTr
   r   r   r   r   r   Úproximal_biasÚproximal_initc	                    s  t ƒ  ¡  || _|| _|| _|| _|| _|| _|| _|| _	t
 |¡| _t
 ¡ | _t
 ¡ | _t
 ¡ | _t
 ¡ | _t
 ¡ | _t
 ¡ | _t| jƒD ]B}
| j t||||||d�¡ | j t|ƒ¡ | j t||||d�¡ | j t|ƒ¡ | j t|||||dd�¡ | j t|ƒ¡ qFd S )N)r   r:   r;   r   T)r   Úcausal)r   r   r
   r   r   r   r   r   r:   r;   r   r   r   r   Úself_attn_layersÚnorm_layers_0Úencdec_attn_layersr   r   r   r   r   r   r   r   )r   r
   r   r   r   r   r   r:   r;   r    r!   r"   r$   r%   r   N   s\   






úÿ
ÿÿúÿ
äzDecoder.__init__c           	      C   sÔ   t | d¡ƒ |¡}| d¡| d¡ }|| }t| jƒD ]F}| j| |||ƒ}|  |¡}| j| || ƒ}| j	| |||ƒ}|  |¡}| j
| || ƒ}| j| ||ƒ}|  |¡}| j| || ƒ}q|| }|S )z<
        x: decoder input
        h: encoder output
        r&   r'   )r   ÚsizeÚtype_asr(   r   r   r=   r   r>   r?   r   r   r   )	r   r*   r+   ÚhÚh_maskÚself_attn_maskÚencdec_attn_maskr!   r1   r$   r$   r%   r2   ‰   s   


zDecoder.forward)r   r   FT)	r3   r4   r5   r6   r7   Úboolr   r2   r8   r$   r$   r"   r%   r9   M   s.    ÷þýüûúùø	÷;r9   c                       s¬   e Zd Z						d!dedededed	eje d
edeje dedef‡ fdd„Zd"dd„Z	d"dd„Z
dd„ Zdd„ Zdefdd„Zdd„ Zdd„ Zdefdd „Z‡  ZS )#r   r   NTFÚchannelsÚout_channelsr   r   r   Úheads_shareÚblock_lengthr:   r;   c
                    s¢  t ƒ  ¡  || dksJ ‚|| _|| _|| _|| _|| _|| _|| _|| _	|	| _
t d¡| _|| | _t ||d¡| _t ||d¡| _t ||d¡| _t ||d¡| _t |¡| _|d urŒ|radn|}
| jd }t t |
|d d | j¡| ¡| _t t |
|d d | j¡| ¡| _tj | jj¡ tj | jj¡ tj | jj¡ |	rÏt ¡ � | jj | jj¡ | jj  | jj ¡ W d   ƒ d S 1 sÈw   Y  d S d S )Nr   r   g      à¿r&   )!r   r   rG   rH   r   r   r   rI   rJ   r:   r;   ÚtorchÚzerosÚattnÚ
k_channelsr   ÚConv1dÚconv_qÚconv_kÚconv_vÚconv_or   r   Ú	ParameterÚrandnÚ	emb_rel_kÚ	emb_rel_vÚinitÚxavier_uniform_ÚweightÚno_gradÚcopy_Úbias)r   rG   rH   r   r   r   rI   rJ   r:   r;   Ún_heads_relÚ
rel_stddevr"   r$   r%   r   ¢   sP   


ÿÿÿÿ
"þÿzMultiHeadAttention.__init__c                 C   sD   |   |¡}|  |¡}|  |¡}| j||||d�\}| _|  |¡}|S )N)Úmask)rP   rQ   rR   Ú	attentionrM   rS   )r   r*   Úcr,   ÚqÚkÚvr$   r$   r%   r2   ×   s   



zMultiHeadAttention.forwardc                 C   sò  |  d¡|  d¡|  d¡|  d¡f\}}}}| || j| j|¡ dd¡}| || j| j|¡ dd¡}| || j| j|¡ dd¡}t |t | j¡ | dd¡¡}	| j	d ury||ks]J dƒ‚|  
| j|¡}
|  |t | j¡ |
¡}|  |¡}|	| }	| jrŽ||ks„J dƒ‚|	|  |¡ |	¡ }	|d ur½|	 |dkd	¡}	| jd ur½||ks§J d
ƒ‚t |	¡ | j ¡ | j¡}|	 |dkd	¡}	tj|	dd�}|  |¡}t ||¡}| j	d urè|  |¡}|  
| j|¡}||  ||¡ }| dd¡ ¡  |||¡}||fS )Nr   r   r&   é   éþÿÿÿr'   z8Relative attention is only available for self-attention.z3Proximal bias is only available for self-attention.g     ˆÃÀz5Local attention is only available for self-attention.)Údim)r@   Úviewr   rN   Ú	transposerK   ÚmatmulÚmathÚsqrtr   Ú_get_relative_embeddingsrV   Ú_matmul_with_relative_keysÚ'_relative_position_to_absolute_positionr:   Ú_attention_bias_proximalrA   Úmasked_fillrJ   Ú	ones_likeÚtriuÚtrilÚFÚsoftmaxr   Ú'_absolute_position_to_relative_positionrW   Ú_matmul_with_relative_valuesÚ
contiguous)r   ÚqueryÚkeyÚvaluer`   ÚbÚdÚt_sÚt_tÚscoresÚkey_relative_embeddingsÚ
rel_logitsÚscores_localÚ
block_maskÚp_attnÚoutputÚrelative_weightsÚvalue_relative_embeddingsr$   r$   r%   ra   á   sV   , 

ÿÿ


ÿ
ý


ÿÿÿzMultiHeadAttention.attentionc                 C   s   t  || d¡¡}|S )zU
        x: [b, h, l, m]
        y: [h or 1, m, d]
        ret: [b, h, l, d]
        r   )rK   rk   r(   ©r   r*   r1   Úretr$   r$   r%   ry     s   z/MultiHeadAttention._matmul_with_relative_valuesc                 C   s   t  || d¡ dd¡¡}|S )zU
        x: [b, h, l, d]
        y: [h or 1, m, d]
        ret: [b, h, l, m]
        r   rg   r'   )rK   rk   r(   rj   r‹   r$   r$   r%   ro     s   z-MultiHeadAttention._matmul_with_relative_keysÚlengthc              	   C   sv   t || jd  dƒ}t | jd | dƒ}|d|  d }|dkr-t |dd||ddf¡}n|}|d d …||…f }|S )Nr   r   r&   )Úmaxr   rv   Úpad)r   Úrelative_embeddingsr�   Ú
pad_lengthÚslice_start_positionÚslice_end_positionÚpadded_relative_embeddingsÚused_relative_embeddingsr$   r$   r%   rn   $  s   ýÿz+MultiHeadAttention._get_relative_embeddingsc              	   C   s’   |  ¡ \}}}}t |d¡}| |||d | g¡}t |d|d ddddf¡}| |||d d| d g¡dd…dd…d|…|d d…f }|S )z?
        x: [b, h, l, 2*l-1]
        ret: [b, h, l, l]
        )r   r   r   r   r   r   r   r   r&   r   r   N©r@   rv   r�   ri   ©r   r*   ÚbatchÚheadsr�   Ú_Úx_flatÚx_finalr$   r$   r%   rp   7  s   ÿz:MultiHeadAttention._relative_position_to_absolute_positionc                 C   sž   |  ¡ \}}}}t |d|d ddddddf¡}| |||| ||d   g¡}t ||dddddf¡}| |||d| g¡dd…dd…dd…dd…f }|S )z?
        x: [b, h, l, l]
        ret: [b, h, l, 2*l-1]
        r   r   r&   Nr–   r—   r$   r$   r%   rx   M  s     2z:MultiHeadAttention._absolute_position_to_relative_positionc              	   C   sJ   t j|t jd�}t  |d¡t  |d¡ }t  t  t  t  |¡¡ d¡d¡S )zÄBias for self-attention to encourage attention to close positions.
        Args:
          length: an integer scalar.
        Returns:
          a Tensor with shape [1, 1, length, length]
        )Údtyper   r   )rK   ÚarangeÚfloat32r(   Úlog1pÚabs)r   r�   ÚrÚdiffr$   r$   r%   rq   ^  s   "z+MultiHeadAttention._attention_bias_proximal)r   NTNFF©N)r3   r4   r5   r6   r7   ÚtypingÚOptionalrF   r   r2   ra   ry   ro   rn   rp   rx   rq   r8   r$   r$   r"   r%   r   ¡   sD    öþýüûúùø	÷
ö
5

1		r   c                       sZ   e Zd Z			ddededededed	ed
ef‡ fdd„Zdd„ Zdd„ Z	dd„ Z
‡  ZS )r   r   Ú FÚin_channelsrH   r   r   r   Ú
activationr<   c                    sd   t ƒ  ¡  || _|| _|| _|| _|| _|| _|| _t	 
|||¡| _t	 
|||¡| _t	 |¡| _d S r¤   )r   r   r¨   rH   r   r   r   r©   r<   r   rO   Úconv_1Úconv_2r   r   )r   r¨   rH   r   r   r   r©   r<   r"   r$   r%   r   k  s   

zFFN.__init__c                 C   s–   | j r|  || ¡}n|  || ¡}|  |¡}| jdkr&|t d| ¡ }nt |¡}|  |¡}| j r;|  || ¡}n|  || ¡}|  	|¡}|| S )NÚgelug¬Zd;û?)
r<   Ú_causal_paddingÚ_same_paddingrª   r©   rK   ÚsigmoidÚrelur   r«   )r   r*   r+   Úpadding1Úpadding2r$   r$   r%   r2   ‚  s   




zFFN.forwardc              	   C   s8   | j dkr|S | j d }d}t |||ddddf¡}|S )Nr   r   ©r   rv   r�   ©r   r*   Úpad_lÚpad_rr$   r$   r%   r­   ™  s   

zFFN._causal_paddingc              	   C   sB   | j dkr|S | j d d }| j d }t |||ddddf¡}|S )Nr   r&   r   r³   r´   r$   r$   r%   r®   £  s   

zFFN._same_padding)r   r§   F)r3   r4   r5   r6   r7   ÚstrrF   r   r2   r­   r®   r8   r$   r$   r"   r%   r   j  s,    øþýüûúùø
r   )rl   r¥   rK   r   Útorch.nnr   rv   Úcommonsr   Úmodulesr   ÚModuler   r9   r   r   r$   r$   r$   r%   Ú<module>   s    AT J