o
    ŒôjS  ã                   @   s  d dl Z d dl mZmZ d dlm  mZ d dlmZ d dlm	Z	 dd„ Z
dd„ Zd	d
„ ZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZdS )é    N)ÚnnÚeinsum©Ú	rearrange)Ú	Rearrangec                 C   s   | d uS ©N© )Úvalr   r   úf/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/conformer/conformer.pyÚexists
   s   r   c                 C   s   t | ƒr| S |S r   )r   )r	   Údr   r   r
   Údefault   s   r   c                 C   s   | d }||| d d  fS )Né   é   r   )Úkernel_sizeÚpadr   r   r
   Úcalc_same_padding   s   r   c                   @   s   e Zd Zdd„ ZdS )ÚSwishc                 C   s   ||  ¡  S r   )Úsigmoid©ÚselfÚxr   r   r
   Úforward   s   zSwish.forwardN)Ú__name__Ú
__module__Ú__qualname__r   r   r   r   r
   r      s    r   c                       ó$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚGLUc                    s   t ƒ  ¡  || _d S r   )ÚsuperÚ__init__Údim)r   r    ©Ú	__class__r   r
   r      s   

zGLU.__init__c                 C   s    |j d| jd�\}}|| ¡  S )Nr   ©r    )Úchunkr    r   )r   r   ÚoutÚgater   r   r
   r      s   zGLU.forward©r   r   r   r   r   Ú__classcell__r   r   r!   r
   r      s    r   c                       r   )ÚDepthWiseConv1dc                    s(   t ƒ  ¡  || _tj||||d�| _d S )N)Úgroups)r   r   Úpaddingr   ÚConv1dÚconv)r   Zchan_inZchan_outr   r+   r!   r   r
   r   $   s   
zDepthWiseConv1d.__init__c                 C   s   t  || j¡}|  |¡S r   )ÚFr   r+   r-   r   r   r   r
   r   )   s   
zDepthWiseConv1d.forwardr'   r   r   r!   r
   r)   #   ó    r)   c                       r   )ÚScalec                    s   t ƒ  ¡  || _|| _d S r   )r   r   ÚfnÚscale)r   r2   r1   r!   r   r
   r   0   s   

zScale.__init__c                 K   s   | j |fi |¤Ž| j S r   )r1   r2   ©r   r   Úkwargsr   r   r
   r   5   s   zScale.forwardr'   r   r   r!   r
   r0   /   r/   r0   c                       r   )ÚPreNormc                    s    t ƒ  ¡  || _t |¡| _d S r   )r   r   r1   r   Ú	LayerNormÚnorm)r   r    r1   r!   r   r
   r   9   s   
zPreNorm.__init__c                 K   s   |   |¡}| j|fi |¤ŽS r   )r7   r1   r3   r   r   r
   r   >   s   
zPreNorm.forwardr'   r   r   r!   r
   r5   8   r/   r5   c                       s6   e Zd Z				d
‡ fdd„	Z			ddd	„Z‡  ZS )Ú	Attentioné   é@   ç        é   c                    s„   t ƒ  ¡  || }|| _|d | _tj||dd�| _tj||d dd�| _t ||¡| _|| _	t 
d| d |¡| _t |¡| _d S )Ng      à¿F)Úbiasr   r   )r   r   Úheadsr2   r   ÚLinearÚto_qÚto_kvÚto_outÚmax_pos_embÚ	EmbeddingÚrel_pos_embÚDropoutÚdropout)r   r    r>   Údim_headrG   rC   Ú	inner_dimr!   r   r
   r   C   s   

zAttention.__init__Nc                    s–  ˆj d ˆj| j| jtˆ ƒf\}‰‰}}tˆ ˆƒ‰ |  ˆ¡g|  ˆ ¡jddd�¢R \}}	}
t	‡fdd„||	|
fƒ\}}	}
t
d||	ƒ| j }tj|ˆd�}t|d	ƒt|d
ƒ }| | |¡| }|  |¡ |¡}t
d||ƒ| j }|| }t|ƒs|t|ƒr°t|‡‡fdd„ƒ}|s�t||ƒn	t|‡ ‡fdd„ƒ}t |j¡j }t|dƒt|dƒ }| | |¡ |jdd�}t
d||
ƒ}t|dƒ}|  |¡}|  |¡S )Néþÿÿÿr   éÿÿÿÿr#   c                    s   t | dˆ d�S )Nzb n (h d) -> b h n d©Úhr   )ÚtrL   r   r
   Ú<lambda>c   s    z#Attention.forward.<locals>.<lambda>zb h i d, b h j d -> b h i j)Údevicez	i -> i ()z	j -> () jzb h n d, n r d -> b h n rc                      s   t jˆjd d… dˆ iŽS ©Nr   rP   ©ÚtorchÚonesÚshaper   )rP   r   r   r
   rO   q   ó    c                      s   t jˆ jd d… dˆiŽS rQ   rR   r   )ÚcontextrP   r   r
   rO   r   rV   zb i -> b () i ()zb j -> b () () jzb h i j, b h j d -> b h i dzb h n d -> b n (h d))rU   rP   r>   rC   r   r   r@   rA   r$   Úmapr   r2   rS   Úaranger   ÚclamprE   ÚtoÚfinfoÚdtypeÚmaxÚmasked_fill_ÚsoftmaxrB   rG   )r   r   rW   ÚmaskÚcontext_maskÚnrC   Úhas_contextÚqÚkÚvÚdotsÚseqÚdistrE   Zpos_attnÚ
mask_valueÚattnr%   r   )rW   rP   rM   r   r
   r   X   s,   (
("


zAttention.forward)r9   r:   r;   r<   )NNNr'   r   r   r!   r
   r8   B   s    úûr8   c                       s*   e Zd Z		d‡ fdd„	Zdd„ Z‡  ZS )ÚFeedForwardé   r;   c              	      sH   t ƒ  ¡  t t ||| ¡tƒ t |¡t || |¡t |¡¡| _d S r   )r   r   r   Ú
Sequentialr?   r   rF   Únet)r   r    ÚmultrG   r!   r   r
   r      s   

ûzFeedForward.__init__c                 C   ó
   |   |¡S r   ©rp   r   r   r   r
   r   Ž   ó   
zFeedForward.forward)rn   r;   r'   r   r   r!   r
   rm   ~   s
    ürm   c                       s.   e Zd Z				d	‡ fdd„	Zdd„ Z‡  ZS )
ÚConformerConvModuleFr   é   r;   c                    sž   t ƒ  ¡  || }|st|ƒn|d df}t t |¡tdƒt ||d d¡tdd�t	||||d�|s8t 
|¡nt ¡ tƒ t ||d¡tdƒt |¡¡
| _d S )Nr   r   zb n c -> b c nr   r#   )r   r+   zb c n -> b n c)r   r   r   r   ro   r6   r   r,   r   r)   ÚBatchNorm1dÚIdentityr   rF   rp   )r   r    ÚcausalÚexpansion_factorr   rG   rI   r+   r!   r   r
   r   ’   s   

özConformerConvModule.__init__c                 C   rr   r   rs   r   r   r   r
   r   ¬   rt   zConformerConvModule.forward)Fr   rv   r;   r'   r   r   r!   r
   ru   ‘   s    úru   c                
       s<   e Zd Zddddddddddœ	‡ fd	d
„
Zddd„Z‡  ZS )ÚConformerBlockr:   r9   rn   r   rv   r;   F©	rH   r>   Úff_multÚconv_expansion_factorÚconv_kernel_sizeÚattn_dropoutÚ
ff_dropoutÚconv_dropoutÚconv_causalc       
            s–   t ƒ  ¡  t|||d�| _t||||d�| _t||
|||	d�| _t|||d�| _t	|| jƒ| _t
dt	|| jƒƒ| _t
dt	|| jƒƒ| _t |¡| _d S )N)r    rq   rG   )r    rH   r>   rG   )r    ry   rz   r   rG   g      à?)r   r   rm   Úff1r8   rl   ru   r-   Úff2r5   r0   r   r6   Ú	post_norm)r   r    rH   r>   r}   r~   r   r€   r�   r‚   rƒ   r!   r   r
   r   ²   s   
zConformerBlock.__init__Nc                 C   sJ   |   |¡| }| j||d�| }|  |¡| }|  |¡| }|  |¡}|S )N)ra   )r„   rl   r-   r…   r†   )r   r   ra   r   r   r
   r   Ì   s   
zConformerBlock.forwardr   r'   r   r   r!   r
   r{   ±   s    ôr{   c                
       s:   e Zd Zddddddddddœ	‡ fd	d
„
Zdd„ Z‡  ZS )Ú	Conformerr:   r9   rn   r   rv   r;   Fr|   c       
            sL   t ƒ  ¡  || _t g ¡| _t|ƒD ]}| j t|||||||d�¡ qd S )N)r    rH   r>   r}   r~   r   rƒ   )	r   r   r    r   Ú
ModuleListÚlayersÚrangeÚappendr{   )r   r    ÚdepthrH   r>   r}   r~   r   r€   r�   r‚   rƒ   Ú_r!   r   r
   r   ×   s   

ùÿzConformer.__init__c                 C   s   | j D ]}||ƒ}q|S r   )r‰   )r   r   Úblockr   r   r
   r   ö   s   

zConformer.forwardr'   r   r   r!   r
   r‡   Ö   s    ór‡   )rS   r   r   Útorch.nn.functionalÚ
functionalr.   Úeinopsr   Zeinops.layers.torchr   r   r   r   ÚModuler   r   r)   r0   r5   r8   rm   ru   r{   r‡   r   r   r   r
   Ú<module>   s$    		
< %