o
    óôj  ã                   @   sV   d Z ddlZddlmZmZmZ ddlmZ G dd„ dejjƒZG dd„ dejƒZ	dS )	z*Swish() activation function for Conformer.é    N)ÚnnÚsinÚpow)Ú	Parameterc                   @   s&   e Zd ZdZdejdejfdd„ZdS )ÚSwishzConstruct an Swish object.ÚxÚreturnc                 C   s   |t  |¡ S )z!Return Swish activation function.)ÚtorchÚsigmoid)Úselfr   © r   úV/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/transformer/activation.pyÚforward   s   zSwish.forwardN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   ÚTensorr   r   r   r   r   r      s    r   c                       s*   e Zd ZdZd	‡ fdd„	Zdd„ Z‡  ZS )
ÚSnakeaÓ  
    Implementation of a sine-based periodic activation function
    Shape:
        - Input: (B, C, T)
        - Output: (B, C, T), same shape as the input
    Parameters:
        - alpha - trainable parameter
    References:
        - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
        https://arxiv.org/abs/2006.08195
    Examples:
        >>> a1 = snake(256)
        >>> x = torch.randn(256)
        >>> x = a1(x)
    ç      ð?TFc                    s\   t t| ƒ ¡  || _|| _| jrtt |¡| ƒ| _n
tt 	|¡| ƒ| _|| j_
d| _dS )a   
        Initialization.
        INPUT:
            - in_features: shape of the input
            - alpha: trainable parameter
            alpha is initialized to 1 by default, higher values = higher-frequency.
            alpha will be trained along with the rest of your model.
        g•Ö&è.>N)Úsuperr   Ú__init__Úin_featuresÚalpha_logscaler   r	   ÚzerosÚalphaÚonesÚrequires_gradÚno_div_by_zero)r   r   r   Zalpha_trainabler   ©Ú	__class__r   r   r   2   s   	
zSnake.__init__c                 C   sJ   | j  d¡ d¡}| jrt |¡}|d|| j  tt|| ƒdƒ  }|S )uŽ   
        Forward pass of the function.
        Applies the function to the input elementwise.
        Snake âˆ¶= x + 1/a * sin^2 (xa)
        r   éÿÿÿÿr   é   )r   Ú	unsqueezer   r	   Úexpr   r   r   )r   r   r   r   r   r   r   I   s
   
$zSnake.forward)r   TF)r   r   r   r   r   r   Ú__classcell__r   r   r   r   r   "   s    r   )
r   r	   r   r   r   Útorch.nnr   ÚModuler   r   r   r   r   r   Ú<module>   s   
