o
    šôj  ã                   @   sÐ   d dl Z d dlm  mZ d dl mZ ddlmZ ddlmZ eƒ r&d dl	Z	e 
¡ e 
¡ e ¡ e ¡ e ¡ dœZdedejfd	d
„ZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZdS )é    N)Únné   )Ú	deprecate)Úis_torch_npu_available)ÚswishÚsiluÚmishÚgeluÚreluÚact_fnÚreturnc                 C   s&   |   ¡ } | tv rt|  S td| › �ƒ‚)z°Helper function to get activation function from string.

    Args:
        act_fn (str): Name of activation function.

    Returns:
        nn.Module: Activation function.
    z!Unsupported activation function: )ÚlowerÚACTIVATION_FUNCTIONSÚ
ValueError)r   © r   úo/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/diffusers/models/activations.pyÚget_activation$   s   
r   c                       s6   e Zd ZdZ‡ fdd„Zdejdejfdd„Z‡  ZS )ÚFP32SiLUzH
    SiLU activation function with input upcasted to torch.float32.
    c                    s   t ƒ  ¡  d S ©N)ÚsuperÚ__init__)Úself©Ú	__class__r   r   r   :   s   zFP32SiLU.__init__Úinputsr   c                 C   s   t j| ¡ dd� |j¡S )NF)Úinplace)ÚFr   ÚfloatÚtoÚdtype)r   r   r   r   r   Úforward=   s   zFP32SiLU.forward)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚTensorr    Ú__classcell__r   r   r   r   r   5   s    r   c                	       sR   e Zd ZdZddedededef‡ fdd	„Zd
ej	dej	fdd„Z
dd„ Z‡  ZS )ÚGELUa¢  
    GELU activation function with tanh approximation support with `approximate="tanh"`.

    Parameters:
        dim_in (`int`): The number of channels in the input.
        dim_out (`int`): The number of channels in the output.
        approximate (`str`, *optional*, defaults to `"none"`): If `"tanh"`, use tanh approximation.
        bias (`bool`, defaults to True): Whether to use a bias in the linear layer.
    ÚnoneTÚdim_inÚdim_outÚapproximateÚbiasc                    s&   t ƒ  ¡  tj|||d�| _|| _d S ©N©r-   )r   r   r   ÚLinearÚprojr,   )r   r*   r+   r,   r-   r   r   r   r   L   s   

zGELU.__init__Úgater   c                 C   s@   |j jdkrtj|| jd�S tj|jtjd�| jd�j|jd�S )NÚmps)r,   ©r   )	ÚdeviceÚtyper   r	   r,   r   r%   Úfloat32r   ©r   r2   r   r   r   r	   Q   s   $z	GELU.geluc                 C   s   |   |¡}|  |¡}|S r   )r1   r	   )r   Úhidden_statesr   r   r   r    W   s   

zGELU.forward)r)   T)r!   r"   r#   r$   ÚintÚstrÚboolr   r%   r&   r	   r    r'   r   r   r   r   r(   A   s
     
r(   c                       sN   e Zd ZdZddededef‡ fdd„Zdejd	ejfd
d„Z	dd„ Z
‡  ZS )ÚGEGLUaF  
    A [variant](https://arxiv.org/abs/2002.05202) of the gated linear unit activation function.

    Parameters:
        dim_in (`int`): The number of channels in the input.
        dim_out (`int`): The number of channels in the output.
        bias (`bool`, defaults to True): Whether to use a bias in the linear layer.
    Tr*   r+   r-   c                    s$   t ƒ  ¡  tj||d |d�| _d S )Nr   r/   ©r   r   r   r0   r1   ©r   r*   r+   r-   r   r   r   r   g   s   
zGEGLU.__init__r2   r   c                 C   s4   |j jdkrt |¡S t |jtjd�¡j|jd�S )Nr3   r4   )r5   r6   r   r	   r   r%   r7   r   r8   r   r   r   r	   k   s   
z
GEGLU.geluc                 O   sp   t |ƒdks| dd ¡d urd}tdd|ƒ |  |¡}tƒ r(tj|ddd�d S |jddd	�\}}||  |¡ S )
Nr   ÚscalezðThe `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.z1.0.0éÿÿÿÿé   )Údimr,   r   )rC   )	ÚlenÚgetr   r1   r   Ú	torch_npuZ	npu_gegluÚchunkr	   )r   r9   ÚargsÚkwargsZdeprecation_messager2   r   r   r   r    q   s   
zGEGLU.forward©T)r!   r"   r#   r$   r:   r<   r   r%   r&   r	   r    r'   r   r   r   r   r=   ]   s
    	r=   c                       sF   e Zd ZdZddededef‡ fdd„Zdejd	ejfd
d„Z	‡  Z
S )ÚApproximateGELUa€  
    The approximate form of the Gaussian Error Linear Unit (GELU). For more details, see section 2 of this
    [paper](https://arxiv.org/abs/1606.08415).

    Parameters:
        dim_in (`int`): The number of channels in the input.
        dim_out (`int`): The number of channels in the output.
        bias (`bool`, defaults to True): Whether to use a bias in the linear layer.
    Tr*   r+   r-   c                    s    t ƒ  ¡  tj|||d�| _d S r.   r>   r?   r   r   r   r   ‰   s   
zApproximateGELU.__init__Úxr   c                 C   s   |   |¡}|t d| ¡ S )Ng¬Zd;û?)r1   r%   Úsigmoid)r   rL   r   r   r   r    �   s   
zApproximateGELU.forwardrJ   )r!   r"   r#   r$   r:   r<   r   r%   r&   r    r'   r   r   r   r   rK   ~   s    
rK   )r%   Útorch.nn.functionalr   Ú
functionalr   Úutilsr   Zutils.import_utilsr   rF   ÚSiLUÚMishr(   ÚReLUr   r;   ÚModuler   r   r=   rK   r   r   r   r   Ú<module>   s$   û	!