o
    æôj   ã                   @   s  d dl Z d dlmZ d dlZd dlmZ d dlmZmZ ddlm	Z	 e	 
e¡ZG dd„ dejƒZG d	d
„ d
ejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ deƒZi de“d ed!d"d#œf“d$e“d%e“d&ed'd(if“d)e“d*e“d+e“d,ej“d-e“d.e“d/e“d0ej“d1e“d2ej“d3ej“d4ej“ejejej d5œ¥Z!ee!ƒZ"d6d7„ Z#e#d&ƒZ$e#d%ƒZ%e#dƒZ&e#d$ƒZ'e#d/ƒZ(e#d4ƒZ)e#d.ƒZ*e#d-ƒZ+dS )8é    N)ÚOrderedDict)Úversion)ÚTensorÚnné   )Úloggingc                       ó2   e Zd ZdZ‡ fdd„Zdedefdd„Z‡  ZS )ÚPytorchGELUTanha  
    A fast C implementation of the tanh approximation of the GeLU activation function. See
    https://arxiv.org/abs/1606.08415.

    This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical
    match due to rounding errors.
    c                    s6   t ƒ  ¡  t tj¡t d¡k rtdtj› d�ƒ‚d S )Nz1.12.0zYou are using torch==zM, but torch>=1.12.0 is required to use PytorchGELUTanh. Please upgrade torch.)ÚsuperÚ__init__r   ÚparseÚtorchÚ__version__ÚImportError©Úself©Ú	__class__© úk/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/transformers/activations.pyr   %   s   
ÿÿzPytorchGELUTanh.__init__ÚinputÚreturnc                 C   s   t jj|dd�S )NÚtanh)Úapproximate)r   Ú
functionalÚgelu©r   r   r   r   r   Úforward-   s   zPytorchGELUTanh.forward©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   Ú__classcell__r   r   r   r   r	      s    r	   c                   @   ó"   e Zd ZdZdedefdd„ZdS )ÚNewGELUActivationzÆ
    Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see
    the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
    r   r   c                 C   s6   d| dt  t dtj ¡|dt  |d¡   ¡  S )Nç      à?ç      ð?ç       @ç÷Hmâä¦?g      @)r   r   ÚmathÚsqrtÚpiÚpowr   r   r   r   r   7   s   6zNewGELUActivation.forwardN©r   r    r!   r"   r   r   r   r   r   r   r%   1   s    r%   c                       sL   e Zd ZdZddef‡ fdd„Zdedefdd	„Zdedefd
d„Z‡  Z	S )ÚGELUActivationa«  
    Original Implementation of the GELU activation function in Google BERT repo when initially created. For
    information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional
    Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
    FÚuse_gelu_pythonc                    s(   t ƒ  ¡  |r| j| _d S tjj| _d S ©N)r
   r   Ú_gelu_pythonÚactr   r   r   )r   r0   r   r   r   r   C   s   
zGELUActivation.__init__r   r   c                 C   s    |d dt  |t d¡ ¡  S )Nr&   r'   r(   )r   Úerfr*   r+   r   r   r   r   r2   J   s    zGELUActivation._gelu_pythonc                 C   ó
   |   |¡S r1   ©r3   r   r   r   r   r   M   ó   
zGELUActivation.forward)F)
r   r    r!   r"   Úboolr   r   r2   r   r#   r   r   r   r   r/   ;   s
    r/   c                   @   r$   )ÚFastGELUActivationz}
    Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs
    r   r   c                 C   s*   d| dt  |d dd| |   ¡  S )Nr&   r'   g€ÑÓ3Eˆé?r)   )r   r   r   r   r   r   r   V   s   *zFastGELUActivation.forwardNr.   r   r   r   r   r9   Q   ó    r9   c                   @   r$   )ÚQuickGELUActivationzr
    Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs
    r   r   c                 C   s   |t  d| ¡ S )Ng¬Zd;û?)r   Úsigmoidr   r   r   r   r   _   s   zQuickGELUActivation.forwardNr.   r   r   r   r   r;   Z   r:   r;   c                       s<   e Zd ZdZdedef‡ fdd„Zdedefdd	„Z‡  ZS )
ÚClippedGELUActivationa‚  
    Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as
    it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to
    https://arxiv.org/abs/2004.09602.

    Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when
    initially created.

    For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 +
    torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://arxiv.org/abs/1606.08415
    ÚminÚmaxc                    s8   ||krt d|› d|› d�ƒ‚tƒ  ¡  || _|| _d S )Nzmin should be < max (got min: z, max: ú))Ú
ValueErrorr
   r   r>   r?   )r   r>   r?   r   r   r   r   p   s
   

zClippedGELUActivation.__init__Úxr   c                 C   s   t  t|ƒ| j| j¡S r1   )r   Úclipr   r>   r?   )r   rB   r   r   r   r   x   ó   zClippedGELUActivation.forward)	r   r    r!   r"   Úfloatr   r   r   r#   r   r   r   r   r=   c   s    r=   c                       r   )ÚAccurateGELUActivationzÙ
    Applies GELU approximation that is faster than default and more accurate than QuickGELU. See:
    https://github.com/hendrycks/GELUs

    Implemented along with MEGA (Moving Average Equipped Gated Attention)
    c                    s    t ƒ  ¡  t dtj ¡| _d S )Né   )r
   r   r*   r+   r,   Úprecomputed_constantr   r   r   r   r   „   s   
zAccurateGELUActivation.__init__r   r   c                 C   s,   d| dt  | j|dt  |d¡   ¡  S )Nr&   r   r)   é   )r   r   rH   r-   r   r   r   r   r   ˆ   s   ,zAccurateGELUActivation.forwardr   r   r   r   r   rF   |   s    rF   c                       sD   e Zd ZdZ‡ fdd„Zdedefdd„Zdedefdd	„Z‡  ZS )
ÚMishActivationzÑ
    See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://arxiv.org/abs/1908.08681). Also
    visit the official repository for the paper: https://github.com/digantamisra98/Mish
    c                    s:   t ƒ  ¡  t tj¡t d¡k r| j| _d S tj	j
| _d S )Nz1.9.0)r
   r   r   r   r   r   Ú_mish_pythonr3   r   r   Úmishr   r   r   r   r   ’   s   
zMishActivation.__init__r   r   c                 C   s   |t  tj |¡¡ S r1   )r   r   r   r   Úsoftplusr   r   r   r   rK   ™   rD   zMishActivation._mish_pythonc                 C   r5   r1   r6   r   r   r   r   r   œ   r7   zMishActivation.forward)	r   r    r!   r"   r   r   rK   r   r#   r   r   r   r   rJ   Œ   s
    rJ   c                   @   r$   )ÚLinearActivationz[
    Applies the linear activation function, i.e. forwarding input directly to output.
    r   r   c                 C   s   |S r1   r   r   r   r   r   r   ¥   s   zLinearActivation.forwardNr.   r   r   r   r   rN       r:   rN   c                   @   s   e Zd ZdZddd„ZdS )ÚLaplaceActivationzó
    Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See
    https://arxiv.org/abs/2209.10655

    Inspired by squared relu, but with bounded range and gradient for better stability
    ç»¹øÛž æ?ç ^×/ØÒ?c                 C   s*   ||   |t d¡ ¡}ddt |¡  S )Nr(   r&   r'   )Údivr*   r+   r   r4   )r   r   ÚmuÚsigmar   r   r   r   ±   s   zLaplaceActivation.forwardN)rP   rQ   ©r   r    r!   r"   r   r   r   r   r   rO   ©   s    rO   c                   @   s   e Zd ZdZdd„ ZdS )ÚReLUSquaredActivationzX
    Applies the relu^2 activation introduced in https://arxiv.org/abs/2109.08668v2
    c                 C   s   t j |¡}t |¡}|S r1   )r   r   Úrelur   Úsquare)r   r   Zrelu_appliedÚsquaredr   r   r   r   »   s   
zReLUSquaredActivation.forwardNrU   r   r   r   r   rV   ¶   s    rV   c                       s   e Zd Z‡ fdd„Z‡  ZS )ÚClassInstantierc                    s4   t ƒ  |¡}t|tƒr|n|i f\}}|di |¤ŽS )Nr   )r
   Ú__getitem__Ú
isinstanceÚtuple)r   ÚkeyÚcontentÚclsÚkwargsr   r   r   r[   Â   s   zClassInstantier.__getitem__)r   r    r!   r[   r#   r   r   r   r   rZ   Á   s    rZ   r   Zgelu_10iöÿÿÿé
   )r>   r?   Ú	gelu_fastÚgelu_newÚgelu_pythonr0   TZgelu_pytorch_tanhZgelu_accurateÚlaplaceÚ
leaky_reluÚlinearrL   Ú
quick_gelurW   Úrelu2Úrelu6r<   Úsilu)Zswishr   Úpreluc                 C   s,   | t v rt |  S td| › dtt  ¡ ƒ› �ƒ‚)Nz	function z not found in ACT2FN mapping )ÚACT2FNÚKeyErrorÚlistÚkeys)Úactivation_stringr   r   r   Úget_activationá   s   rs   ),r*   Úcollectionsr   r   Ú	packagingr   r   r   Úutilsr   Ú
get_loggerr   ÚloggerÚModuler	   r%   r/   r9   r;   r=   rF   rJ   rN   rO   rV   rZ   Ú	LeakyReLUÚReLUÚReLU6ÚSigmoidÚSiLUÚTanhÚPReLUZACT2CLSrn   rs   re   rd   r   rc   ri   rl   rL   Z
linear_actr   r   r   r   Ú<module>   sˆ   

			ÿþýüûúùø	÷
öõôóòñðïì