o
    /ôj U  ã                   @   s–  d Z ddlZddlZddlmZ ddlZddlmZ dFdd„ZdFdd„Z	dFd	d
„Z
dd„ Zdd„ ZdFdd„Z			dGdedededeej def
dd„Z			dGdedededeej def
dd„Z					dHdedededededeej defd d!„Zded"edefd#d$„Zdedefd%d&„Zdedefd'd(„Zd)d*„ ZdId,d-„Zd.d/„ Z	dJded0edeej defd1d2„Z		dJded0edeej defd3d4„Zd5d6„ Z		7	8	dKdeded9ed:edeej f
d;d<„Z		7	8	dKdeded9ed:edeej f
d=d>„Z	+	dLdeej fd?d@„Z	A	dMdeej fdBdC„Z dDdE„ Z!e!eƒZ"e!eƒZ#e!eƒZ$e!eƒZ%e!eƒZ&e!eƒZ'e!eƒZ(e!eƒZ)e!eƒZ*e!eƒZ+e!e ƒZ,dS )NzHThis file contains utilities for initializing neural network parameters.é    N)ÚTensor)ÚOptionalc                 C   ó<   t  ¡ � | j|||d�W  d   ƒ S 1 sw   Y  d S ©N©Ú	generator)ÚtorchÚno_gradÚuniform_©ÚtensorÚaÚbr   © r   ú^/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/nn/init.pyÚ_no_grad_uniform_   ó   
$ÿr   c                 C   r   r   )r   r	   Únormal_©r   ÚmeanÚstdr   r   r   r   Ú_no_grad_normal_   r   r   c           	      C   sÚ   dd„ }||d|  k s||d|  krt jddd� t ¡ �D ||| | ƒ}||| | ƒ}| jd| d d| d |d� |  ¡  |  |t d¡ ¡ |  	|¡ | j
||d	� | W  d   ƒ S 1 sfw   Y  d S )
Nc                 S   s   dt  | t  d¡ ¡ d S )Nç      ð?ç       @)ÚmathÚerfÚsqrt)Úxr   r   r   Únorm_cdf   s   z(_no_grad_trunc_normal_.<locals>.norm_cdfé   zjmean is more than 2 std from [a, b] in nn.init.trunc_normal_. The distribution of values may be incorrect.©Ú
stacklevelé   r   r   )ÚminÚmax)ÚwarningsÚwarnr   r	   r
   Úerfinv_Úmul_r   r   Úadd_Úclamp_)	r   r   r   r   r   r   r   ÚlÚur   r   r   Ú_no_grad_trunc_normal_   s    þ
 
$ër-   c                 C   s6   t  ¡ � |  |¡W  d   ƒ S 1 sw   Y  d S ©N)r   r	   Úfill_©r   Úvalr   r   r   Ú_no_grad_fill_;   s   
$ÿr2   c                 C   s4   t  ¡ � |  ¡ W  d   ƒ S 1 sw   Y  d S r.   )r   r	   Úzero_©r   r   r   r   Ú_no_grad_zero_@   s   
$ÿr5   c                 C   s´   g d¢}| |v s| dkrdS | dkrdS | dkrt  d¡S | dkrM|d	u r(d
}nt|tƒs2t|tƒs7t|tƒr:|}ntd|› d�ƒ‚t  dd|d   ¡S | dkrSdS td| › �ƒ‚)aÞ  Return the recommended gain value for the given nonlinearity function.

    The values are as follows:

    ================= ====================================================
    nonlinearity      gain
    ================= ====================================================
    Linear / Identity :math:`1`
    Conv{1,2,3}D      :math:`1`
    Sigmoid           :math:`1`
    Tanh              :math:`\frac{5}{3}`
    ReLU              :math:`\sqrt{2}`
    Leaky Relu        :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}`
    SELU              :math:`\frac{3}{4}`
    ================= ====================================================

    .. warning::
        In order to implement `Self-Normalizing Neural Networks`_ ,
        you should use ``nonlinearity='linear'`` instead of ``nonlinearity='selu'``.
        This gives the initial weights a variance of ``1 / N``,
        which is necessary to induce a stable fixed point in the forward pass.
        In contrast, the default gain for ``SELU`` sacrifices the normalization
        effect for more stable gradient flow in rectangular layers.

    Args:
        nonlinearity: the non-linear function (`nn.functional` name)
        param: optional parameter for the non-linear function

    Examples:
        >>> gain = nn.init.calculate_gain('leaky_relu', 0.2)  # leaky_relu with negative_slope=0.2

    .. _Self-Normalizing Neural Networks: https://papers.nips.cc/paper/2017/hash/5d44ee6f2c3f71b73125876103c8f6c4-Abstract.html
    )ÚlinearÚconv1dÚconv2dÚconv3dÚconv_transpose1dÚconv_transpose2dÚconv_transpose3dÚsigmoidr"   Útanhg«ªªªªªú?Úrelur   Ú
leaky_reluNç{®Gáz„?znegative_slope z not a valid numberr   Úselug      è?zUnsupported nonlinearity )r   r   Ú
isinstanceÚboolÚintÚfloatÚ
ValueError)ÚnonlinearityÚparamZ
linear_fnsÚnegative_sloper   r   r   Úcalculate_gainE   s"   "
rK   ç        r   r   r   r   r   Úreturnc                 C   ó4   t j | ¡rt jjt| f| |||d�S t| |||ƒS )a«  Fill the input Tensor with values drawn from the uniform distribution.

    :math:`\mathcal{U}(a, b)`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the lower bound of the uniform distribution
        b: the upper bound of the uniform distribution
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.uniform_(w)
    r   )r   Ú	overridesÚhas_torch_function_variadicÚhandle_torch_functionr
   r   r   r   r   r   r
   }   ó
   ÿr
   r   r   c                 C   rN   )aÁ  Fill the input Tensor with values drawn from the normal distribution.

    :math:`\mathcal{N}(\text{mean}, \text{std}^2)`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        mean: the mean of the normal distribution
        std: the standard deviation of the normal distribution
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.normal_(w)
    r   )r   rO   rP   rQ   r   r   r   r   r   r   r   ˜   rR   r   ç       Àr   c                 C   s   t | |||||d�S )a  Fill the input Tensor with values drawn from a truncated normal distribution.

    The values are effectively drawn from the
    normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`
    with values outside :math:`[a, b]` redrawn until they are within
    the bounds. The method used for generating the random values works
    best when :math:`a \leq \text{mean} \leq b`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        mean: the mean of the normal distribution
        std: the standard deviation of the normal distribution
        a: the minimum cutoff value
        b: the maximum cutoff value
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.trunc_normal_(w)
    r   )r-   )r   r   r   r   r   r   r   r   r   Útrunc_normal_²   s   rT   r1   c                 C   s,   t j | ¡rt jjt| f| |d�S t| |ƒS )zþFill the input Tensor with the value :math:`\text{val}`.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        val: the value to fill the tensor with

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.constant_(w, 0.3)
    r0   )r   rO   rP   rQ   Ú	constant_r2   r0   r   r   r   rU   Ñ   s   
rU   c                 C   s
   t | dƒS )z¾Fill the input Tensor with the scalar value `1`.

    Args:
        tensor: an n-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.ones_(w)
    r   )r2   r4   r   r   r   Úones_á   s   

rV   c                 C   s   t | ƒS )z¿Fill the input Tensor with the scalar value `0`.

    Args:
        tensor: an n-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.zeros_(w)
    )r5   r4   r   r   r   Úzeros_î   s   
rW   c                 C   sX   |   ¡ dkr
tdƒ‚t ¡ � tj| j| | jdœŽ W d  ƒ | S 1 s%w   Y  | S )a=  Fill the 2-dimensional input `Tensor` with the identity matrix.

    Preserves the identity of the inputs in `Linear` layers, where as
    many inputs are preserved as possible.

    Args:
        tensor: a 2-dimensional `torch.Tensor`

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.eye_(w)
    r   ú,Only tensors with 2 dimensions are supported)ÚoutÚrequires_gradN)Ú
ndimensionrG   r   r	   ÚeyeÚshaperZ   r4   r   r   r   Úeye_û   s   

ÿþr^   r"   c                 C   s<  |   ¡ }|dvrtdƒ‚|  ¡ }|d | dkrtdƒ‚|d | }t||d ƒ}t ¡ �g |  ¡  t|ƒD ]U}t|ƒD ]N}|dkrSd| || | ||  d¡d f< q<|dkrnd| || | ||  d¡d |  d¡d f< q<d| || | ||  d¡d |  d¡d |  d¡d f< q<q6W d	  ƒ | S 1 s—w   Y  | S )
aF  Fill the {3, 4, 5}-dimensional input `Tensor` with the Dirac delta function.

    Preserves the identity of the inputs in `Convolutional`
    layers, where as many input channels are preserved as possible. In case
    of groups>1, each group of channels preserves identity

    Args:
        tensor: a {3, 4, 5}-dimensional `torch.Tensor`
        groups (int, optional): number of groups in the conv layer (default: 1)
    Examples:
        >>> w = torch.empty(3, 16, 5, 5)
        >>> nn.init.dirac_(w)
        >>> w = torch.empty(3, 24, 5, 5)
        >>> nn.init.dirac_(w, 3)
    )é   é   é   z5Only tensors with 3, 4, or 5 dimensions are supportedr   z!dim 0 must be divisible by groupsr"   r_   r   r`   N)r[   rG   Úsizer#   r   r	   r3   Úrange)r   ÚgroupsÚ
dimensionsÚsizesZout_chans_per_grpZmin_dimÚgÚdr   r   r   Údirac_  s:   
"ÿÿÿÿùÿ
ýóri   c                 C   sp   |   ¡ }|dk rtdƒ‚|  d¡}|  d¡}d}|   ¡ dkr,| jdd … D ]}||9 }q%|| }|| }||fS )Nr   zNFan in and fan out can not be computed for tensor with fewer than 2 dimensionsr"   r   )ÚdimrG   rb   r]   )r   re   Znum_input_fmapsZnum_output_fmapsZreceptive_field_sizeÚsÚfan_inÚfan_outr   r   r   Ú_calculate_fan_in_and_fan_out<  s   


rn   Úgainc                 C   sD   t | ƒ\}}|t dt|| ƒ ¡ }t d¡| }t| | ||ƒS )aì  Fill the input `Tensor` with values using a Xavier uniform distribution.

    The method is described in `Understanding the difficulty of training
    deep feedforward neural networks` - Glorot, X. & Bengio, Y. (2010).
    The resulting tensor will have values sampled from
    :math:`\mathcal{U}(-a, a)` where

    .. math::
        a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}}

    Also known as Glorot initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        gain: an optional scaling factor
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.xavier_uniform_(w, gain=nn.init.calculate_gain('relu'))
    r   ç      @)rn   r   r   rF   r   )r   ro   r   rl   rm   r   r   r   r   r   Úxavier_uniform_O  s   rq   c                 C   s4   t | ƒ\}}|t dt|| ƒ ¡ }t| d||ƒS )aÔ  Fill the input `Tensor` with values using a Xavier normal distribution.

    The method is described in `Understanding the difficulty of training deep feedforward
    neural networks` - Glorot, X. & Bengio, Y. (2010). The resulting tensor
    will have values sampled from :math:`\mathcal{N}(0, \text{std}^2)` where

    .. math::
        \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}}

    Also known as Glorot initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        gain: an optional scaling factor
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.xavier_normal_(w)
    r   rL   )rn   r   r   rF   r   )r   ro   r   rl   rm   r   r   r   r   Úxavier_normal_n  s   rr   c                 C   sH   |  ¡ }ddg}||vrtd|› d|› �ƒ‚t| ƒ\}}|dkr"|S |S )Nrl   rm   zMode z" not supported, please use one of )ÚlowerrG   rn   )r   ÚmodeZvalid_modesrl   rm   r   r   r   Ú_calculate_correct_fan�  s   ru   rl   r@   rt   rH   c           	   	   C   s®   t j | ¡rt jjt| f| ||||d�S d| jv r t d¡ | S t| |ƒ}t	||ƒ}|t
 |¡ }t
 d¡| }t  ¡ � | j| ||d�W  d  ƒ S 1 sPw   Y  dS )að  Fill the input `Tensor` with values using a Kaiming uniform distribution.

    The method is described in `Delving deep into rectifiers: Surpassing
    human-level performance on ImageNet classification` - He, K. et al. (2015).
    The resulting tensor will have values sampled from
    :math:`\mathcal{U}(-\text{bound}, \text{bound})` where

    .. math::
        \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}}

    Also known as He initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the negative slope of the rectifier used after this layer (only
            used with ``'leaky_relu'``)
        mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
            preserves the magnitude of the variance of the weights in the
            forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
            backwards pass.
        nonlinearity: the non-linear function (`nn.functional` name),
            recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.kaiming_uniform_(w, mode='fan_in', nonlinearity='relu')
    )r   r   rt   rH   r   r   ú,Initializing zero-element tensors is a no-oprp   r   N)r   rO   rP   rQ   Úkaiming_uniform_r]   r%   r&   ru   rK   r   r   r	   r
   )	r   r   rt   rH   r   Úfanro   r   Úboundr   r   r   rw   —  s(   #ù
	



$ÿrw   c                 C   sv   d| j v rt d¡ | S t| |ƒ}t||ƒ}|t |¡ }t ¡ � | j	d||d�W  d  ƒ S 1 s4w   Y  dS )aØ  Fill the input `Tensor` with values using a Kaiming normal distribution.

    The method is described in `Delving deep into rectifiers: Surpassing
    human-level performance on ImageNet classification` - He, K. et al. (2015).
    The resulting tensor will have values sampled from
    :math:`\mathcal{N}(0, \text{std}^2)` where

    .. math::
        \text{std} = \frac{\text{gain}}{\sqrt{\text{fan\_mode}}}

    Also known as He initialization.

    Args:
        tensor: an n-dimensional `torch.Tensor`
        a: the negative slope of the rectifier used after this layer (only
            used with ``'leaky_relu'``)
        mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'``
            preserves the magnitude of the variance of the weights in the
            forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the
            backwards pass.
        nonlinearity: the non-linear function (`nn.functional` name),
            recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default).
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.kaiming_normal_(w, mode='fan_out', nonlinearity='relu')
    r   rv   r   N)
r]   r%   r&   ru   rK   r   r   r   r	   r   )r   r   rt   rH   r   rx   ro   r   r   r   r   Úkaiming_normal_Ï  s   
#



$ÿrz   c           
      C   sæ   |   ¡ dk r
tdƒ‚|  ¡ dkr| S |  d¡}|  ¡ | }|  ||¡jdd|d�}||k r1| ¡  tj 	|¡\}}t 
|d¡}| ¡ }	||	9 }||k rO| ¡  t ¡ � |  |¡ |¡ |  |¡ W d  ƒ | S 1 slw   Y  | S )a   Fill the input `Tensor` with a (semi) orthogonal matrix.

    Described in `Exact solutions to the nonlinear dynamics of learning in deep
    linear neural networks` - Saxe, A. et al. (2013). The input tensor must have
    at least 2 dimensions, and for tensors with more than 2 dimensions the
    trailing dimensions are flattened.

    Args:
        tensor: an n-dimensional `torch.Tensor`, where :math:`n \geq 2`
        gain: optional scaling factor
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK)
        >>> w = torch.empty(3, 5)
        >>> nn.init.orthogonal_(w)
    r   z4Only tensors with 2 or more dimensions are supportedr   r"   r   N)r[   rG   Únumelrb   Únewr   Út_r   ÚlinalgÚqrÚdiagÚsignr	   Úview_asÚcopy_r(   )
r   ro   r   ÚrowsÚcolsÚ	flattenedÚqÚrrh   Úphr   r   r   Úorthogonal_ü  s,   


þýrŠ   rA   c           
      C   sž   |   ¡ dkr
tdƒ‚| j\}}tt || ¡ƒ}t ¡ �) | jd||d� t	|ƒD ]}t 
|¡}|d|… }	d| |	|f< q)W d  ƒ | S 1 sHw   Y  | S )aŽ  Fill the 2D input `Tensor` as a sparse matrix.

    The non-zero elements will be drawn from the normal distribution
    :math:`\mathcal{N}(0, 0.01)`, as described in `Deep learning via
    Hessian-free optimization` - Martens, J. (2010).

    Args:
        tensor: an n-dimensional `torch.Tensor`
        sparsity: The fraction of elements in each column to be set to zero
        std: the standard deviation of the normal distribution used to generate
            the non-zero values
        generator: the torch Generator to sample from (default: None)

    Examples:
        >>> w = torch.empty(3, 5)
        >>> nn.init.sparse_(w, sparsity=0.1)
    r   rX   r   r   N)r[   rG   r]   rE   r   Úceilr   r	   r   rc   Úrandperm)
r   Zsparsityr   r   r„   r…   Z	num_zerosZcol_idxÚrow_indicesZzero_indicesr   r   r   Úsparse_/  s   


ý
þúrŽ   c                    sF   ˆ j ‰ˆd d… ‰‡ ‡‡fdd„}dˆ› dˆ› dˆ› d�|_ˆ|_ |S )Néÿÿÿÿc                     s,   t jdˆ› dˆ› d�tdd� ˆ | i |¤ŽS )Nz	`nn.init.z)` is now deprecated in favor of `nn.init.z`.r   r    )r%   r&   ÚFutureWarning)ÚargsÚkwargs©ÚmethÚnew_nameÚold_namer   r   Údeprecated_initZ  s   ýz(_make_deprecate.<locals>.deprecated_initz
    z_(...)

    .. warning::
        This method is now deprecated in favor of :func:`torch.nn.init.z"`.

    See :func:`~torch.nn.init.z` for details.)Ú__name__Ú__doc__)r”   r—   r   r“   r   Ú_make_deprecateV  s   ÿü
úrš   r.   )rL   r   N)rL   r   rS   r   N)r"   )r   N)r   rl   r@   N)r"   N)rA   N)-r™   r   r%   r   r   Útypingr   Ú	_Optionalr   r   r-   r2   r5   rK   rF   Ú	Generatorr
   r   rT   rU   rV   rW   r^   ri   rn   rq   rr   ru   Ústrrw   rz   rŠ   rŽ   rš   ÚuniformÚnormalÚconstantr\   ÚdiracÚxavier_uniformÚxavier_normalÚkaiming_uniformÚkaiming_normalÚ
orthogonalÚsparser   r   r   r   Ú<module>   s  


#
:üÿþýü
ûüÿþýü
ûúÿþýüûú
ù
,ÿÿÿÿ
þ!ýÿþý
üûÿþýü
û:ûÿþýü
û/ý
ý6ü
ü'