o
    !ôj+8  ã                   @   s–  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	 d dl
mZmZ ddgZG dd„ deƒZd	d
e› de› de› d� e_				d"dee dee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dededededededededefd d!„ZdS )#é    N)ÚTensoré   )Ú	OptimizerÚ_default_to_fused_or_foreachÚ_use_grad_for_differentiableÚ_differentiable_docÚ_foreach_docÚ_maximize_docÚ_view_as_real)ÚListÚOptionalÚRMSpropÚrmspropc                       sd   e Zd Z									ddee ded	ef‡ fd
d„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   ç{®Gáz„?ç®Gáz®ï?ç:Œ0âŽyE>r   FNÚforeachÚmaximizeÚdifferentiablec                    sš   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|ks!t d|› �ƒ‚d|ks,t d|› �ƒ‚d|ks7t d|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ d S )Ng        zInvalid learning rate: zInvalid epsilon value: zInvalid momentum value: zInvalid weight_decay value: zInvalid alpha value: )	ÚlrÚmomentumÚalphaÚepsÚcenteredÚweight_decayr   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr   r   r   r   r   r   r   r   r   Údefaults©Ú	__class__© úf/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/rmsprop.pyr      s,   ÷zRMSprop.__init__c                    sX   t ƒ  |¡ | jD ] }| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ q	d S )Nr   r   r   Fr   r   r   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr"   r$   r%   r&   0   s   
ûzRMSprop.__setstate__c           
      C   s0  d}|d D ]�}|j d u rq|t |¡O }| |¡ |j jr"tdƒ‚| |j ¡ | j| }	t|	ƒdkr_d|	d< tj|tj	d�|	d< |d dkrQtj|tj	d�|	d	< |d
 r_tj|tj	d�|	d< | |	d ¡ |d dkrs| |	d	 ¡ |d
 r~| |	d ¡ |d r�t
|	d tƒr�tdƒ‚|	d  d7  < q|S )NFr    z)RMSprop does not support sparse gradientsr   Ústep)Úmemory_formatÚ
square_avgr   Zmomentum_bufferr   Úgrad_avgr   z`step` can't be a tensorr   )ÚgradÚtorchÚ
is_complexÚappendÚ	is_sparseÚRuntimeErrorr)   ÚlenÚ
zeros_likeÚpreserve_formatÚ
isinstancer   )
r   r*   Úparams_with_gradÚgradsÚsquare_avgsÚmomentum_buffer_listÚ	grad_avgsÚhas_complexÚpr)   r$   r$   r%   Ú_init_group9   s@   



ÿ
ÿ
ÿzRMSprop._init_groupc           
      C   s¼   d}|durt  ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ];}g }g }g }g }g }|  ||||||¡}	t||||||d |d |d |d |d |d |d |d	 |d
 |	d� q |S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   r   )
r   r   r   r   r   r   r   r   r   r>   )r0   Úenable_gradr'   r@   r   )
r   ÚclosureÚlossr*   r9   r:   r;   r=   r<   r>   r$   r$   r%   r+   b   s<   
ÿ
ñzRMSprop.step)	r   r   r   r   r   FNFF)N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr   r&   r@   r   r+   Ú__classcell__r$   r$   r"   r%   r   
   s*    õ	÷
öõ%	)a  Implements RMSprop algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \alpha \text{ (alpha)},\: \gamma \text{ (lr)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm}   \lambda \text{ (weight decay)},\: \mu \text{ (momentum)},\: centered\\
            &\textbf{initialize} : v_0 \leftarrow 0 \text{ (square average)}, \:
                \textbf{b}_0 \leftarrow 0 \text{ (buffer)}, \: g^{ave}_0 \leftarrow 0     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}v_t           \leftarrow   \alpha v_{t-1} + (1 - \alpha) g^2_t
                \hspace{8mm}                                                                     \\
            &\hspace{5mm} \tilde{v_t} \leftarrow v_t                                             \\
            &\hspace{5mm}if \: centered                                                          \\
            &\hspace{10mm} g^{ave}_t \leftarrow g^{ave}_{t-1} \alpha + (1-\alpha) g_t            \\
            &\hspace{10mm} \tilde{v_t} \leftarrow \tilde{v_t} -  \big(g^{ave}_{t} \big)^2        \\
            &\hspace{5mm}if \: \mu > 0                                                           \\
            &\hspace{10mm} \textbf{b}_t\leftarrow \mu \textbf{b}_{t-1} +
                g_t/ \big(\sqrt{\tilde{v_t}} +  \epsilon \big)                                   \\
            &\hspace{10mm} \theta_t \leftarrow \theta_{t-1} - \gamma \textbf{b}_t                \\
            &\hspace{5mm} else                                                                   \\
            &\hspace{10mm}\theta_t      \leftarrow   \theta_{t-1} -
                \gamma  g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big)  \hspace{3mm}              \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to
    `lecture notes <https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_ by G. Hinton.
    and centered version `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.
    The implementation here takes the square root of the gradient average before
    adding epsilon (note that TensorFlow interchanges these two operations). The effective
    learning rate is thus :math:`\gamma/(\sqrt{v} + \epsilon)` where :math:`\gamma`
    is the scheduled learning rate and :math:`v` is the weighted moving average
    of the squared gradient.
    aŸ  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        momentum (float, optional): momentum factor (default: 0)
        alpha (float, optional): smoothing constant (default: 0.99)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        centered (bool, optional) : if ``True``, compute the centered RMSProp,
            the gradient is normalized by an estimation of its variance
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        z

    Fr    r:   r;   r=   r<   r   r   r   r>   r   r   r   r   r   r   c	                C   sp   |du rt | |dd�\}}|rtj ¡ rtdƒ‚|r"tj ¡ s"t}nt}|| |||||	|
|||||||d� dS )zsFunctional API that performs rmsprop algorithm computation.
    See :class:`~torch.optim.RMSProp` for details.
    NF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r   r   r   r   r>   )r   r0   ÚjitÚis_scriptingr4   Ú_multi_tensor_rmspropÚ_single_tensor_rmsprop)r    r:   r;   r=   r<   r   r   r   r>   r   r   r   r   r   r   Ú_Úfuncr$   r$   r%   r   Ê   s.   
òc       	         C   sF  t | ƒD ]œ\}}|| }|s|n| }|| }|dkr"|j||d�}t |¡}|r8t |¡}t |¡}t |¡}| |¡j||d| d� |
re|| }|rRt |¡}| |d| ¡ |j||dd� 	¡ }n| 
¡ }|rq| |¡}n| |¡}|	dkr—|| }|r…t |¡}| |	¡ ||¡ |j|| d� q|j||| d� qd S )Nr   ©r   r   ©Úvalueéÿÿÿÿ)Ú	enumerateÚaddr0   r1   Úview_as_realÚmul_Úaddcmul_Úlerp_ÚaddcmulÚsqrt_ÚsqrtÚadd_Úaddcdiv_)r    r:   r;   r=   r<   r   r   r   r   r   r   r   r   r>   ÚiÚparamr/   r-   Zis_complex_paramr.   ÚavgÚbufr$   r$   r%   rM   ÿ   s<   






ÛrM   c       	         C   sŠ  t | ƒdkrd S |rJ dƒ‚t | ||||g¡}| ¡ D ]¦\\}}}}}}|rC||g}|	dkr4| |¡ |
r;| |¡ t|g|¢R Ž  |rJt |¡}|dkra|rYtj|||d� ntj	|||d�}t 
||¡ tj|||d| d� |
r’t ||d| ¡ tj|||dd�}t |¡ t ||¡ nt |¡}t ||¡ |	dkr¸t 
||	¡ t |||¡ tj||| d� qtj|||| d� qd S )Nr   z#_foreach ops don't support autogradrP   r   rQ   rS   )r5   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr2   r
   r0   Ú_foreach_negÚ_foreach_add_Ú_foreach_addÚ_foreach_mul_Ú_foreach_addcmul_Ú_foreach_lerp_Z_foreach_addcmulÚ_foreach_sqrt_Ú_foreach_sqrtÚ_foreach_addcdiv_)r    r:   r;   r=   r<   r   r   r   r   r   r   r   r   r>   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsZgrouped_square_avgsZgrouped_grad_avgsZgrouped_momentum_buffer_listrN   Ústate_and_gradsra   r$   r$   r%   rL   9  sF   ÿ




ÛrL   )NFFF)r0   r   Ú	optimizerr   r   r   r   r   r	   r
   Útypingr   r   Ú__all__r   Ú__doc__rG   Úfloatr   rM   rL   r$   r$   r$   r%   Ú<module>   sÚ    $ *ôóòÖEõÿþýüûø	÷
öõóòñðï
î5ÿþýüûùø	÷
öõôóò
ñ:ÿþýüûùø	÷
öõôóòñ