o
    ôj8K  ã                $   @   sà  d dl mZmZ d dlZd dlmZ ddlmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e
› de› de› de› d�	 e_dee dee dee dee dee dee dedededededededededef dd„Zdee dee dee dee dee dee dedededededededededef dd „Ze	ed!�		"	"	"	"d%dee dee dee dee dee dee d#ee dedededededededededef"d$d„ƒZdS )&é    )ÚListÚOptionalN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚRMSpropÚrmspropc                       s~   e Zd Z										ddeded	ed
edededee dedef‡ fdd„Z‡ fdd„Zdd„ Z	e
ddd„ƒZ‡  ZS )r   ç{®Gáz„?ç®Gáz®ï?ç:Œ0âŽyE>r   FNÚparamsÚlrÚalphaÚepsÚweight_decayÚmomentumÚforeachÚmaximizeÚdifferentiablec                    sœ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|ks!t d|› �ƒ‚d|ks,t d|› �ƒ‚d|ks7t d|› �ƒ‚t||||||||	|
|d�
}tƒ  ||¡ d S )Ng        zInvalid learning rate: zInvalid epsilon value: zInvalid momentum value: zInvalid weight_decay value: zInvalid alpha value: )
r   r   r   r   Úcenteredr   Ú
capturabler   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r   r    r!   r   r   r   Údefaults©Ú	__class__© úd/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/rmsprop.pyr%      s.   özRMSprop.__init__c                    sÖ   t ƒ  |¡ | jD ]_}| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ | dd¡ |d	 D ]4}| j |g ¡}t|ƒdkrgt |d
 ¡sgt	|d
 ƒ}|d r]tj
|tƒ |jd�ntj
|tƒ d�|d
< q3q	d S )Nr   r   r    Fr   r   r   r!   r   Ústep©ÚdtypeÚdevice©r.   )r$   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r/   )r&   r4   ÚgroupÚpÚp_stateÚstep_valr(   r*   r+   r1   @   s*   
ý
ÿû€üùzRMSprop.__setstate__c                 C   s8  d}|d D ]“}	|	j d u rq|t |	¡O }| |	¡ |	j jr"tdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrs|d rAtjdt	ƒ |	j
d�ntjdt	ƒ d�|
d	< tj|	tjd
�|
d< |d dkretj|	tjd
�|
d< |d rstj|	tjd
�|
d< | |
d ¡ | |
d	 ¡ |d dkrŽ| |
d ¡ |d r™| |
d ¡ q|S )NFr   z)RMSprop does not support sparse gradientsr   r!   r*   r-   r0   r,   )Úmemory_formatÚ
square_avgr   Zmomentum_bufferr    Úgrad_avg)Úgradr7   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr4   r6   Úzerosr   r/   Ú
zeros_likeÚpreserve_format)r&   r;   Úparams_with_gradÚgradsÚsquare_avgsÚmomentum_buffer_listÚ	grad_avgsÚstate_stepsÚhas_complexr<   r4   r*   r*   r+   Ú_init_groupU   sD   



ÿý
ÿ
ÿ
ÿ€zRMSprop._init_groupc                 C   sÒ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]B}g }g }g }g }g }g }	|  |||||||	¡}
t||||||	|d |d |d |d |d |d |d |d	 |d
 |d |
d� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r    r   r   r   r!   )r   r   r   r   r   r    r   r   r   r!   rP   )Ú _cuda_graph_capture_health_checkr7   Úenable_gradr2   rQ   r   )r&   ÚclosureÚlossr;   rJ   rK   rL   rN   rM   rO   rP   r*   r*   r+   r,   ˆ   sT   
ÿ
ù
ïzRMSprop.step)
r   r   r   r   r   FFNFF©N)Ú__name__Ú
__module__Ú__qualname__r   r9   r   Úboolr%   r1   rQ   r   r,   Ú__classcell__r*   r*   r(   r+   r      sD    ôþýüûúù
öõô'3a  Implements RMSprop algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \alpha \text{ (alpha)},\: \gamma \text{ (lr)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm}   \lambda \text{ (weight decay)},\: \mu \text{ (momentum)},\: centered\\
            &\textbf{initialize} : v_0 \leftarrow 0 \text{ (square average)}, \:
                \textbf{b}_0 \leftarrow 0 \text{ (buffer)}, \: g^{ave}_0 \leftarrow 0     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}v_t           \leftarrow   \alpha v_{t-1} + (1 - \alpha) g^2_t
                \hspace{8mm}                                                                     \\
            &\hspace{5mm} \tilde{v_t} \leftarrow v_t                                             \\
            &\hspace{5mm}if \: centered                                                          \\
            &\hspace{10mm} g^{ave}_t \leftarrow g^{ave}_{t-1} \alpha + (1-\alpha) g_t            \\
            &\hspace{10mm} \tilde{v_t} \leftarrow \tilde{v_t} -  \big(g^{ave}_{t} \big)^2        \\
            &\hspace{5mm}if \: \mu > 0                                                           \\
            &\hspace{10mm} \textbf{b}_t\leftarrow \mu \textbf{b}_{t-1} +
                g_t/ \big(\sqrt{\tilde{v_t}} +  \epsilon \big)                                   \\
            &\hspace{10mm} \theta_t \leftarrow \theta_{t-1} - \gamma \textbf{b}_t                \\
            &\hspace{5mm} else                                                                   \\
            &\hspace{10mm}\theta_t      \leftarrow   \theta_{t-1} -
                \gamma  g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big)  \hspace{3mm}              \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to
    `lecture notes <https://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf>`_ by G. Hinton.
    and centered version `Generating Sequences
    With Recurrent Neural Networks <https://arxiv.org/pdf/1308.0850v5.pdf>`_.
    The implementation here takes the square root of the gradient average before
    adding epsilon (note that TensorFlow interchanges these two operations). The effective
    learning rate is thus :math:`\gamma/(\sqrt{v} + \epsilon)` where :math:`\gamma`
    is the scheduled learning rate and :math:`v` is the weighted moving average
    of the squared gradient.
    aŸ  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        momentum (float, optional): momentum factor (default: 0)
        alpha (float, optional): smoothing constant (default: 0.99)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        centered (bool, optional) : if ``True``, compute the centered RMSProp,
            the gradient is normalized by an estimation of its variance
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        z

    r   rK   rL   rN   rM   rO   r   r   r   r   r   r    r   r   r!   rP   c       
         C   s–  t | ƒD ]Ä\}}|| }tj ¡ s,|r,tƒ }|jj|jjkr$|jj|v s,J d|› d�ƒ‚|| }|s4|n| }|| }|d7 }|	dkrJ|j||	d�}t |¡}|r`t 	|¡}t 	|¡}t 	|¡}| 
|¡j||d| d� |r�|| }|rzt 	|¡}| |d| ¡ |j||dd� ¡ }n| ¡ }|r™| |¡}n| |¡}|
dkr¿|| }|r­t 	|¡}| 
|
¡ ||¡ |j|| d� q|j||| d� qd S )NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©r   ©Úvalueéÿÿÿÿ)Ú	enumerater7   Ú_utilsÚis_compilingr   r/   ÚtypeÚaddrC   Úview_as_realÚmul_Úaddcmul_Úlerp_ÚaddcmulÚsqrt_ÚsqrtÚadd_Úaddcdiv_)r   rK   rL   rN   rM   rO   r   r   r   r   r   r    r   r   r!   rP   ÚiÚparamr,   Úcapturable_supported_devicesrB   r@   Zis_complex_paramrA   ÚavgÚbufr*   r*   r+   Ú_single_tensor_rmsprop  sL   
ÿ






Ïru   c       
            sd  t | ƒdkrd S |rJ dƒ‚tj ¡ s.|r.tƒ ‰ t‡ fdd„t| |ƒD ƒƒs.J dˆ › d�ƒ‚t | |||||g¡}| 	¡ D ]ò\\}}}}}}}|re||g}|
dkrV| 
|¡ |r]| 
|¡ t|g|¢R Ž  |rlt |¡}|d jrtj|tjddd	�dd
� nt |d¡ |	dkrœ|r”tj|||	d
� ntj|||	d
�}t ||¡ tj|||d| d� |rÍt ||d| ¡ tj|||dd�}t |¡ t ||¡ nt |¡}t ||¡ |
dk�rt ||
¡ t |||¡ |�rt|tjƒ�rt || ¡}t ||¡ q=tj||| d
� q=|�r%t|tjƒ�r%t || ¡ t |||¡ q=tj|||| d� q=d S )Nr   z#_foreach ops don't support autogradc                 3   s0   � | ]\}}|j j|j jko|j jˆ v V  qd S rV   )r/   re   )Ú.0r<   r,   ©rr   r*   r+   Ú	<genexpr>c  s   € þ

þz(_multi_tensor_rmsprop.<locals>.<genexpr>r\   r]   g      ð?Úcpu)r/   r^   r   r_   ra   )r6   r7   rc   rd   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesrD   r   Ú_foreach_negÚis_cpuÚ_foreach_add_r:   Ú_foreach_addÚ_foreach_mul_Ú_foreach_addcmul_Ú_foreach_lerp_Z_foreach_addcmulÚ_foreach_sqrt_Ú_foreach_sqrtÚ_foreach_addcdiv_Ú
isinstancer   Ú_foreach_mulÚ_foreach_div_)r   rK   rL   rN   rM   rO   r   r   r   r   r   r    r   r   r!   rP   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsZgrouped_square_avgsZgrouped_grad_avgsZgrouped_momentum_buffer_listÚgrouped_state_stepsÚ_Ústate_and_gradsrs   Zmomentum_lrr*   rw   r+   Ú_multi_tensor_rmspropH  sŒ   
ý
üÿ÷



ÿÿÿÿ


ÿ´r‘   )Úsingle_tensor_fnFr   c                C   s˜   t j ¡ stdd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r*t j ¡ r*tdƒ‚|r4t j ¡ s4t}nt	}|| |||||||||||||	||
d� dS )	zsFunctional API that performs rmsprop algorithm computation.
    See :class:`~torch.optim.RMSProp` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rV   )rˆ   r7   r   )rv   Útr*   r*   r+   rx   ×  s   € 
ÿzrmsprop.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)
r   r   r   r   r   r    r   r!   r   rP   )
r7   rc   rd   rz   rF   r   ÚjitÚis_scriptingr‘   ru   )r   rK   rL   rN   rM   rO   r   r   r   r!   rP   r   r   r   r   r   r    r�   Úfuncr*   r*   r+   r   »  sB   ÿÿ
ÿ
ð)NFFFF)Útypingr   r   r7   r   Ú	optimizerr   r   r   r	   r
   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r9   rZ   ru   r‘   r   r*   r*   r*   r+   Ú<module>   sü   8 *+ôóòñÕÿAÿþýüûúø	÷
öõôóòñð
ïGÿþýüûúø	÷
öõôóòñð
ïs
óÿþýüûú	÷
öõôóñðïîíì