o
    ôj@]  ã                "   @   sÞ  d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de› de	› d�	 e_dee dee dee dee dee dedededededededededefdd„Zdee dee dee dee dee dedededededededededefdd„Zeed �	!		!	!	!	!d$dee dee dee dee dee ded"ee dededededededededef d#d„ƒZdS )%é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_dispatch_sqrtÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚRAdamÚradamc                       sŒ   e Zd Z					ddddddœded	ed
eeef dedededee dededef‡ fdd„Z‡ fdd„Z	dd„ Z
eddd„ƒZ‡  ZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiableÚparamsÚlrÚbetasÚepsÚweight_decayÚdecoupled_weight_decayr   r   r   r   c                   sÆ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|d   kr"dk s,n t d|d › �ƒ‚d|d   kr8dk sBn t d|d › �ƒ‚d|ksMt d	|› �ƒ‚t|||||||	||
d
�	}tƒ  ||¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )	r!   r"   r#   r$   r   r   r   r%   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr    r!   r"   r#   r$   r%   r   r   r   r   Údefaults©Ú	__class__© úb/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/radam.pyr+      s,   ÷zRAdam.__init__c                    sÊ   t ƒ  |¡ | jD ]Y}| dd ¡ | dd¡ | dd¡ | dd¡ | dd¡ |d D ]4}| j |g ¡}t|ƒdkrat |d	 ¡sat	|d	 ƒ}|d rWtj
|tƒ |jd
�ntj
|tƒ d�|d	< q-q	d S )Nr   r   Fr   r%   r   r    r   Ústep©ÚdtypeÚdevice©r4   )r*   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r5   )r,   r:   ÚgroupÚpÚp_stateÚstep_valr.   r0   r1   r7   B   s(   
ý
ÿû€üúzRAdam.__setstate__c           
      C   sì   d}|d D ]m}|j d urs|t |¡O }| |¡ |j jr!tdƒ‚| |j ¡ | j| }	t|	ƒdkr^|d r@tjdt	ƒ |j
d�ntjdt	ƒ d	�|	d
< tj|tjd�|	d< tj|tjd�|	d< | |	d ¡ | |	d ¡ | |	d
 ¡ q|S )NFr    z'RAdam does not support sparse gradientsr   r   r0   r3   r&   r6   r2   )Úmemory_formatÚexp_avgÚ
exp_avg_sq)Úgradr=   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr:   r<   Úzerosr   r5   r@   Ú
zeros_likeÚpreserve_format)
r,   rA   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚstate_stepsÚhas_complexrB   r:   r0   r0   r1   Ú_init_groupV   s2   


ÿý
ÿ
ÿ€zRAdam._init_groupc                 C   sÜ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]G}g }g }g }g }g }ttttf |d ƒ\}	}
|  ||||||¡}t||||||	|
|d |d |d |d |d |d |d	 |d
 |d� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr"   r!   r$   r#   r   r   r   r   r%   )Úbeta1Úbeta2r!   r$   r#   r   r   r   r   r%   rU   )	Ú _cuda_graph_capture_health_checkr=   Úenable_gradr8   r   r   r?   rV   r   )r,   ÚclosureÚlossrA   rP   rQ   rR   rS   rT   rW   rX   rU   r0   r0   r1   r2   y   sF   
ÿ
ÿðz
RAdam.step)r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r   r?   r   Úboolr   r+   r7   rV   r   r2   Ú__classcell__r0   r0   r.   r1   r      sH    ù	ôþý
üûúù	÷
öõô&#a  Implements RAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
                \lambda \text{ (weightdecay)}, \:\textit{maximize}                               \\
            &\hspace{13mm} \epsilon \text{ (epsilon)}, \textit{decoupled\_weight\_decay}         \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
            &\rule{110mm}{0.4pt}  \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{6mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{12mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{6mm}\textbf{else}                                                           \\
            &\hspace{12mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{6mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{6mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{12mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{18mm} \theta_t \leftarrow \theta_{t} - \gamma \lambda \theta_{t}            \\
            &\hspace{12mm}\textbf{else}                                                          \\
            &\hspace{18mm} g_t \leftarrow g_t + \lambda \theta_{t}                               \\
            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                    \\[0.1.ex]
            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
            &\hspace{12mm} r_t \leftarrow
      \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
            &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} r_t l_t        \\
            &\hspace{6mm}\textbf{else}                                                           \\
            &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `On the variance of the adaptive learning rate and beyond`_.

    This implementation provides an option to use either the original weight_decay implementation as in Adam
    (where the weight_decay is applied to the gradient) or the one from AdamW (where weight_decay is applied
    to the weight) through the decoupled_weight_decay option. When decoupled_weight_decay is set to False
    (default), it uses the original Adam style weight decay, otherwise, it uses the AdamW style which
    corresponds more closely to the `author's implementation`_ in the RAdam paper. Further information
    about decoupled weight decay can be found in `Decoupled Weight Decay Regularization`_.

    a³  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): whether to use decoupled weight
            decay as in AdamW to obtain RAdamW (default: False)
        z	
        a  

    .. _On the variance of the adaptive learning rate and beyond:
        https://arxiv.org/abs/1908.03265
    .. _author's implementation:
        https://github.com/LiyuanLucasLiu/RAdam
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    r    rQ   rR   rS   rT   rW   rX   r!   r$   r#   r%   r   r   r   rU   c       
            sþ  t | ƒD ]ø\}}|s|| n||  }|| }|| ‰|| }tj ¡ s?|r?tƒ }|jj|jjkr7|jj|v s?J d|› d�ƒ‚t |¡rXt |¡}t |¡}t |¡}t ˆ¡‰|d7 }|r`|nt	|ƒ}|dkr{|
rt| 
d||  ¡ n|j||d�}| |d| ¡ ˆ 
|¡j||d| d� d||  }d||  ‰ || }dd|  d ‰ˆd| ||  ˆ   ‰‡‡fdd	„}‡ ‡‡‡fd
d„}|rßt ˆdk|ƒ |ƒ  d¡}|j|| | dd� qˆdkró|j|| |ƒ  |ƒ  dd� q|j|| dd� qd S )NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©Úalpha)Úvalueé   c                      s,   ˆd ˆd  ˆ  ˆ d ˆ d  ˆ  d S )Né   rh   ç      à?r0   r0   )Úrho_infÚrho_tr0   r1   Ú_compute_rect9  s   ÿþýûz+_single_tensor_radam.<locals>._compute_rectc                     s.   ˆ  ¡ } ˆr|  ˆ¡} n|  ˆ¡} ˆ d |  S )Nrj   )ÚsqrtÚaddÚadd_)Úexp_avg_sq_sqrt)Úbias_correction2r   r#   rG   r0   r1   Ú_compute_adaptive_lrA  s
   
z2_single_tensor_radam.<locals>._compute_adaptive_lrç      @r'   g      ð¿)Ú	enumerater=   Ú_utilsÚis_compilingr   r5   ÚtyperI   Úview_as_realr   Úmul_ro   Úlerp_Úaddcmul_Úwhererp   )r    rQ   rR   rS   rT   rW   rX   r!   r$   r#   r%   r   r   r   rU   ÚiÚparamrH   rF   Ústep_tÚcapturable_supported_devicesr2   Úbias_correction1Zbias_corrected_exp_avgrm   rs   Úupdater0   )rr   r   r#   rG   rk   rl   r1   Ú_single_tensor_radamú   sb   
ÿ





ÿÿþýû²r„   c       
             sÜ  t | ƒdkrd S |rJ dƒ‚tj ¡ s0|r0tdd�‰t‡fdd„t| |ƒD ƒƒs0J dˆ› d�ƒ‚t | ||||g¡}| 	¡ D �]¬\\}}}}}}|d j
r[tj|tjd	d
d�d	d� nt |d¡ |rjt||||ƒ |rqt |¡}ddˆ  d ‰|r²t ˆ|¡}t |¡ t |d¡ t ˆ|¡}t ||¡ t |d¡ t ||¡ t |¡ t |ˆ¡ |}n
‡‡fdd„|D ƒ}|dkrà|
rÍt |dˆ|  ¡ n|rØtj|||d� ntj|||d�}t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ ~|�r•t |d¡}t |d¡}t ||¡ ~t |ˆ¡ ˆd ˆd  ‰t |ˆ¡}t ||¡ ~t |¡ dd„ t||ƒD ƒ}~~dd„ |D ƒ}t |ˆ¡ t ˆ |¡}t |¡ t |d¡ t ||¡ t |¡ t ˆ|¡}t |¡ t |d¡ t |¡ t |ˆ¡ t ||¡ ~t |¡ t ||¡ ~n3‡fdd„|D ƒ}dd„ |D ƒ}‡ fdd„|D ƒ}‡fdd„t||ƒD ƒ}‡‡fdd„t|||ƒD ƒ}t |¡}t ||	¡ t ||¡ t |¡ t ||¡ t |||¡ q>d S )Nr   z#_foreach ops don't support autogradF)Úsupports_xlac                 3   s0   � | ]\}}|j j|j jko|j jˆ v V  qd S r]   )r5   rx   )Ú.0rB   r2   )r�   r0   r1   Ú	<genexpr>y  s   € þ

þz&_multi_tensor_radam.<locals>.<genexpr>rc   rd   r'   Úcpu)r5   re   r   rh   c                    s8   g | ]}ˆd t |ƒ ˆ t |ƒ  dˆ t |ƒ    ‘qS )rh   r   ©r   ©r†   r2   )rX   rk   r0   r1   Ú
<listcomp>®  s    ûÿ
þýÿÿz'_multi_tensor_radam.<locals>.<listcomp>ri   c                 S   s"   g | ]\}}t  |d k|d¡‘qS )rt   r&   ©r=   r}   )r†   Únrl   r0   r0   r1   r‹   Ý  s    ÿc                 S   s   g | ]}t  |d kdd¡‘qS )r   r&   r'   rŒ   ©r†   Úrectr0   r0   r1   r‹   â  s    c                    sD   g | ]}|d krt |d |d  ˆ  ˆ d ˆ d  |  ƒnd‘qS )é   ri   rh   r   )r   )r†   rl   )rk   r0   r1   r‹   ÷  s    	þúÿþýÿøc                 S   s   g | ]
}|d kr
d nd‘qS )r   r'   r0   rŽ   r0   r0   r1   r‹     s    c                    s   g | ]
}d ˆ t |ƒ  ‘qS )r   r‰   rŠ   )rW   r0   r1   r‹     s    ÿc                    s    g | ]\}}ˆ | | d  ‘qS )éÿÿÿÿr0   )r†   r�   Úbc)r!   r0   r1   r‹     s    ÿc                    s6   g | ]\}}}t d ˆ t|ƒ  ƒˆ| |  d ‘qS )r   r‘   )r   r   )r†   r2   r�   r’   )rX   r!   r0   r1   r‹   
  s    "ÿÿ)r<   r=   rv   rw   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesÚis_cpuÚ_foreach_add_r@   r   Ú_foreach_negÚ_foreach_powÚ_foreach_neg_Ú_foreach_mul_Ú_foreach_div_Ú_foreach_addÚ_foreach_lerp_Ú_foreach_addcmul_Ú_foreach_subÚ_foreach_mulÚ_foreach_sqrt_Ú_foreach_sqrtÚ_foreach_reciprocal_) r    rQ   rR   rS   rT   rW   rX   r!   r$   r#   r%   r   r   r   rU   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsÚgrouped_exp_avgsÚgrouped_exp_avg_sqsÚgrouped_state_stepsÚ_r‚   rr   Z
rho_t_listÚnumZsub2Údenomr�   Zunrect_step_sizeZunrectifiedÚbufferr0   )rW   rX   r�   r!   rk   r1   Ú_multi_tensor_radam]  sî   ÿ
ý
üÿ	
ú
ÿÿ


ú	ÿÿÿ
ÿ





	÷
ÿ
ÿ
þ

 �ër°   )Úsingle_tensor_fnFr   c                C   sŒ   t dd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r%tj ¡ r%tdƒ‚|r/tj ¡ s/t}nt}|| ||||||||||
||||	d� dS )	zpFunctional API that performs RAdam algorithm computation.

    See :class:`~torch.optim.RAdam` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S r]   )Ú
isinstancer=   r   )r†   Útr0   r0   r1   r‡   4  s   € zradam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)
rW   rX   r!   r$   r#   r   r%   r   r   rU   )r“   rL   r
   r=   ÚjitÚis_scriptingr°   r„   )r    rQ   rR   rS   rT   r%   r   r   r   rU   r   rW   rX   r!   r$   r#   r¬   Úfuncr0   r0   r1   r     s<   ÿ
ÿ
ñ)FNFFFF)Útypingr   r   r   r   r   r=   r   Ú	optimizerr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r?   ra   r„   r°   r   r0   r0   r0   r1   Ú<module>   sô   @ 3ôóòñÍÿPÿþýüûùø	÷
öõôóòñ
ðcÿþýüûùø	÷
öõôóòñ
ð =	óÿþýüûø	÷
öõôóñðïîí