o
    !ôjÔR  ã                   @   s¤  d dl mZmZ d dlZd dlmZ ddlmZmZmZm	Z	m
Z
mZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de	› d� e_					d"dee dee dee dee dee dedee dedededededededefdd„Zdee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dededededededededefd d!„ZdS )#é    )ÚListÚOptionalN)ÚTensoré   )
Ú	OptimizerÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_capturable_docÚ_dispatch_sqrtÚ_foreach_docÚ_get_scalar_dtypeÚ
_get_valueÚ_use_grad_for_differentiableÚ_view_as_realÚRAdamÚradamc                
       sj   e Zd Z					dddddœded	ee d
edef‡ fdd„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚ
capturableÚdifferentiableÚdecoupled_weight_decayr   r   r   c             
      sÄ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|d   kr"dk s,n t d|d › �ƒ‚d|d   kr8dk sBn t d|d › �ƒ‚d|ksMt d	|› �ƒ‚t||||||||	d
�}
tƒ  ||
¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )ÚlrÚbetasÚepsÚweight_decayr   r   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr   r   r   r   r   r   r   r   Údefaults©Ú	__class__© úd/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/radam.pyr"      s*   ø
zRAdam.__init__c                    s¾   t ƒ  |¡ | jD ]S}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]4}| j |g ¡}t|ƒdkr[t |d ¡s[t	|d ƒ}|d rQtj
|tƒ |jd	�ntj
|tƒ d
�|d< q'q	d S )Nr   r   Fr   r   r$   r   Ústep©ÚdtypeÚdevice©r,   )r!   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r-   )r#   r2   ÚgroupÚpÚp_stateÚstep_valr&   r(   r)   r/   ;   s    
ÿ€üûzRAdam.__setstate__c           
      C   sì   d}|d D ]m}|j d urs|t |¡O }| |¡ |j jr!tdƒ‚| |j ¡ | j| }	t|	ƒdkr^|d r@tjdt	ƒ |j
d�ntjdt	ƒ d	�|	d
< tj|tjd�|	d< tj|tjd�|	d< | |	d ¡ | |	d ¡ | |	d
 ¡ q|S )NFr$   z'RAdam does not support sparse gradientsr   r   r(   r+   r   r.   r*   )Úmemory_formatÚexp_avgÚ
exp_avg_sq)Úgradr5   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr2   r4   Úzerosr   r-   r8   Ú
zeros_likeÚpreserve_format)
r#   r9   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚstate_stepsÚhas_complexr:   r2   r(   r(   r)   Ú_init_groupI   s2   


ÿý
ÿ
ÿ€zRAdam._init_groupc                 C   sÈ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]=}g }g }g }g }g }|d \}	}
|  ||||||¡}t||||||	|
|d |d |d |d |d |d |d	 |d
� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   )
Úbeta1Úbeta2r   r   r   r   r   r   r   rM   )Ú _cuda_graph_capture_health_checkr5   Úenable_gradr0   rN   r   )r#   ÚclosureÚlossr9   rH   rI   rJ   rK   rL   rO   rP   rM   r(   r(   r)   r*   j   s@   
ÿ
ñz
RAdam.step)r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__Úboolr   r"   r/   rN   r   r*   Ú__classcell__r(   r(   r&   r)   r      s.    ù	õù	÷
öõ$!aê  Implements RAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \beta_1, \beta_2
                \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \:
                \lambda \text{ (weightdecay)},                                                   \\
            &\hspace{13mm} \epsilon \text{ (epsilon)}, \textit{decoupled\_weight\_decay}         \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)},                                       \\
            &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1                      \\[-1.ex]
            &\rule{110mm}{0.4pt}  \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{6mm} g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1})                      \\
            &\hspace{6mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{6mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{12mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{18mm} \theta_t \leftarrow \theta_{t} - \gamma \lambda \theta_{t}            \\
            &\hspace{12mm}\textbf{else}                                                          \\
            &\hspace{18mm} g_t \leftarrow g_t + \lambda \theta_{t}                               \\
            &\hspace{6mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{6mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{6mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} -
                2 t \beta^t_2 /\big(1-\beta_2^t \big)                                    \\[0.1.ex]
            &\hspace{6mm}\textbf{if} \: \rho_t > 5                                               \\
            &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon  } \\
            &\hspace{12mm} r_t \leftarrow
      \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\
            &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} r_t l_t        \\
            &\hspace{6mm}\textbf{else}                                                           \\
            &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `On the variance of the adaptive learning rate and beyond`_.

    This implementation provides an option to use either the original weight_decay implementation as in Adam
    (where the weight_decay is applied to the gradient) or the one from AdamW (where weight_decay is applied
    to the weight) through the decoupled_weight_decay option. When decoupled_weight_decay is set to False
    (default), it uses the original Adam style weight decay, otherwise, it uses the AdamW style which
    corresponds more closely to the `author's implementation`_ in the RAdam paper. Further information
    about decoupled weight decay can be found in `Decoupled Weight Decay Regularization`_.

    a³  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): whether to use decoupled weight
            decay as in AdamW to obtain RAdamW (default: False)
        z	
        a  

    .. _On the variance of the adaptive learning rate and beyond:
        https://arxiv.org/abs/1908.03265
    .. _author's implementation:
        https://github.com/LiyuanLucasLiu/RAdam
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    Fr$   rI   rJ   rK   rL   r   r   r   r   rM   rO   rP   r   r   r   c
                C   sŠ   t dd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r%tj ¡ r%tdƒ‚|r/tj ¡ s/t}nt}|| |||||
||||||||	d� dS )	zpFunctional API that performs RAdam algorithm computation.

    See :class:`~torch.optim.RAdam` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rU   )Ú
isinstancer5   r   )Ú.0Útr(   r(   r)   Ú	<genexpr>ú   s   € zradam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	rO   rP   r   r   r   r   r   r   rM   )ÚallrD   r   r5   ÚjitÚis_scriptingÚ_multi_tensor_radamÚ_single_tensor_radam)r$   rI   rJ   rK   rL   r   r   r   r   rM   rO   rP   r   r   r   Ú_Úfuncr(   r(   r)   r   á   s6   ÿ
òc       	            sÞ  t | ƒD ]è\}}|| }|| }|| ‰|| }tj ¡ s/|r/|jr%|js/|jr+|js/J dƒ‚t |¡rHt |¡}t |¡}t |¡}t ˆ¡‰|d7 }|rP|nt|ƒ}|dkrk|rd| 	d||  ¡ n|j
||d�}| |d| ¡ ˆ 	|¡j||d| d� d||  }d||  ‰ || }dd|  d ‰ˆd| ||  ˆ   ‰‡‡fdd„}‡ ‡‡‡fd	d
„}|rÏt ˆdk|ƒ |ƒ  d¡}|j|| | dd� qˆdkrã|j|| |ƒ  |ƒ  dd� q|j|| dd� qd S )NzGIf capturable=True, params and state_steps must be CUDA or XLA tensors.r   r   ©Úalpha)Úvalueé   c                      s,   ˆd ˆd  ˆ  ˆ d ˆ d  ˆ  d S )Né   rj   ç      à?r(   r(   )Úrho_infÚrho_tr(   r)   Ú_compute_rectX  s   ÿþýûz+_single_tensor_radam.<locals>._compute_rectc                     s.   ˆ  ¡ } ˆr|  ˆ¡} n|  ˆ¡} ˆ d |  S )Nrl   )ÚsqrtÚaddÚadd_)Úexp_avg_sq_sqrt)Úbias_correction2r   r   r?   r(   r)   Ú_compute_adaptive_lr`  s
   
z2_single_tensor_radam.<locals>._compute_adaptive_lrç      @r   g      ð¿)Ú	enumerater5   Ú_utilsÚis_compilingÚis_cudaÚis_xlarA   Úview_as_realr   Úmul_rq   Úlerp_Úaddcmul_Úwhererr   )r$   rI   rJ   rK   rL   rO   rP   r   r   r   r   r   r   rM   ÚiÚparamr@   r>   Ústep_tr*   Úbias_correction1Zbias_corrected_exp_avgro   ru   Úupdater(   )rt   r   r   r?   rm   rn   r)   rd     sP   ÿÿþ





 ¼rd   c       	            s¢  t | ƒdkrd S |rJ dƒ‚tj ¡ s%|r%tdd„ t| |ƒD ƒƒs%J dƒ‚t | ||||g¡}| ¡ D �]š\\}}}}}}|d j	rPtj
|tjddd�dd	� nt 
|d
¡ |r_t||||ƒ dd
ˆ  d
 ‰|r t ˆ|¡}t |¡ t 
|d
¡ t ˆ|¡}t ||¡ t |d¡ t ||¡ t |¡ t 
|ˆ¡ |}n
‡‡fdd„|D ƒ}|dkrÃ|
r»t |d
ˆ|  ¡ ntj|||d	�}t ||d
ˆ  ¡ t |ˆ¡ t |||d
ˆ ¡ ~|�rxt |d¡}t |d¡}t ||¡ ~t |ˆ¡ ˆd ˆd  ‰t |ˆ¡}t ||¡ ~t |¡ dd„ t||ƒD ƒ}~~dd„ |D ƒ}t |ˆ¡ t ˆ |¡}t |¡ t 
|d
¡ t ||¡ t |¡ t ˆ|¡}t |¡ t 
|d
¡ t |¡ t |ˆ¡ t ||¡ ~t |¡ t ||¡ ~n3‡fdd„|D ƒ}dd„ |D ƒ}‡ fdd„|D ƒ}‡fdd„t||ƒD ƒ}‡‡fdd„t|||ƒD ƒ}t |¡}t 
||	¡ t ||¡ t |¡ t 
||¡ t |||¡ q3d S )Nr   z#_foreach ops don't support autogradc                 s   s    � | ]\}}|j o|j V  qd S rU   )rz   )r\   r:   r*   r(   r(   r)   r^   �  s   € z&_multi_tensor_radam.<locals>.<genexpr>z@If capturable=True, params and state_steps must be CUDA tensors.r   Úcpu)r-   rg   r   rj   c                    s8   g | ]}ˆd t |ƒ ˆ t |ƒ  dˆ t |ƒ    ‘qS )rj   r   ©r   ©r\   r*   )rP   rm   r(   r)   Ú
<listcomp>³  s
    ÿ
ÿz'_multi_tensor_radam.<locals>.<listcomp>rk   c                 S   s"   g | ]\}}t  |d k|d¡‘qS )rv   r   ©r5   r€   )r\   Únrn   r(   r(   r)   r‰   Ó  s   " c                 S   s   g | ]}t  |d kdd¡‘qS )r   r   r   rŠ   ©r\   Úrectr(   r(   r)   r‰   Ö  s    c                    sD   g | ]}|d krt |d |d  ˆ  ˆ d ˆ d  |  ƒnd‘qS )é   rk   rj   r   )r
   )r\   rn   )rm   r(   r)   r‰   ë  s    	þúÿþýÿøc                 S   s   g | ]
}|d kr
d nd‘qS )r   r   r(   rŒ   r(   r(   r)   r‰   ö  ó    c                    s   g | ]
}d ˆ t |ƒ  ‘qS )r   r‡   rˆ   )rO   r(   r)   r‰   ø  r�   c                    s    g | ]\}}ˆ | | d  ‘qS )éÿÿÿÿr(   )r\   r�   Úbc)r   r(   r)   r‰   ù  s     c                    s6   g | ]\}}}t d ˆ t|ƒ  ƒˆ| |  d ‘qS )r   r�   )r
   r   )r\   r*   r�   r‘   )rP   r   r(   r)   r‰   ú  s    "ÿÿ)r4   r5   rx   ry   r`   Úzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesÚis_cpuÚ_foreach_add_r8   r   Ú_foreach_powÚ_foreach_neg_Ú_foreach_mul_Ú_foreach_div_Ú_foreach_addÚ_foreach_lerp_Ú_foreach_addcmul_Ú_foreach_subÚ_foreach_mulÚ_foreach_sqrt_Ú_foreach_sqrtÚ_foreach_reciprocal_)r$   rI   rJ   rK   rL   rO   rP   r   r   r   r   r   r   rM   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsÚgrouped_exp_avgsÚgrouped_exp_avg_sqsÚgrouped_state_stepsre   r„   rt   Z
rho_t_listÚnumZsub2Údenomr�   Zunrect_step_sizeZunrectifiedÚbufferr(   )rO   rP   r   rm   r)   rc   t  s¶   ÿ
ú


ÿ






	÷
þ

Šrc   )FNFFF)Útypingr   r   r5   r   Ú	optimizerr   r   r   r	   r
   r   r   r   r   r   Ú__all__r   Ú__doc__rY   r7   r   rd   rc   r(   r(   r(   r)   Ú<module>   sÜ    0 /ôóòÑQôÿþýüûø	÷
öõôòñðï
î;ÿþýüûùø	÷
öõôóò
ñXÿþýüûùø	÷
öõôóòñ