o
    !ôjñX  ã                #   @   sÌ  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	m
Z
mZmZmZ d dlmZmZ ddgZG dd„ deƒZd	d
e› de› de› d� e_					d$dee dee dee dee dee dee dedee dededededededededef"dd„Zdee dee dee dee dee dee dedededededededededef d d!„Zdee dee dee dee dee dee dedededededededededef d"d#„ZdS )%é    N)ÚTensoré   )Ú	OptimizerÚ_use_grad_for_differentiableÚ
_get_valueÚ_dispatch_sqrtÚ_stack_if_compilingÚ_get_scalar_dtypeÚ_default_to_fused_or_foreachÚ_view_as_realÚ_capturable_docÚ_differentiable_docÚ_foreach_doc)ÚListÚOptionalÚNAdamÚnadamc                
       sd   e Zd Z		dddddœd	ed
ee dedef‡ fdd„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   çü©ñÒMbp?FN)ÚforeachÚ
capturableÚdifferentiableÚdecoupled_weight_decayr   r   r   c                   sÜ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|d   kr"dk s,n t d|d › �ƒ‚d|d   kr8dk sBn t d|d › �ƒ‚d|ksMt d	|› �ƒ‚d|ksXt d
|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: zInvalid momentum_decay value: )	ÚlrÚbetasÚepsÚweight_decayÚmomentum_decayr   r   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr   r   r   r    r!   r   r   r   r   Údefaults©Ú	__class__© úd/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/nadam.pyr%      s$   ýzNAdam.__init__c                    s  t ƒ  |¡ | jD ]v}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]W}| j |g ¡}t|ƒdkr~t |d ¡s[t	|d ƒ}|d rQtj
|tƒ |jd	�ntj
|tƒ d
�|d< t |d ¡s~|d }|d rttj
|tƒ |jd	�ntj
|tƒ d
�|d< q'q	d S )Nr   r   Fr   r   r'   r   Ústep©ÚdtypeÚdevice©r/   Ú
mu_product)r$   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr	   r0   )r&   r6   ÚgroupÚpÚp_stateÚstep_valZmu_prod_valr)   r+   r,   r3   !   s0   
ÿÿÿÿ€÷ûzNAdam.__setstate__c                 C   s*  d}|d D ]Œ}	|	j d ur’|t |	¡O }| |	¡ |	j jr!tdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrv|d r@tjdt	ƒ |	j
d�ntjdt	ƒ d	�|
d
< |d rXtjdt	ƒ |	j
d�ntjdt	ƒ d	�|
d< tj|	tjd�|
d< tj|	tjd�|
d< | |
d ¡ | |
d ¡ | |
d ¡ | |
d
 ¡ q|S )NFr'   z'NAdam does not support sparse gradientsr   r   r+   r.   r   r1   r-   r   r2   )Úmemory_formatÚexp_avgÚ
exp_avg_sq)Úgradr9   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr6   r8   Úzerosr	   r0   r<   ÚonesÚ
zeros_likeÚpreserve_format)r&   r=   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmu_productsÚstate_stepsÚhas_complexr>   r6   r+   r+   r,   Ú_init_group5   s4   


ÿþÿþ€zNAdam._init_groupc                 C   sÖ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]D}g }g }g }g }g }g }	|d \}
}|  |||||||	¡}t||||||	|
||d |d |d |d |d |d |d	 |d
 |d� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r    r!   r   r   r   r   r   )Úbeta1Úbeta2r   r    r!   r   r   r   r   r   rS   )Ú _cuda_graph_capture_health_checkr9   Úenable_gradr4   rT   r   )r&   ÚclosureÚlossr=   rM   rN   rO   rP   rQ   rR   rU   rV   rS   r+   r+   r,   r-   X   sD   
ÿ
ðz
NAdam.step)r   r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__Úboolr   r%   r3   rT   r   r-   Ú__classcell__r+   r+   r)   r,   r   
   s&    ÿýÿþþý#a�  Implements NAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)}    \\
            &\hspace{13mm} \: \textit{decoupled\_weight\_decay}                                  \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{15mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}                    \\
            &\hspace{10mm}\textbf{else}                                                          \\
            &\hspace{15mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2}  0.96^{t \psi} \big)     \\
            &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{(t+1)\psi}\big)\\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i)\\[-1.ex]
            & \hspace{11mm} + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i})                         \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Incorporating Nesterov Momentum into Adam`_.
    a  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        momentum_decay (float, optional): momentum momentum_decay (default: 4e-3)
        decoupled_weight_decay (bool, optional): whether to use decoupled weight
            decay as in AdamW to obtain NAdamW (default: False)
        z	
        zÏ

    .. _Incorporating Nesterov Momentum into Adam:
        https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    Fr'   rN   rO   rP   rQ   rR   r   r   r   r   rS   rU   rV   r   r    r!   r   c                C   s¨   t dd„ |D ƒƒstdƒ‚t dd„ |D ƒƒstdƒ‚|du r't| |	dd�\}}|r2tj ¡ r2td	ƒ‚|r<tj ¡ s<t}nt}|| ||||||||||||||	|
d
� dS )zpFunctional API that performs NAdam algorithm computation.

    See :class:`~torch.optim.NAdam` for details.
    c                 s   ó   � | ]	}t |tjƒV  qd S r[   ©Ú
isinstancer9   r   ©Ú.0Útr+   r+   r,   Ú	<genexpr>Ü   ó   € znadam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsc                 s   ra   r[   rb   rd   r+   r+   r,   rg   ß   rh   zPAPI has changed, `mu_products` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)
rU   rV   r   r    r!   r   r   r   r   rS   )ÚallrH   r
   r9   ÚjitÚis_scriptingÚ_multi_tensor_nadamÚ_single_tensor_nadam)r'   rN   rO   rP   rQ   rR   r   r   r   r   rS   rU   rV   r   r    r!   r   Ú_Úfuncr+   r+   r,   r   Â   s8   
ñc       
         C   s2  t | ƒD �]\}}|| }|| }|| }|| }|| }t |¡r6t |¡}t |¡}t |¡}t |¡}tj ¡ sS|rS|jrF|jrF|jsS|jrO|jrO|jsSJ dƒ‚|d7 }|r\|}nt|ƒ}d||  }|	dkr}|rv| 	d||	  ¡ n|j
||	d�}|ddd||
     }|ddd|d |
     }||9 }| |d| ¡ | 	|¡j||d| d� | |¡ ¡ }|s»|ré| 
|¡}|| }|| d|  d|   }|| | d|   }| ||¡ | ||¡ qt|ƒ| }| |¡ |j||| d|  dt|ƒ  d� |j||| | d|  d� qd S )	NzUIf capturable=True, params, mu_products, and state_steps must be CUDA or XLA tensors.r   r   ©Úalphar   ç      à?ç¸…ëQ¸î?)Úvalue)Ú	enumerater9   rE   Úview_as_realÚ_utilsÚis_compilingÚis_cudaÚis_xlar   Úmul_ÚaddÚlerp_Úaddcmul_ÚdivÚsqrtÚaddcdiv_Úadd_)r'   rN   rO   rP   rQ   rR   rU   rV   r   r    r!   r   r   r   r   rS   ÚiÚparamrD   rB   rC   r2   Ústep_tr-   Úbias_correction2ÚmuÚmu_nextÚdenomZmu_product_nextr+   r+   r,   rn   ÿ   sj   




ÿÿÿÿÿÿþ

& Ârn   c       
   !         sB  t | ƒdkrd S |rJ dƒ‚tj ¡ s&|r&tdd„ t| ||ƒD ƒƒs&J dƒ‚t | |||||g¡}| ¡ D �]h\\}}}}}}}|rIt	||||ƒ |d j
r\tj|tjddd�dd	� nt |d
¡ |	dkr{|rst |d
ˆ|	  ¡ ntj|||	d	�}t ||d
ˆ  ¡ t |ˆ¡ t |||d
ˆ ¡ t |¡}|rït |ˆ¡}t d|¡}t |d¡ t |d¡ t |ˆ ¡ t |ˆ¡ t d|¡}t |d¡ t |d¡ t |ˆ ¡ ~t ˆ|¡}t |d¡ t |¡ t |¡ n‡fdd„|D ƒ}‡ ‡fdd„|D ƒ}‡ ‡fdd„|D ƒ}t ||¡ t ||¡ t ||¡ ~|�rrt |d¡ t |ˆ¡ t |d¡}t |¡ t ||¡ |}~t ||¡}t |ˆ¡ t |d¡ t ||¡ |}~t ||¡} t | ||¡ t || |¡ q5t‡fdd„t||ƒD ƒƒ}t‡fdd„t||ƒD ƒƒ}t ||||¡ t ||||¡ q5d S )Nr   z#_foreach ops don't support autogradc                 s   s(   � | ]\}}}|j o|j o|j V  qd S r[   )rz   )re   r>   Úmpr-   r+   r+   r,   rg   k  s   € ÿz&_multi_tensor_nadam.<locals>.<genexpr>zNIf capturable=True, params, mu_products, and state_steps must be CUDA tensors.r   Úcpu)r0   rq   r   rt   g      à¿c                    s    g | ]}t d ˆ t|ƒ  ƒ‘qS )r   )r   r   ©re   r-   )rV   r+   r,   Ú
<listcomp>¨  s     z'_multi_tensor_nadam.<locals>.<listcomp>c                    s(   g | ]}ˆ d ddt |ƒˆ     ‘qS )r   rs   rt   ©r   r�   ©rU   r!   r+   r,   rŽ   ©  s   ( c                    s,   g | ]}ˆ d ddt |ƒd ˆ     ‘qS )r   rs   rt   r   r�   r�   r�   r+   r,   rŽ   ª  s    $ÿc                    s,   g | ]\}}ˆ d |  d t |ƒ  d ‘qS ©r   éÿÿÿÿr�   )re   r2   rˆ   ©r   r+   r,   rŽ   ×  ó     ÿc                    s,   g | ]\}}ˆ | d t |ƒ|   d ‘qS r‘   r�   )re   r2   r‰   r“   r+   r,   rŽ   Ù  r”   )r8   r9   rx   ry   rj   Úzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Úis_cpuÚ_foreach_add_r<   Ú_foreach_mul_Ú_foreach_addÚ_foreach_lerp_Ú_foreach_addcmul_Ú_foreach_sqrtÚ_foreach_mulÚ_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_sqrt_Ú_foreach_div_Ú_foreach_subÚ_foreach_addcdiv_r   )!r'   rN   rO   rP   rQ   rR   rU   rV   r   r    r!   r   r   r   r   rS   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsÚgrouped_exp_avgsZgrouped_exp_avg_sqsZgrouped_mu_productsÚgrouped_state_stepsro   Úexp_avg_sq_sqrtÚexponentÚmusZmu_nextsZbias_correction_sqrtrŠ   Zstep_size_gradsZstep_size_expavgÚ	numeratorr+   )rU   rV   r   r!   r,   rm   R  sš   

ÿþÿ


ÿ
ÿÿ”rm   )FNFFF)r9   r   Ú	optimizerr   r   r   r   r   r	   r
   r   r   r   r   Útypingr   r   Ú__all__r   Ú__doc__r_   r;   r   rn   rm   r+   r+   r+   r,   Ú<module>   sæ    4|#óòñÝDôÿþýüûø	÷
öõôòñðïî
í
=ÿþýüûùø	÷
öõôóòñ
ð
Sÿþýüûùø	÷
öõôóòñð