o
    ôj²c  ã                &   @   s  d dl mZmZmZmZmZ d dlZd dlmZ ddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de	› de› d�	 e_dee dee dee dee dee dee dededededededededededef"dd„Zdee dee dee dee dee dee dededededededededededef"d d!„Zeed"�	#		#	#	#	#d&dee dee dee dee dee dee ded$ee dedededededededededef$d%d„ƒZdS )'é    )ÚcastÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_dispatch_sqrtÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_stack_if_compilingÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚNAdamÚnadamc                       s’   e Zd Z						ddddddœd	ed
edeeef dededededee dededef‡ fdd„Z‡ fdd„Z	dd„ Z
eddd„ƒZ‡  ZS )r   çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   çü©ñÒMbp?FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiableÚparamsÚlrÚbetasÚepsÚweight_decayÚmomentum_decayÚdecoupled_weight_decayr   r   r    r!   c                   sÞ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|d   kr"dk s,n t d|d › �ƒ‚d|d   kr8dk sBn t d|d › �ƒ‚d|ksMt d	|› �ƒ‚d|ksXt d
|› �ƒ‚t|||||||	||
|d�
}tƒ  ||¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: zInvalid momentum_decay value: )
r#   r$   r%   r&   r'   r(   r   r   r    r!   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr"   r#   r$   r%   r&   r'   r(   r   r   r    r!   Údefaults©Ú	__class__© úb/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/nadam.pyr.      s2   özNAdam.__init__c                    s  t ƒ  |¡ | jD ]|}| dd¡ | dd ¡ | dd¡ | dd¡ | dd¡ |d D ]W}| j |g ¡}t|ƒdkr„t |d	 ¡sat	|d	 ƒ}|d rWtj
|tƒ |jd
�ntj
|tƒ d�|d	< t |d ¡s„|d }|d rztj
|tƒ |jd
�ntj
|tƒ d�|d< q-q	d S )Nr   Fr   r    r!   r(   r"   r   Ústep©ÚdtypeÚdevice©r7   Ú
mu_product)r-   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r8   )r/   r>   ÚgroupÚpÚp_stateÚstep_valZmu_prod_valr1   r3   r4   r;   E   s:   
ý
ÿûý
ÿû€òúzNAdam.__setstate__c                 C   s*  d}|d D ]Œ}	|	j d ur’|t |	¡O }| |	¡ |	j jr!tdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrv|d r@tjdt	ƒ |	j
d�ntjdt	ƒ d	�|
d
< |d rXtjdt	ƒ |	j
d�ntjdt	ƒ d	�|
d< tj|	tjd�|
d< tj|	tjd�|
d< | |
d ¡ | |
d ¡ | |
d ¡ | |
d
 ¡ q|S )NFr"   z'NAdam does not support sparse gradientsr   r    r3   r6   r)   r9   r5   r*   r:   )Úmemory_formatÚexp_avgÚ
exp_avg_sq)ÚgradrA   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr>   r@   Úzerosr   r8   rD   ÚonesÚ
zeros_likeÚpreserve_format)r/   rE   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmu_productsÚstate_stepsÚhas_complexrF   r>   r3   r3   r4   Ú_init_groupc   s<   



ÿýÿý
ÿ
ÿ€zNAdam._init_groupc                 C   sê   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]N}g }g }g }g }g }g }	ttttf |d ƒ\}
}|  |||||||	¡}t||||||	|
||d |d |d |d |d |d |d	 |d
 |d |d� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr$   r#   r&   r'   r%   r   r(   r   r    r!   )Úbeta1Úbeta2r#   r&   r'   r%   r   r(   r   r    r!   r[   )	Ú _cuda_graph_capture_health_checkrA   Úenable_gradr<   r   r   rC   r\   r   )r/   ÚclosureÚlossrE   rU   rV   rW   rX   rY   rZ   r]   r^   r[   r3   r3   r4   r5   •   sX   
ÿ
ù
îz
NAdam.step)r   r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r   rC   r   Úboolr   r.   r;   r\   r   r5   Ú__classcell__r3   r3   r1   r4   r      sN    ø
óþý
üûúùø
öõôó)2a¼  Implements NAdam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)},
                \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)}                   \\
            &\hspace{13mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)}    \\
            &\hspace{13mm} \: \textit{decoupled\_weight\_decay}, \:\textit{maximize}             \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0 \leftarrow 0 \text{ ( second moment)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1}                                       \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm}\textbf{if} \: \textit{decoupled\_weight\_decay}                       \\
            &\hspace{15mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}                    \\
            &\hspace{10mm}\textbf{else}                                                          \\
            &\hspace{15mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2}  0.96^{t \psi} \big)     \\
            &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{(t+1)\psi}\big)\\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i)\\[-1.ex]
            & \hspace{11mm} + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i})                         \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Incorporating Nesterov Momentum into Adam`_.
    a  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        momentum_decay (float, optional): momentum momentum_decay (default: 4e-3)
        decoupled_weight_decay (bool, optional): whether to use decoupled weight
            decay as in AdamW to obtain NAdamW (default: False)
        z	
        zÏ

    .. _Incorporating Nesterov Momentum into Adam:
        https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101

    r"   rV   rW   rX   rY   rZ   r]   r^   r#   r&   r'   r%   r(   r   r    r!   r[   c                C   sZ  t | ƒD �]%\}}|s|| n||  }|| }|| }|| }|| }t |¡r=t |¡}t |¡}t |¡}t |¡}tj ¡ sg|rgtƒ }|jj|jj  krW|jjkr_n n|jj|v sgJ d|› d�ƒ‚|d7 }|rp|}nt	|ƒ}d||  }|	dkr‘|rŠ| 
d||	  ¡ n|j||	d�}|ddd||
     }|ddd|d |
     }||9 }| |d| ¡ | 
|¡j||d| d	� | |¡ ¡ }|sÏ|rý| |¡}|| }|| d|  d|   }|| | d|   }| ||¡ | ||¡ qt	|ƒ| }| |¡ |j||| d|  dt	|ƒ  d	� |j||| | d|  d	� qd S )
NzVIf capturable=True, params, mu_products and state_steps must be on supported devices: Ú.r   r   ©Úalphar*   ç      à?ç¸…ëQ¸î?)Úvalue)Ú	enumeraterA   rM   Úview_as_realÚ_utilsÚis_compilingr   r8   Útyper   Úmul_ÚaddÚlerp_Úaddcmul_ÚdivÚsqrtÚaddcdiv_Úadd_)r"   rV   rW   rX   rY   rZ   r]   r^   r#   r&   r'   r%   r(   r   r    r!   r[   ÚiÚparamrL   rJ   rK   r:   Ústep_tÚcapturable_supported_devicesr5   Úbias_correction2ÚmuÚmu_nextÚdenomZmu_product_nextr3   r3   r4   Ú_single_tensor_nadam  sd   




$ÿþ

ÿÿ»r„   c          "         s~  t | ƒdkrd S |rJ dƒ‚tj ¡ s1|r1tdd�‰t‡fdd„t| ||ƒD ƒƒs1J dˆ› d�ƒ‚t | |||||g¡}| 	¡ D �]{\\}}}}}}}|rTt
||||ƒ |r[t |¡}|d jrntj|tjd	d
d�d	d� nt |d¡ |	dkr˜|r…t |dˆ|	  ¡ n|r�tj|||	d� ntj|||	d�}t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ t |¡}|�rt |ˆ¡}t d|¡}t |d¡ t |d	¡ t |ˆ ¡ t |ˆ¡ t d|¡}t |d¡ t |d	¡ t |ˆ ¡ ~t ˆ|¡}t |d	¡ t |¡ t |¡ n‡fdd„|D ƒ}‡ ‡fdd„|D ƒ}‡ ‡fdd„|D ƒ}t ||¡ t ||¡ t ||¡ ~|�r�t |d	¡ t |ˆ¡ t |d	¡}t |¡ t ||¡ |}~t ||¡}t |ˆ¡ t |d	¡ t ||¡ |} ~t ||¡}!t |!| |¡ t ||!|¡ q@t‡fdd„t||ƒD ƒƒ}t‡fdd„t||ƒD ƒƒ} t ||||¡ t |||| ¡ q@d S )Nr   z#_foreach ops don't support autogradF)Úsupports_xlac                 3   sF   � | ]\}}}|j j|j j  ko|j jkn  o|j jˆ v V  qd S rc   )r8   rs   )Ú.0rF   Úmpr5   )r   r3   r4   Ú	<genexpr>Ž  s   € $þ

þz&_multi_tensor_nadam.<locals>.<genexpr>zWIf capturable=True, params, mu_products, and state_steps must be on supported devices: ri   r*   Úcpu)r8   rj   r   rm   g      à¿c                    s    g | ]}t d ˆ t|ƒ  ƒ‘qS )r   )r   r   ©r†   r5   )r^   r3   r4   Ú
<listcomp>ç  s    ÿÿz'_multi_tensor_nadam.<locals>.<listcomp>c                    s(   g | ]}ˆ d ddt |ƒˆ     ‘qS )r*   rl   rm   ©r   rŠ   ©r]   r'   r3   r4   r‹   ë  s    ÿÿc                    s,   g | ]}ˆ d ddt |ƒd ˆ     ‘qS )r*   rl   rm   r   rŒ   rŠ   r�   r3   r4   r‹   ï  s    þÿÿc                    s0   g | ]\}}t ˆ ƒd |  d t |ƒ  d ‘qS ©r*   éÿÿÿÿrŒ   )r†   r:   r�   ©r#   r3   r4   r‹      s    ÿÿc                    s0   g | ]\}}t ˆ ƒ| d t |ƒ|   d ‘qS rŽ   rŒ   )r†   r:   r‚   r�   r3   r4   r‹   &  s    ûÿþûÿ)r@   rA   rq   rr   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Ú_foreach_negÚis_cpuÚ_foreach_add_rD   Ú_foreach_mul_Ú_foreach_addÚ_foreach_lerp_Ú_foreach_addcmul_Ú_foreach_sqrtÚ_foreach_mulÚ_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_sqrt_Ú_foreach_div_Ú_foreach_subÚ_foreach_addcdiv_r   )"r"   rV   rW   rX   rY   rZ   r]   r^   r#   r&   r'   r%   r(   r   r    r!   r[   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsÚgrouped_exp_avgsZgrouped_exp_avg_sqsZgrouped_mu_productsÚgrouped_state_stepsÚ_Úexp_avg_sq_sqrtÚexponentÚmusZmu_nextsZbias_correction_sqrtrƒ   Zstep_size_gradsZstep_size_expavgÚ	numeratorr3   )r]   r^   r   r#   r'   r4   Ú_multi_tensor_nadamp  sæ   ÿ

ý
üÿ

ùÿ

ÿÿÿÿ


þþý

þÿ
ùÿÿÿ �âr¯   )Úsingle_tensor_fnFr   c                C   sª   t dd„ |D ƒƒstdƒ‚t dd„ |D ƒƒstdƒ‚|du r't| |	dd�\}}|r2tj ¡ r2td	ƒ‚|r<tj ¡ s<t}nt}|| |||||||||||||||	|
d
� dS )zpFunctional API that performs NAdam algorithm computation.

    See :class:`~torch.optim.NAdam` for details.
    c                 s   ó   � | ]	}t |tjƒV  qd S rc   ©Ú
isinstancerA   r   ©r†   Útr3   r3   r4   rˆ   V  ó   € znadam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsc                 s   r±   rc   r²   r´   r3   r3   r4   rˆ   [  r¶   zPAPI has changed, `mu_products` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)r]   r^   r#   r&   r'   r   r(   r%   r    r!   r[   )r‘   rP   r
   rA   ÚjitÚis_scriptingr¯   r„   )r"   rV   rW   rX   rY   rZ   r(   r   r    r!   r[   r   r]   r^   r#   r&   r'   r%   rª   Úfuncr3   r3   r4   r   9  sH   ÿÿ
ÿ
ï)FNFFFF) Útypingr   r   r   r   r   rA   r   Ú	optimizerr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__rC   rg   r„   r¯   r   r3   r3   r3   r4   Ú<module>   s  D 6'óòñðÙÿCÿþýüûúø	÷
öõôóòñðï
î^ÿþýüûúø	÷
öõôóòñðï
î J
òÿþýüûú	÷
öõôóòðïîíìë