o
    ôjRA  ã                    @   sÈ  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de
› de› d�	 e_dee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dededededededededefdd„Zeed�		 	 	 	 d#dee dee dee dee dee d!ee dededededededededefd"d„ƒZdS )$é    )ÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚAdamaxÚadamaxc                       s†   e Zd Z					dddddœded	ed
eeef dededee dededef‡ fdd„Z‡ fdd„Z	dd„ Z
eddd„ƒZ‡  ZS )r   çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   NF)ÚmaximizeÚdifferentiableÚ
capturableÚparamsÚlrÚbetasÚepsÚweight_decayÚforeachr   r   r   c             
      sÄ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|d   kr"dk s,n t d|d › �ƒ‚d|d   kr8dk sBn t d|d › �ƒ‚d|ksMt d	|› �ƒ‚t||||||||	d
�}
tƒ  ||
¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )r   r   r    r!   r"   r   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r    r!   r"   r   r   r   Údefaults©Ú	__class__© úc/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/adamax.pyr(      s*   ø
zAdamax.__init__c                    s¾   t ƒ  |¡ | jD ]S}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]4}| j |g ¡}t|ƒdkr[t |d ¡s[t	|d ƒ}|d rQtj
|tƒ |jd	�ntj
|tƒ d
�|d< q'q	d S )Nr"   r   Fr   r   r   r   Ústep©ÚdtypeÚdevice©r1   )r'   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r2   )r)   r7   ÚgroupÚpÚp_stateÚstep_valr+   r-   r.   r4   ?   s&   
ý
ÿû€üûzAdamax.__setstate__c           
      C   sî   d}|d D ]n}|j d u rq|t |¡O }| |¡ |j jr"tdƒ‚| |j ¡ | j| }	t|	ƒdkr_|d rAtjdt	ƒ |j
d�ntjdt	ƒ d	�|	d
< tj|tjd�|	d< tj|tjd�|	d< | |	d ¡ | |	d ¡ | |	d
 ¡ q|S )NFr   z(Adamax does not support sparse gradientsr   r   r-   r0   r#   r3   r/   )Úmemory_formatÚexp_avgÚexp_inf)Úgradr:   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr7   r9   Úzerosr   r2   r=   Ú
zeros_likeÚpreserve_format)
r)   r>   Úparams_with_gradÚgradsÚexp_avgsÚexp_infsÚstate_stepsÚhas_complexr?   r7   r-   r-   r.   Ú_init_groupR   s2   


ÿý
ÿ
ÿzAdamax._init_groupc                 C   sä   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]K}g }g }g }g }g }|d \}	}
|d }|d }|d }|d }|d }|d }|d	 }|  ||||||¡}t|||||||	|
|||||||d
� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r    r   r!   r"   r   r   r   )
r    Úbeta1Úbeta2r   r!   r"   r   r   r   rR   )Ú _cuda_graph_capture_health_checkr:   Úenable_gradr5   rS   r   )r)   ÚclosureÚlossr>   rM   rN   rO   rP   rQ   rT   rU   r    r   r!   r"   r   r   r   rR   r-   r-   r.   r/   u   sR   
ÿ
ÿñzAdamax.step)r   r   r   r   N©N)Ú__name__Ú
__module__Ú__qualname__r   r<   r   r   Úboolr(   r4   rS   r   r/   Ú__classcell__r-   r-   r+   r.   r      sB    ù	õþý
üûúù	÷
öõ$#aÁ  Implements Adamax algorithm (a variant of Adam based on infinity norm).

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)},
                \: \lambda \text{ (weight decay)},                                                \\
            &\hspace{13mm}    \epsilon \text{ (epsilon)}                                          \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                u_0 \leftarrow 0 \text{ ( infinity norm)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t      \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t               \\
            &\hspace{5mm}u_t      \leftarrow   \mathrm{max}(\beta_2 u_{t-1}, |g_{t}|+\epsilon)   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \frac{\gamma m_t}{(1-\beta^t_1) u_t} \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    a
  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zd

    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980

    r   rN   rO   rP   rQ   r    rT   rU   r   r!   r   r   r   rR   c       	         C   s¦  t | ƒD ]Ì\}}|| }|
s|n| }|| }|| }|| }tj ¡ s?|r?tƒ }|jj|jjkr7|jj|v s?J d|› d�ƒ‚|d7 }|	dkrN|j||	d�}t |¡rgt 	|¡}t 	|¡}t 	|¡}t 	|¡}| 
|d| ¡ |s‚tj| |¡| ¡  |¡|d� n!t | |¡ d¡| ¡  |¡ d¡gd¡}| tj|ddd�¡ |r»|| d }| |¡ || }| ||¡ qd|t|ƒ  }|| }|j||| d	� qd S )
NúIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©Úalpha)ÚoutF)Úkeepdim)Úvalue)Ú	enumerater:   Ú_utilsÚis_compilingr   r2   ÚtypeÚaddrF   Úview_as_realÚlerp_ÚmaximumÚmul_ÚabsÚadd_ÚcatÚ	unsqueezeÚ
unsqueeze_Úcopy_ÚamaxÚdiv_Úaddcdiv_r   )r   rN   rO   rP   rQ   r    rT   rU   r   r!   r   r   r   rR   ÚiÚparamrE   rC   rD   Ústep_tÚcapturable_supported_devicesZnorm_bufZneg_bias_correctionÚdenomZbias_correctionÚclrr-   r-   r.   Ú_single_tensor_adamaxÞ   sR   
ÿ




ý"þ
Êr   c       	            sä  |rJ dƒ‚t | ƒdkrd S tj ¡ s0|r0tdd�‰t‡fdd„t| |ƒD ƒƒs0J dˆ› d�ƒ‚t | ||||g¡}| 	¡ D ]±\\}}}}}}|rPt
||||ƒ |
rWt |¡}|d jrjtj|tjd	d
d�d	d� nt |d¡ |	dkr‡|
rtj|||	d� ntj|||	d�}t ||dˆ  ¡ t ||¡ |
s¢|	dkr¢t |¡}nt |¡ t ||¡ t ||¡ |rÕt ˆ |¡}t |d¡ t |ˆ¡ t ||¡}t |||¡ q>‡ fdd„|D ƒ}‡fdd„|D ƒ}t ||||¡ q>d S )Nz#_foreach ops don't support autogradr   F)Úsupports_xlac                 3   s0   � | ]\}}|j j|j jko|j jˆ v V  qd S rZ   )r2   rj   )Ú.0r?   r/   )r|   r-   r.   Ú	<genexpr>C  s   € þ

þz'_multi_tensor_adamax.<locals>.<genexpr>r`   ra   r$   Úcpu)r2   rb   r   c                    s   g | ]
}d ˆ t |ƒ  ‘qS )r   ©r   )r�   r/   )rT   r-   r.   Ú
<listcomp>‰  s    ÿz(_multi_tensor_adamax.<locals>.<listcomp>c                    s   g | ]
}t ˆ ƒ| d  ‘qS )éÿÿÿÿr„   )r�   Úbc)r   r-   r.   r…   Œ  s    )r9   r:   rh   ri   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Ú_foreach_negÚis_cpuÚ_foreach_add_r=   Ú_foreach_addÚ_foreach_lerp_Ú_foreach_mul_Z_foreach_absZ_foreach_abs_Ú_foreach_maximum_Ú_foreach_powÚ_foreach_sub_Ú_foreach_div_Ú_foreach_mulÚ_foreach_addcdiv_)r   rN   rO   rP   rQ   r    rT   rU   r   r!   r   r   r   rR   Úgrouped_tensorsZgrouped_paramsÚgrouped_gradsZgrouped_exp_avgsZgrouped_exp_infsZgrouped_state_stepsÚ_Zbias_correctionsr}   Ú	step_sizer-   )rT   r|   r   r.   Ú_multi_tensor_adamax(  sz   ÿ
ý
üÿ	úÿ

ÿÿ

ÿÿ¿rœ   )Úsingle_tensor_fnFr"   c
                C   s”   t j ¡ stdd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r*t j ¡ r*tdƒ‚|r4t j ¡ s4t}nt	}|| |||||
|||||||	|d� dS )	zrFunctional API that performs adamax algorithm computation.

    See :class:`~torch.optim.Adamax` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rZ   )Ú
isinstancer:   r   )r�   Útr-   r-   r.   r‚   ¬  s   € 
ÿzadamax.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r    rT   rU   r   r!   r   r   rR   r   )
r:   rh   ri   rˆ   rI   r	   ÚjitÚis_scriptingrœ   r   )r   rN   rO   rP   rQ   r"   r   r   r   rR   r    rT   rU   r   r!   rš   Úfuncr-   r-   r.   r   ’  s>   ÿÿ
ÿ
ò)NFFFF)Útypingr   r   r   r   r:   r   Ú	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r<   r^   r   rœ   r   r-   r-   r-   r.   Ú<module>   sä   < 
öõôóæÿ1ÿþýüûùø	÷
öõôóò
ñJÿþýüûùø	÷
öõôóò
ñj	ôÿþýüûø	÷
öõôòñðïî