o
    !ôjï;  ã                   @   sª  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	m
Z
mZmZ d dlmZmZ ddgZG dd„ deƒZd	d
e
› de	› de› de› d�	 e_					d"dee dee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dededededededededefd d!„ZdS )#é    N)ÚTensoré   )
Ú	OptimizerÚ_use_grad_for_differentiableÚ
_get_valueÚ_default_to_fused_or_foreachÚ_get_scalar_dtypeÚ_differentiable_docÚ_maximize_docÚ_foreach_docÚ_view_as_realÚ_capturable_doc)ÚListÚOptionalÚAdamaxÚadamaxc                
       sj   e Zd Z					dddddœdee d	ed
edef‡ fdd„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   çü©ñÒMb`?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   NF)ÚmaximizeÚdifferentiableÚ
capturableÚforeachr   r   r   c             
      sÄ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|d   kr"dk s,n t d|d › �ƒ‚d|d   kr8dk sBn t d|d › �ƒ‚d|ksMt d	|› �ƒ‚t||||||||	d
�}
tƒ  ||
¡ d S )Nç        zInvalid learning rate: zInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )ÚlrÚbetasÚepsÚweight_decayr   r   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr   r   r   r   r   r   r   r   Údefaults©Ú	__class__© úe/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/adamax.pyr"      s*   ø
zAdamax.__init__c                    s¾   t ƒ  |¡ | jD ]S}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]4}| j |g ¡}t|ƒdkr[t |d ¡s[t	|d ƒ}|d rQtj
|tƒ |jd	�ntj
|tƒ d
�|d< q'q	d S )Nr   r   Fr   r   r$   r   Ústep©ÚdtypeÚdevice©r,   )r!   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r-   )r#   r2   ÚgroupÚpÚp_stateÚstep_valr&   r(   r)   r/   1   s    
ÿ€üûzAdamax.__setstate__c           
      C   sî   d}|d D ]n}|j d u rq|t |¡O }| |¡ |j jr"tdƒ‚| |j ¡ | j| }	t|	ƒdkr_|d rAtjdt	ƒ |j
d�ntjdt	ƒ d	�|	d
< tj|tjd�|	d< tj|tjd�|	d< | |	d ¡ | |	d ¡ | |	d
 ¡ q|S )NFr$   z(Adamax does not support sparse gradientsr   r   r(   r+   r   r.   r*   )Úmemory_formatÚexp_avgÚexp_inf)Úgradr5   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr2   r4   Úzerosr   r-   r8   Ú
zeros_likeÚpreserve_format)
r#   r9   Úparams_with_gradÚgradsÚexp_avgsÚexp_infsÚstate_stepsÚhas_complexr:   r2   r(   r(   r)   Ú_init_group?   s2   


ÿÿ
ÿ
ÿzAdamax._init_groupc                 C   sä   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]K}g }g }g }g }g }|d \}	}
|d }|d }|d }|d }|d }|d }|d	 }|  ||||||¡}t|||||||	|
|||||||d
� q$|S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   )
r   Úbeta1Úbeta2r   r   r   r   r   r   rM   )Ú _cuda_graph_capture_health_checkr5   Úenable_gradr0   rN   r   )r#   ÚclosureÚlossr9   rH   rI   rJ   rK   rL   rO   rP   r   r   r   r   r   r   r   rM   r(   r(   r)   r*   ]   sN   
ÿ
ñzAdamax.step)r   r   r   r   N©N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr"   r/   rN   r   r*   Ú__classcell__r(   r(   r&   r)   r      s.    ù	õù	÷
öõ$aÁ  Implements Adamax algorithm (a variant of Adam based on infinity norm).

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)},
                \: \lambda \text{ (weight decay)},                                                \\
            &\hspace{13mm}    \epsilon \text{ (epsilon)}                                          \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                u_0 \leftarrow 0 \text{ ( infinity norm)}                                 \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t      \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t               \\
            &\hspace{5mm}u_t      \leftarrow   \mathrm{max}(\beta_2 u_{t-1}, |g_{t}|+\epsilon)   \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \frac{\gamma m_t}{(1-\beta^t_1) u_t} \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    a
  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 2e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zd

    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980

    Fr$   rI   rJ   rK   rL   r   r   r   r   rM   r   rO   rP   r   r   c
                C   sŠ   t dd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r%tj ¡ r%tdƒ‚|r/tj ¡ s/t}nt}|| |||||
|||||||	|d� dS )	zrFunctional API that performs adamax algorithm computation.

    See :class:`~torch.optim.Adamax` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rU   )Ú
isinstancer5   r   )Ú.0Útr(   r(   r)   Ú	<genexpr>Ú   s   € zadamax.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   rO   rP   r   r   r   r   rM   r   )ÚallrD   r   r5   ÚjitÚis_scriptingÚ_multi_tensor_adamaxÚ_single_tensor_adamax)r$   rI   rJ   rK   rL   r   r   r   r   rM   r   rO   rP   r   r   Ú_Úfuncr(   r(   r)   r   Á   s6   ÿ
òc       	         C   s”  t | ƒD ]Ã\}}|| }|
s|n| }|| }|| }|| }tj ¡ s6|r6|jr,|js6|jr2|js6J dƒ‚|d7 }|	dkrE|j||	d�}t |¡r^t |¡}t |¡}t |¡}t |¡}| 	|d| ¡ |sytj
| |¡| ¡  |¡|d� n!t | |¡ d¡| ¡  |¡ d¡gd¡}| tj|ddd�¡ |r²|| d }| |¡ || }| ||¡ qd|t|ƒ  }|| }|j||| d� qd S )	NzGIf capturable=True, params and state_steps must be CUDA or XLA tensors.r   r   ©Úalpha)ÚoutF)Úkeepdim)Úvalue)Ú	enumerater5   Ú_utilsÚis_compilingÚis_cudaÚis_xlaÚaddrA   Úview_as_realÚlerp_ÚmaximumÚmul_ÚabsÚadd_ÚcatÚ	unsqueezeÚ
unsqueeze_Úcopy_ÚamaxÚdiv_Úaddcdiv_r   )r$   rI   rJ   rK   rL   r   rO   rP   r   r   r   r   r   rM   ÚiÚparamr@   r>   r?   Ústep_tZnorm_bufZneg_bias_correctionÚdenomZbias_correctionÚclrr(   r(   r)   rd   ü   sT   ÿÿþ




ý$ÿ
Írd   c       	            sÎ  |rJ dƒ‚t | ƒdkrd S tj ¡ s%|r%tdd„ t| |ƒD ƒƒs%tdƒ‚t | ||||g¡}| 	¡ D ]±\\}}}}}}|rEt
||||ƒ |
rLt |¡}|d jr_tj|tjddd�dd	� nt |d
¡ |dkr||
rttj|||d	� ntj|||d	�}t ||d
ˆ  ¡ t ||¡ |
s—|dkr—t |¡}nt |¡ t ||	¡ t ||¡ |rÊt ˆ |¡}t |d
¡ t |ˆ¡ t ||¡}t |||¡ q3‡ fdd„|D ƒ}‡fdd„|D ƒ}t ||||¡ q3d S )Nz#_foreach ops don't support autogradr   c                 s   s    � | ]\}}|j o|j V  qd S rU   )ro   )r\   r:   r*   r(   r(   r)   r^   \  s   € z'_multi_tensor_adamax.<locals>.<genexpr>z@If capturable=True, params and state_steps must be CUDA tensors.r   Úcpu)r-   rg   r   c                    s   g | ]
}d ˆ t |ƒ  ‘qS )r   )r   )r\   r*   )rO   r(   r)   Ú
<listcomp>‘  s    z(_multi_tensor_adamax.<locals>.<listcomp>c                    s   g | ]}ˆ | d  ‘qS )éÿÿÿÿr(   )r\   Úbc)r   r(   r)   r…   ’  s    )r4   r5   rm   rn   r`   ÚziprD   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Ú_foreach_negÚis_cpuÚ_foreach_add_r8   Ú_foreach_addÚ_foreach_lerp_Ú_foreach_mul_Z_foreach_absZ_foreach_abs_Ú_foreach_maximum_Ú_foreach_powÚ_foreach_sub_Ú_foreach_div_Ú_foreach_mulÚ_foreach_addcdiv_)r$   rI   rJ   rK   rL   rO   rP   r   r   r   r   r   r   rM   Úgrouped_tensorsZgrouped_paramsÚgrouped_gradsZgrouped_exp_avgsZgrouped_exp_infsZgrouped_state_stepsre   Zbias_correctionsr‚   Ú	step_sizer(   )rO   r   r)   rc   C  sJ   ÿ


Írc   )NFFFF)r5   r   Ú	optimizerr   r   r   r   r   r	   r
   r   r   r   Útypingr   r   Ú__all__r   Ú__doc__rY   r7   r   rd   rc   r(   r(   r(   r)   Ú<module>   sà    0 
öõôóç6ôÿþýüûø	÷
öõôòñðï
î;ÿþýüûùø	÷
öõôóò
ñGÿþýüûùø	÷
öõôóòñ