o
    !ôjt  ã                )   @   sÊ  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZ d dlmZ ddgZG d	d„ deƒZd
de› de› de› de› de› d� e_							d+dee dee dee dee dee dee dee dededee dee dee dededededeeef d ed!ed"ef(d#d„Zdee dee dee dee dee dee dee dee dededededeeef d ed!ed"ededef$d$d%„Zdee dee dee dee dee dee dee dee dededededeeef d ed!ed"ededef$d&d'„Zdee dee dee dee dee dee dee dee dededededeeef d ed!ed"ededed(df&d)d*„Z dS ),é    )ÚListÚOptionalÚUnionÚTupleN)ÚTensoré   )Ú	OptimizerÚParamsTÚ_use_grad_for_differentiableÚ
_get_valueÚ_stack_if_compilingÚ_dispatch_sqrtÚ_default_to_fused_or_foreachÚ_get_scalar_dtypeÚ_capturable_docÚ_differentiable_docÚ_foreach_docÚ
_fused_docÚ_maximize_docÚ_view_as_real)Ú$_get_fused_kernels_supported_devicesÚAdamÚadamc                       sž   e Zd Z					dddddddœded	eeef d
eeef dededede	e dededede	e f‡ fdd„Z
‡ fdd„Zdd„ Zeddd„ƒZ‡  ZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>r   FN)ÚforeachÚmaximizeÚ
capturableÚdifferentiableÚfusedÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr   r   r   r   r    c                   s6  d|kst d|› �ƒ‚t|tƒr|r|	st dƒ‚d|ks#t d|› �ƒ‚d|d   kr/dk s9n t d|d › �ƒ‚d|d   krEdk sOn t d	|d › �ƒ‚d|ksZt d
|› �ƒ‚t||||||||	|
|d�
}tƒ  ||¡ |r—|
rwtdƒ‚d| _tƒ ‰ t	‡ fdd„| j
D ƒƒs‘tdˆ › d�ƒ‚|r™tdƒ‚d S d S )Nç        zInvalid learning rate: úElr as a Tensor is not supported for capturable=False and foreach=TruezInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )
r"   r#   r$   r%   r&   r   r   r   r   r    z)`fused` does not support `differentiable`Tc                 3   s4   � | ]}|d  D ]}|j jˆ v ot |¡V  qqdS )r!   N)ÚdeviceÚtypeÚtorchÚis_floating_point)Ú.0ÚpgÚp©Úfused_supported_devices© úc/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/adam.pyÚ	<genexpr>8   s   € þÿþz Adam.__init__.<locals>.<genexpr>zX`fused=True` requires all the params to be floating point Tensors of supported devices: Ú.z0`fused` and `foreach` cannot be `True` together.)Ú
ValueErrorÚ
isinstancer   ÚdictÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scalingr   ÚallÚparam_groups)Úselfr!   r"   r#   r$   r%   r&   r   r   r   r   r    Údefaults©Ú	__class__r1   r4   r;      sB   ýþ
ÿðzAdam.__init__c                    sâ   t ƒ  |¡ | jD ]e}| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ | dd ¡}|d D ]:}| j |g ¡}t|ƒd	krmt |d
 ¡smt	|d
 ƒ}|d sW|d rctj
|t|d�|jd�ntj
|tƒ d�|d
< q3q	d S )Nr&   Fr   r   r   r   r    r!   r   Ústep©Úis_fused©Údtyper*   ©rH   )r:   Ú__setstate__r?   Ú
setdefaultÚstateÚgetÚlenr,   Ú	is_tensorÚfloatÚtensorr   r*   )r@   rL   Úgroupr    r0   Zp_stateZstep_valrB   r3   r4   rJ   A   s*   
ÿÿþ€üùzAdam.__setstate__c                 C   sn  d}|d D ]®}	|	j d ur´|t |	¡O }| |	¡ |	j jr!tdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrt|d s:|d rHtjdt	|d d�|	j
d	�ntjd
t	ƒ d�|
d< tj|	tjd�|
d< tj|	tjd�|
d< |d rttj|	tjd�|
d< | |
d ¡ | |
d ¡ |d r�| |
d ¡ |d rš|
d jrštdƒ‚|d r­t |d ¡r­|d s­tdƒ‚| |
d ¡ q|S )NFr!   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r   r    r3   rE   rG   r'   rI   rD   )Úmemory_formatÚexp_avgÚ
exp_avg_sqr&   Úmax_exp_avg_sqr   zB`requires_grad` is not supported for `step` in differentiable moder   r"   r(   )Úgradr,   Ú
is_complexÚappendÚ	is_sparser<   rL   rN   Úzerosr   r*   rQ   Ú
zeros_likeÚpreserve_formatÚrequires_gradrO   )r@   rR   Úparams_with_gradÚgradsÚexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexr0   rL   r3   r3   r4   Ú_init_groupR   s>   



ÿÿý€zAdam._init_groupc                 C   sô   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]S}g }g }g }g }g }g }	|d \}
}|  |||||||	¡}t||||||	f|d ||
||d |d |d |d |d |d	 |d
 |d t| ddƒt| ddƒdœŽ q$|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr#   r&   r"   r%   r$   r   r   r   r   r    Ú
grad_scaleÚ	found_inf)r&   re   Úbeta1Úbeta2r"   r%   r$   r   r   r   r   r    rg   rh   )Ú _cuda_graph_capture_health_checkr,   Úenable_gradr?   rf   r   Úgetattr)r@   ÚclosureÚlossrR   r_   r`   ra   rb   rc   rd   ri   rj   re   r3   r3   r4   rD   ‡   s^   
ÿ
ù	ú


ìz	Adam.step)r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r	   r   rP   r   r   Úboolr   r;   rJ   rf   r
   rD   Ú__classcell__r3   r3   rB   r4   r      sN    úôÿ
þ
ýüûúø	÷
öõô25a  Implements Adam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}          \\
            &\hspace{13mm}      \lambda \text{ (weight decay)},  \: \textit{amsgrad},
                \:\textit{maximize}                                                              \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0\leftarrow 0 \text{ (second moment)},\: \widehat{v_0}^{max}\leftarrow 0\\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max},
                \widehat{v_t})                                                                   \\
            &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big)                                 \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    a�  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        zÄ
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    Fr!   r`   ra   rb   rc   rd   r   r   r   r    rg   rh   re   r&   ri   rj   r"   r%   r$   r   c                C   sü   |	du r|du rt | |dd�\}}|rt|tƒr|sd}|	du r"d}	|du r(d}tj ¡ s:tdd„ |D ƒƒs:tdƒ‚|rEtj 	¡ rEtdƒ‚|	rPtj 	¡ rPtdƒ‚|	rZtj 	¡ sZt
}n|rdtj 	¡ sdt}nt}|| ||||||||||||||||
|d	� dS )
znFunctional API that performs Adam algorithm computation.

    See :class:`~torch.optim.Adam` for details.
    NF)Ú	use_fusedc                 s   s   � | ]	}t |tjƒV  qd S rp   )r8   r,   r   )r.   Útr3   r3   r4   r5   /  s   € zadam.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r&   re   ri   rj   r"   r%   r$   r   r   r   rg   rh   )r   r8   r   r,   Ú_utilsÚis_compilingr>   r<   ÚjitÚis_scriptingÚ_fused_adamÚ_multi_tensor_adamÚ_single_tensor_adam)r!   r`   ra   rb   rc   rd   r   r   r   r    rg   rh   re   r&   ri   rj   r"   r%   r$   r   Ú_Úfuncr3   r3   r4   r     sL   
ïc       
          C   sž  |d u r|d u s
J ‚t j ¡ rt|tƒsJ ‚t| ƒD �]1\}}|s%|| n||  }|| }|| }|| }t j ¡ sM|rM|jrC|jsM|j	rI|j	sMJ dƒ‚|d7 }|dkr\|j
||d�}t  |¡r€t  |¡}t  |¡}t  |¡}|r{t  || ¡||< t  |¡}| |d|
 ¡ | |¡j|| ¡ d| d� |s›|ró|}d|
|  }d||  }|| }| ¡ }| ¡ }|rß|rÀ||  ¡ }n|| }||  t  ||¡¡ ||  ¡ ||   || ¡}n| ¡ ||   || ¡}| ||¡ nEt|ƒ}d|
|  }d||  }|| }t|ƒ}|�r&t j|| ||| d� ||  ¡ |  |¡}n	| ¡ |  |¡}|j||| d� |�rLt  | | ¡�rLt  || ¡||< qd S )NzGIf capturable=True, params and state_steps must be CUDA or XLA tensors.r   r   ©Úalpha)Úvalue)Úout)r,   rz   r{   r8   rP   Ú	enumeraterx   ry   Úis_cudaÚis_xlaÚaddrX   Úview_as_realÚlerp_Úmul_Úaddcmul_ÚconjÚnegÚsqrtÚcloneÚcopy_ÚmaximumÚadd_Úaddcdiv_r   r   Úview_as_complex) r!   r`   ra   rb   rc   rd   rg   rh   r&   re   ri   rj   r"   r%   r$   r   r   r   ÚiÚparamrW   rT   rU   Ústep_trD   Úbias_correction1Úbias_correction2Ú	step_sizeZstep_size_negÚbias_correction2_sqrtrV   Údenomr3   r3   r4   r~   R  sx   
ÿÿÿÿþ




 €¯r~   c       
            sÒ  t | ƒdkrd S tˆtƒr|stdƒ‚tj ¡ s*|r*tdd„ t| |ƒD ƒƒs*J dƒ‚|d u r2|d u s4J ‚|r:J dƒ‚t	 
| |||||g¡}| ¡ D �]\\}}}}}}}|	rh|rat|||||ƒ nt||||ƒ |rot |¡}|d jr‚tj|tjddd	�dd
� nt |d¡ |dkrŸ|r—tj|||d
� ntj|||d
�}t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ ~|�rt ˆ |¡}t ˆ|¡}t |d¡ t |d¡ t |¡ t |ˆ¡ t |¡ t |¡ |}|}|rût ||¡ t |¡}nt |¡}t ||¡ t ||¡ t ||¡ t |||¡ qI‡ fdd„|D ƒ}‡fdd„|D ƒ}t‡fdd„|D ƒƒ}dd„ |D ƒ}|�rMt ||¡ t |¡}nt |¡}t ||¡ t ||¡ t ||||¡ qId S )Nr   r(   c                 s   s    � | ]\}}|j o|j V  qd S rp   )r†   )r.   r0   rD   r3   r3   r4   r5   Ý  s   € z%_multi_tensor_adam.<locals>.<genexpr>z@If capturable=True, params and state_steps must be CUDA tensors.z#_foreach ops don't support autogradr)   Úcpu)r*   r�   r   c                    ó   g | ]
}d ˆ t |ƒ  ‘qS ©r   ©r   ©r.   rD   )ri   r3   r4   Ú
<listcomp>7  ó    z&_multi_tensor_adam.<locals>.<listcomp>c                    rŸ   r    r¡   r¢   )rj   r3   r4   r£   8  r¤   c                    s   g | ]}ˆ | d  ‘qS )éÿÿÿÿr3   ©r.   Zbc)r"   r3   r4   r£   :  s    c                 S   s   g | ]}t |ƒ‘qS r3   )r   r¦   r3   r3   r4   r£   <  s    )rN   r8   r   r<   r,   rx   ry   r>   Úzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Ú_foreach_negÚis_cpuÚ_foreach_add_rQ   Ú_foreach_addZ_foreach_lerp_Ú_foreach_mul_Ú_foreach_addcmul_Z_foreach_powÚ_foreach_sub_Z_foreach_neg_Ú_foreach_div_Z_foreach_reciprocal_Ú_foreach_sqrt_Z_foreach_maximum_Ú_foreach_sqrtÚ_foreach_addcdiv_r   )r!   r`   ra   rb   rc   rd   rg   rh   r&   re   ri   rj   r"   r%   r$   r   r   r   Úgrouped_tensorsÚdevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_max_exp_avg_sqsÚdevice_state_stepsr   r™   rš   r›   rœ   Zexp_avg_sq_sqrtr3   )ri   rj   r"   r4   r}   Â  sŒ   ÿÿ	
ù






�r}   Úreturnc       
          C   s~  | sd S |r
t dƒ‚|d ur|j|ind }|d ur|j|ind }t|tƒr1t|jƒdkr1|j|ind }t | |||||g¡}| ¡ D ]z\\}}\\}}}}}}}d\}}|d uri||vre|j|dd�||< || }|d ur~||vrz|j|dd�||< || }|d ur“||vr“|j|dd�||< || }t	 
|d¡ t	j|||||||||
||||||d� |d ur¼t	 ||gt|ƒ ¡ qBd S )	Nz9Adam with fused=True does not support differentiable=Truerž   )NNT)Únon_blocking)r*   r½   r   )	r&   r"   ri   rj   r%   r$   r   rg   rh   )r<   r*   r8   r   Ústrr   r¨   ÚitemsÚtor,   r¬   Z_fused_adam_r°   rN   ) r!   r`   ra   rb   rc   rd   rg   rh   r&   re   ri   rj   r"   r%   r$   r   r   r   Zgrad_scale_dictZfound_inf_dictZlr_dictrµ   r*   r   r¶   r·   r¸   r¹   rº   r»   Zdevice_grad_scaleZdevice_found_infr3   r3   r4   r|   L  sf   &ÿûñ€Ûr|   )NFFNNNF)!Útypingr   r   r   r   r,   r   Ú	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   Útorch.utils._foreach_utilsr   Ú__all__r   Ú__doc__rt   rP   r   r~   r}   r|   r3   r3   r3   r4   Ú<module>   sb   @ 5&ñðïîíÚJòÿþýüûø	÷
öõôóòðïî
íìë
ê
Nÿþýüûúù	÷
öõô
óòñðï
î
pÿþýüûúù	÷
öõô
óòñðï
î ÿþýüûúùø
öõôó
òñðïîíì