o
    !ôjŠo  ã                )   @   sÊ  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZ d dlmZmZmZmZ d dlmZ ddgZG d	d„ deƒZd
de› de› de	› de
› de› d� e_							d+dee dee dee dee dee dee dee dededee dee dee dededededeeef d ed!ed"ef(d#d„Zdee dee dee dee dee dee dee dee dedededeeef d ed!ed"edededef$d$d%„Zdee dee dee dee dee dee dee dee dedededeeef d ed!ed"edededef$d&d'„Zdee dee dee dee dee dee dee dee dedededeeef d ed!ed"edededed(df&d)d*„Z dS ),é    N)ÚTensoré   )Ú	OptimizerÚ_use_grad_for_differentiableÚ
_get_valueÚ_dispatch_sqrtÚ_stack_if_compilingÚ_get_scalar_dtypeÚ_capturable_docÚ_differentiable_docÚ_foreach_docÚ
_fused_docÚ_maximize_docÚ_default_to_fused_or_foreachÚParamsTÚ_view_as_real)ÚListÚOptionalÚTupleÚUnion)Ú$_get_fused_kernels_supported_devicesÚAdamWÚadamwc                       sž   e Zd Z					dddddddœded	eeef d
eeef dedededede	e dedede	e f‡ fdd„Z
‡ fdd„Zdd„ Zeddd„ƒZ‡  ZS )r   çü©ñÒMbP?©gÍÌÌÌÌÌì?g+‡ÙÎ÷ï?ç:Œ0âŽyE>ç{®Gáz„?FN)ÚmaximizeÚforeachÚ
capturableÚdifferentiableÚfusedÚparamsÚlrÚbetasÚepsÚweight_decayÚamsgradr   r   r   r    r!   c                   s6  d|kst d|› �ƒ‚t|tƒr|r|	st dƒ‚d|ks#t d|› �ƒ‚d|d   kr/dk s9n t d|d › �ƒ‚d|d   krEdk sOn t d	|d › �ƒ‚d|ksZt d
|› �ƒ‚t||||||||	|
|d�
}tƒ  ||¡ |r—|
rwtdƒ‚d| _tƒ ‰ t	‡ fdd„| j
D ƒƒs‘tdˆ › d�ƒ‚|r™tdƒ‚d S d S )Nç        zInvalid learning rate: úElr as a Tensor is not supported for capturable=False and foreach=TruezInvalid epsilon value: r   ç      ð?z#Invalid beta parameter at index 0: r   z#Invalid beta parameter at index 1: zInvalid weight_decay value: )
r#   r$   r%   r&   r'   r   r   r   r    r!   z)`fused` does not support `differentiable`Tc                 3   s4   � | ]}|d  D ]}|j jˆ v ot |¡V  qqdS )r"   N)ÚdeviceÚtypeÚtorchÚis_floating_point)Ú.0ÚpgÚp©Úfused_supported_devices© úd/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/adamw.pyÚ	<genexpr>@   s   € ýþþz!AdamW.__init__.<locals>.<genexpr>zX`fused=True` requires all the params to be floating point Tensors of supported devices: Ú.z0`fused` and `foreach` cannot be `True` together.)Ú
ValueErrorÚ
isinstancer   ÚdictÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scalingr   ÚallÚparam_groups)Úselfr"   r#   r$   r%   r&   r'   r   r   r   r    r!   Údefaults©Ú	__class__r2   r5   r<      sP   öý
ÿïzAdamW.__init__c                    sâ   t ƒ  |¡ | jD ]e}| dd¡ | dd¡ | dd ¡ | dd¡ | dd¡ | dd ¡}|d D ]:}| j |g ¡}t|ƒd	krmt |d
 ¡smt	|d
 ƒ}|d sW|d rctj
|t|d�|jd�ntj
|tƒ d�|d
< q3q	d S )Nr'   Fr   r   r   r    r!   r"   r   Ústep©Úis_fused©Údtyper+   ©rI   )r;   Ú__setstate__r@   Ú
setdefaultÚstateÚgetÚlenr-   Ú	is_tensorÚfloatÚtensorr	   r+   )rA   rM   Úgroupr!   r1   Úp_stateÚstep_valrC   r4   r5   rK   J   s*   
ÿÿþ€üùzAdamW.__setstate__c	                 C   sl  d}	|d D ]­}
|
j d u rq|	t |
¡O }	| |
¡ |
j jr"tdƒ‚| |
j ¡ | j|
 }t|ƒdkrs|d s;|d rItjdt	|d d�|
j
d	�ntjd
t	ƒ d�|d< tj|
tjd�|d< tj|
tjd�|d< |rstj|
tjd�|d< | |d ¡ | |d ¡ |d rŒ| |d ¡ |d r™|d jr™tdƒ‚|d r¬t|d tƒr¬|d s¬tdƒ‚| |d ¡ q|	S )NFr"   z'AdamW does not support sparse gradientsr   r   r!   r4   rF   rH   r(   rJ   rE   )Úmemory_formatÚexp_avgÚ
exp_avg_sqÚmax_exp_avg_sqr'   r    zB`requires_grad` is not supported for `step` in differentiable moder   r#   r)   )Úgradr-   Ú
is_complexÚappendÚ	is_sparser=   rM   rO   Úzerosr	   r+   rR   Ú
zeros_likeÚpreserve_formatÚrequires_gradr9   r   )rA   rS   Úparams_with_gradÚgradsr'   Úexp_avgsÚexp_avg_sqsÚmax_exp_avg_sqsÚstate_stepsÚhas_complexr1   rM   r4   r4   r5   Ú_init_group[   sJ   


ÿÿý
ÿ
ÿ
ÿzAdamW._init_groupc                 C   sú   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]V}g }g }g }g }g }g }	|d }
|d \}}|  ||||
||||	¡}t||||||	f|
|||d |d |d |d |d |d	 |d
 |d t| ddƒt| ddƒ|dœŽ q$|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr'   r$   r#   r&   r%   r   r   r   r    r!   Ú
grad_scaleÚ	found_inf)r'   Úbeta1Úbeta2r#   r&   r%   r   r   r   r    r!   rj   rk   rh   )Ú _cuda_graph_capture_health_checkr-   Úenable_gradr@   ri   r   Úgetattr)rA   ÚclosureÚlossrS   rb   rc   rd   re   rf   rg   r'   rl   rm   rh   r4   r4   r5   rE   ˜   sb   
ÿ
øú


ìz
AdamW.step)r   r   r   r   F©N)Ú__name__Ú
__module__Ú__qualname__r   r   rQ   r   r   Úboolr   r<   rK   ri   r   rE   Ú__classcell__r4   r4   rC   r5   r      sN    ù	óþ
ý
üûúù	÷
öõôó<=a  Implements AdamW algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{(lr)}, \: \beta_1, \beta_2
                \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)},
                \: \epsilon \text{ (epsilon)}                                                    \\
            &\hspace{13mm}      \lambda \text{(weight decay)},  \: \textit{amsgrad},
                \: \textit{maximize}                                                             \\
            &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0
                \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0              \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1}         \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max},
                \widehat{v_t})                                                                   \\
            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big)                                 \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_.
    a�  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay coefficient (default: 1e-2)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        z¿
    .. _Decoupled Weight Decay Regularization:
        https://arxiv.org/abs/1711.05101
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    Fr"   rc   rd   re   rf   rg   r   r   r    r!   rj   rk   rh   r'   rl   rm   r#   r&   r%   r   c                C   sü   t j ¡ stdd„ |D ƒƒstdƒ‚|	du r.|du r.t| |dd�\}}|r.t|tƒr.|s.d}|	du r4d}	|du r:d}|rEt j 	¡ rEtdƒ‚|	rPt j 	¡ rPtdƒ‚|	rZt j 	¡ sZt
}n|rdt j 	¡ sdt}nt}|| |||||||||||||||
||d	� dS )
zpFunctional API that performs AdamW algorithm computation.

    See :class:`~torch.optim.AdamW` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rs   )r9   r-   r   )r/   Útr4   r4   r5   r6   5  s   € zadamw.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r'   rl   rm   r#   r&   r%   r   r   r    rj   rk   rh   )r-   Ú_utilsÚis_compilingr?   r=   r   r9   r   ÚjitÚis_scriptingÚ_fused_adamwÚ_multi_tensor_adamwÚ_single_tensor_adamw)r"   rc   rd   re   rf   rg   r   r   r    r!   rj   rk   rh   r'   rl   rm   r#   r&   r%   r   Ú_Úfuncr4   r4   r5   r     sR   ÿ
îc       
          C   s–  |d u r|d u s
J ‚t j ¡ rt|tƒsJ ‚t| ƒD �]-\}}|s%|| n||  }|| }|| }|| }t j ¡ sM|rM|jrC|jsM|j	rI|j	sMJ dƒ‚t  
|¡rqt  |¡}t  |¡}t  |¡}|rlt  || ¡||< t  |¡}|d7 }| d||  ¡ | |d|	 ¡ | |
¡j||d|
 d� |s—|rï|}d|	|  }d|
|  }|| }| ¡ }| ¡ }|rÛ|r¼||  ¡ }n|| }||  t  ||¡¡ ||  ¡ ||   || ¡}n| ¡ ||   || ¡}| ||¡ nEt|ƒ}d|	|  }d|
|  }|| }t|ƒ}|�r"t j|| ||| d� ||  ¡ |  |¡}n	| ¡ |  |¡}|j||| d� |�rHt  
| | ¡�rHt  || ¡||< qd S )NzGIf capturable=True, params and state_steps must be CUDA or XLA tensors.r   )Úvalue)Úout)r-   r}   r~   r9   rQ   Ú	enumerater{   r|   Úis_cudaÚis_xlar[   Úview_as_realÚmul_Úlerp_Úaddcmul_ÚnegÚsqrtÚcloneÚcopy_ÚmaximumÚadd_Úaddcdiv_r   r   Úview_as_complex) r"   rc   rd   re   rf   rg   rj   rk   r'   rl   rm   r#   r&   r%   r   r   r    rh   ÚiÚparamrZ   rW   rX   Ústep_trE   Úbias_correction1Úbias_correction2Ú	step_sizeÚstep_size_negÚbias_correction2_sqrtrY   Údenomr4   r4   r5   r�   j  s~   
ÿÿÿÿþ





þ
þ€«r�   c       
            sÀ  t | ƒdkrd S tˆtƒr|stdƒ‚tj ¡ s*|r*tdd„ t| |ƒD ƒƒs*J dƒ‚|r0J dƒ‚|d u r8|d u s:J ‚t	 
| |||||g¡}| ¡ D �]\\}}}}}}}|rh|rat|||||ƒ nt||||ƒ |rot |¡}|d jr‚tj|tjddd	�dd
� nt |d¡ |dkr–t |dˆ|  ¡ t ||dˆ  ¡ t |ˆ¡ t |||dˆ ¡ ~|�rt ˆ |¡}t ˆ|¡}t |d¡ t |d¡ t |¡ t |ˆ¡ t |¡ t |¡ |}|}|ròt ||¡ t |¡}nt |¡}t ||¡ t ||¡ t ||¡ t |||¡ qI‡ fdd„|D ƒ}‡fdd„|D ƒ}t‡fdd„|D ƒƒ}dd„ |D ƒ}|�rDt ||¡ t |¡}nt |¡}t ||¡ t ||¡ t ||||¡ qId S )Nr   r)   c                 s   s    � | ]\}}|j o|j V  qd S rs   )r‡   )r/   r1   rE   r4   r4   r5   r6   ý  s   € 
ÿz&_multi_tensor_adamw.<locals>.<genexpr>z@If capturable=True, params and state_steps must be CUDA tensors.z#_foreach ops don't support autogradr*   Úcpu)r+   )Úalphar   c                    ó   g | ]
}d ˆ t |ƒ  ‘qS ©r   ©r   ©r/   rE   )rl   r4   r5   Ú
<listcomp>S  ó    z'_multi_tensor_adamw.<locals>.<listcomp>c                    r    r¡   r¢   r£   )rm   r4   r5   r¤   T  r¥   c                    s   g | ]}ˆ | d  ‘qS )éÿÿÿÿr4   ©r/   Úbc)r#   r4   r5   r¤   V  s    c                 S   s   g | ]}t |ƒ‘qS r4   )r   r§   r4   r4   r5   r¤   X  s    )rO   r9   r   r=   r-   r{   r|   r?   Úzipr   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Ú_foreach_negÚis_cpuÚ_foreach_add_rR   Ú_foreach_mul_Ú_foreach_lerp_Ú_foreach_addcmul_Ú_foreach_powÚ_foreach_sub_Ú_foreach_neg_Ú_foreach_div_Ú_foreach_reciprocal_Ú_foreach_sqrt_Ú_foreach_maximum_Ú_foreach_sqrtÚ_foreach_addcdiv_r   )r"   rc   rd   re   rf   rg   rj   rk   r'   rl   rm   r#   r&   r%   r   r   r    rh   Úgrouped_tensorsÚdevice_paramsÚdevice_gradsÚdevice_exp_avgsÚdevice_exp_avg_sqsÚdevice_max_exp_avg_sqsÚdevice_state_stepsr‚   r˜   r™   rš   rœ   Úexp_avg_sq_sqrtr4   )rl   rm   r#   r5   r€   à  sŒ   
ÿþÿ	
ù






¢r€   Úreturnc       
          C   s~  | sd S |r
t dƒ‚|d ur|j|ind }|d ur|j|ind }t|tƒr1t|jƒdkr1|j|ind }t | |||||g¡}| ¡ D ]z\\}}\\}}}}}}}d\}}|d uri||vre|j|dd�||< || }|d ur~||vrz|j|dd�||< || }|d ur“||vr“|j|dd�||< || }t	 
|d¡ t	j|||||||||	|
|||||d� |d ur¼t	 ||gt|ƒ ¡ qBd S )	Nz9Adam with fused=True does not support differentiable=Truerž   )NNT)Únon_blocking)r+   rÄ   r   )	r'   r#   rl   rm   r&   r%   r   rj   rk   )r=   r+   r9   r   Ústrr   rª   ÚitemsÚtor-   r®   Z_fused_adamw_r³   rO   ) r"   rc   rd   re   rf   rg   rj   rk   r'   rl   rm   r#   r&   r%   r   r   r    rh   Úgrad_scale_dictÚfound_inf_dictÚlr_dictr»   r+   r‚   r¼   r½   r¾   r¿   rÀ   rÁ   Údevice_grad_scaleÚdevice_found_infr4   r4   r5   r   h  sf   &ÿûñ€Ûr   )NFFNNNF)!r-   r   Ú	optimizerr   r   r   r   r   r	   r
   r   r   r   r   r   r   r   Útypingr   r   r   r   Útorch.utils._foreach_utilsr   Ú__all__r   Ú__doc__rw   rQ   r   r�   r€   r   r4   r4   r4   r5   Ú<module>   sn   @ J&ñðïîíÚKñÿþýüûú	÷
öõôóòñïîí
ìëê
éRÿþýüûúùø
öõô
óòñðïî
ívÿþýüûúùø
öõô
óòñðïî
í 	ÿþýüûúùø
öõô
óòñðïîíì