o
    !ôjÞ6  ã                !   @   sÈ  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	m
Z
mZmZ d dlmZmZ ddgZd$dd	„ZG d
d„ deƒZde
› de› de	› de› d�	e_					d%dee dee dee dee dee dee dee dededededededededef dd„Zdee dee dee dee dee dee dededededededededefd d!„Zdee dee dee dee dee dee dededededededededefd"d#„ZdS )&é    N)ÚTensoré   )
Ú	OptimizerÚ_use_grad_for_differentiableÚ
_get_valueÚ_default_to_fused_or_foreachÚ_get_scalar_dtypeÚ_view_as_realÚ_differentiable_docÚ_foreach_docÚ_maximize_docÚ_capturable_doc)ÚListÚOptionalÚASGDÚasgdc                 C   s   t | tjƒstj| |d�S | S )N©Údevice)Ú
isinstanceÚtorchr   Útensor)Úxr   © r   úc/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/asgd.pyÚ
_to_tensor   s   r   c                	       sh   e Zd Z									ddee d	ed
edef‡ fdd„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   ç{®Gáz„?ç-Cëâ6?ç      è?ç    €„.Ar   NFÚforeachÚmaximizeÚdifferentiableÚ
capturablec                    sX   d|kst d|› �ƒ‚d|kst d|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ d S )Ng        zInvalid learning rate: zInvalid weight_decay value: )	ÚlrÚlambdÚalphaÚt0Úweight_decayr   r    r!   r"   )Ú
ValueErrorÚdictÚsuperÚ__init__)ÚselfÚparamsr#   r$   r%   r&   r'   r   r    r!   r"   Údefaults©Ú	__class__r   r   r+      s    ÷zASGD.__init__c                    sú   t ƒ  |¡ | jD ]q}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]R}| j |g ¡}t|ƒdkryt |d ¡sOt	|d ƒ}tj
|tƒ |jd	�|d< t |d
 ¡sdtj
|d
 tƒ |jd	�|d
< t |d ¡sytj
|d tƒ |jd	�|d< q'q	d S )Nr   r    Fr!   r"   r-   r   Ústep)Údtyper   ÚetaÚmu)r*   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenr   Ú	is_tensorÚfloatr   r   r   )r,   r8   ÚgroupÚpÚp_stateÚstep_valr/   r   r   r5   1   s&   
€÷ûzASGD.__setstate__c                 C   s  d}|d D ]x}	|	j d ur~|t |	¡O }| |	¡ |	j jr!tdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrbtjd|	j	t
ƒ d�|
d< tj|d |	j	t
ƒ d�|
d	< tjd|	j	t
ƒ d�|
d
< tj|	tjd�|
d< | |
d
 ¡ | |
d ¡ | |
d	 ¡ | |
d ¡ q|S )NFr-   z&ASGD does not support sparse gradientsr   r   )r   r2   r1   r#   r3   r4   )Úmemory_formatÚax)Úgradr   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr8   r:   Úzerosr   r   r   ÚonesÚ
zeros_likeÚpreserve_format)r,   r=   Úparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚhas_complexr>   r8   r   r   r   Ú_init_groupD   s,   



ÿ€zASGD._init_groupc                 C   sÌ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]?}g }g }g }g }g }g }	|  |||||||	¡}
t||||||	|d |d |d |d |d |d |d |d	 |d
 |
d� q$|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr$   r#   r&   r%   r'   r   r    r!   r"   )
r$   r#   r&   r%   r'   r   r    r!   r"   rR   )Ú _cuda_graph_capture_health_checkr   Úenable_gradr6   rS   r   )r,   ÚclosureÚlossr=   rL   rM   rN   rO   rP   rQ   rR   r   r   r   r1   ^   sB   
ÿ
ðz	ASGD.step)	r   r   r   r   r   NFFF©N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr+   r5   rS   r   r1   Ú__classcell__r   r   r/   r   r      s.    õø	÷
öõah  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zx

    .. _Acceleration of stochastic approximation by averaging:
        https://dl.acm.org/citation.cfm?id=131098

    Fr-   rM   rO   rN   rP   rQ   r   r    r!   r"   rR   r$   r#   r&   r%   r'   c                C   sr   |du rt | |dd�\}}|rtj ¡ rtdƒ‚|r"tj ¡ s"t}nt}|| |||||||||||||	|
d� dS )znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r$   r#   r&   r%   r'   r    r!   r"   rR   )r   r   ÚjitÚis_scriptingrG   Ú_multi_tensor_asgdÚ_single_tensor_asgd)r-   rM   rO   rN   rP   rQ   r   r    r!   r"   rR   r$   r#   r&   r%   r'   Ú_Úfuncr   r   r   r   ¥   s0   
ñc       	      
   C   sî  t | ƒD ]ð\}}|| }|s|n| }|| }|| }|| }|| }tj ¡ sF|rF|jr6|jr6|jr6|jsF|jrB|jrB|jrB|jsFJ dƒ‚t |¡rZt |¡}t |¡}t |¡}|d7 }|
dkri|j||
d�}|r}| 	d||  ¡ |j
||dd� nt|ƒ}| 	d||  ¡ |j|| d� |sš| ¡ dkr¦| | |¡ 	|¡¡ n| |¡ |rÍ| |d|| |  |	  ¡ | dt || t |¡¡ ¡ qt|ƒ}t|d|| |  |	  ƒ}| |¡ tdtd|| ƒ ƒ}| |¡ qd S )NzSIf capturable=True, params, mus, etas, and state_steps must be CUDA or XLA tensors.r   r   ©r%   éÿÿÿÿ©Úvalue)Ú	enumerater   Ú_utilsÚis_compilingÚis_cudaÚis_xlarD   Úview_as_realÚaddÚmul_Úaddcmul_r   Úadd_ÚitemÚsubÚcopy_ÚmaximumÚ	ones_liker   Úmax)r-   rM   rO   rN   rP   rQ   r$   r#   r&   r%   r'   r    r!   r"   rR   ÚiÚparamrC   r4   rB   r3   Ústep_tZ	eta_valuer1   Únew_etaÚnew_mur   r   r   rb   Ü   sX   ÿÿÿÿþ




"
Ñrb   c       	         C   sz  t | ƒdkrd S |rJ dƒ‚tj ¡ s'|r'tdd„ t| |||ƒD ƒƒs'J dƒ‚t | |||||g¡}| ¡ D �]\\}}\\}}}}}}}|rMt	|||ƒ |rTt 
|¡}|d jrgtj|tjddd�dd	� nt |d
¡ |
dkr�|r~tj|||
d	� |}ntj|||
d	�}tj|||d	� ntj|||d	�}tj|||dd� ~t ||¡}t |||¡ ~|rót ||¡}t |d¡ t |¡ t ||¡ ~t ||	¡}t ||¡ t ||¡ t |d
¡ t |¡ t ||¡ t ||¡ q6|d  ¡ }g }g }tt |ƒƒD ]*}t|d
|| ||	    |d�}| |¡ td
td
|| ƒ |d�}| |¡ �qt ||¡ t ||¡ q6d S )Nr   z#_foreach ops don't support autogradc                 s   s0   � | ]\}}}}|j o|j o|j o|j V  qd S rX   )rl   )Ú.0r>   r4   r3   r1   r   r   r   Ú	<genexpr>9  s   € 
 ÿz%_multi_tensor_asgd.<locals>.<genexpr>zLIf capturable=True, params, mus, etas, and state_steps must be CUDA tensors.g      ð?Úcpur   re   r   rf   rg   )r:   r   rj   rk   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚitemsr	   Ú_foreach_negÚis_cpuÚ_foreach_add_r   Ú_foreach_addÚ_foreach_addcmul_Z_foreach_subÚ_foreach_maximum_Ú_foreach_reciprocal_Z_foreach_copy_Ú_foreach_powÚ_foreach_mul_rs   Úranger   rE   rx   )r-   rM   rO   rN   rP   rQ   r$   r#   r&   r%   r'   r    r!   r"   rR   Úgrouped_tensorsr   rc   Úgrouped_paramsÚgrouped_gradsZgrouped_axsZgrouped_musZgrouped_etasÚgrouped_state_stepsZintermediateZnew_musZnew_etasr1   ry   r|   r}   r   r   r   ra      st   
ÿþÿ




ÿ
²ra   rX   )NFFFF)r   r   Ú	optimizerr   r   r   r   r   r	   r
   r   r   r   Útypingr   r   Ú__all__r   r   Ú__doc__r\   r<   r   rb   ra   r   r   r   r   Ú<module>   sè    0
|óòñ
ð!óÿþýüûú	÷
öõôóñðïî
í7ÿþýüûúø	÷
öõôóòñ
ðDÿþýüûúø	÷
öõôóòñð