o
    ôj£<  ã                "   @   sÜ  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	e› d
e› d
e
› d
e› d�	e_dee dee dee dee dee dee dededededededededefdd„Zdee dee dee dee dee dee dededededededededefdd„Zeed�		 	 	 	 d#dee dee dee dee dee dee d!ee dededededededededef d"d„ƒZdS )$é    )ÚListÚOptionalÚTupleÚUnionN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚASGDÚasgdc                       s€   e Zd Z									dded	ed
ededededee dededef‡ fdd„Z‡ fdd„Zdd„ Z	e
ddd„ƒZ‡  ZS )r   ç{®Gáz„?ç-Cëâ6?ç      è?ç    €„.Ar   NFÚparamsÚlrÚlambdÚalphaÚt0Úweight_decayÚforeachÚmaximizeÚdifferentiableÚ
capturablec                    sX   d|kst d|› �ƒ‚d|kst d|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ d S )Ng        zInvalid learning rate: zInvalid weight_decay value: )	r   r   r   r   r    r!   r"   r#   r$   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r   r    r!   r"   r#   r$   Údefaults©Ú	__class__© úa/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/asgd.pyr(      s    ÷zASGD.__init__c                    sú   t ƒ  |¡ | jD ]q}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]R}| j |g ¡}t|ƒdkryt |d ¡sOt	|d ƒ}tj
|tƒ |jd	�|d< t |d
 ¡sdtj
|d
 tƒ |jd	�|d
< t |d ¡sytj
|d tƒ |jd	�|d< q'q	d S )Nr!   r"   Fr#   r$   r   r   Ústep)ÚdtypeÚdeviceÚetaÚmu)r'   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r1   )r)   r7   ÚgroupÚpÚp_stateÚstep_valr+   r-   r.   r4   :   s2   


ÿ
ÿ
ÿ€óûzASGD.__setstate__c                 C   s
  d}|d D ]|}	|	j d ur‚|t |	¡O }| |	¡ |	j jr!tdƒ‚| |	j ¡ | j|	 }
t|
ƒdkrftjd|	j	t
ƒ d�|
d< tj|d |	j	t
ƒ d� ¡  ¡ |
d	< tjd|	j	t
ƒ d�|
d
< tj|	tjd�|
d< | |
d
 ¡ | |
d ¡ | |
d	 ¡ | |
d ¡ q|S )NFr   z&ASGD does not support sparse gradientsr   r-   )r1   r0   r/   r   r2   r3   )Úmemory_formatÚax)Úgradr:   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr7   r9   Úzerosr1   r   Ú	as_tensorÚcloneÚdetachÚonesÚ
zeros_likeÚpreserve_format)r)   r>   Úparams_with_gradÚgradsÚmusÚaxsÚetasÚstate_stepsÚhas_complexr?   r7   r-   r-   r.   Ú_init_groupR   s>   




ÿÿû

ÿ
ÿ€zASGD._init_groupc                 C   sÌ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]?}g }g }g }g }g }g }	|  |||||||	¡}
t||||||	|d |d |d |d |d |d |d |d	 |d
 |
d� q$|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r    r!   r"   r#   r$   )
r   r   r   r   r    r!   r"   r#   r$   rV   )Ú _cuda_graph_capture_health_checkr:   Úenable_gradr5   rW   r   )r)   ÚclosureÚlossr>   rP   rQ   rR   rS   rT   rU   rV   r-   r-   r.   r/   v   sF   
ÿ
ÿðz	ASGD.step)	r   r   r   r   r   NFFF©N)Ú__name__Ú
__module__Ú__qualname__r   r<   r   Úboolr(   r4   rW   r   r/   Ú__classcell__r-   r-   r+   r.   r      sF    õþýüûúùø	÷
öõ$ah  Implements Averaged Stochastic Gradient Descent.

    It has been proposed in `Acceleration of stochastic approximation by
    averaging`_.

    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lambd (float, optional): decay term (default: 1e-4)
        alpha (float, optional): power for eta update (default: 0.75)
        t0 (float, optional): point at which to start averaging (default: 1e6)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zx

    .. _Acceleration of stochastic approximation by averaging:
        https://dl.acm.org/citation.cfm?id=131098

    r   rQ   rS   rR   rT   rU   r   r   r   r   r    r"   r#   r$   rV   c       	      
   C   s  t | ƒD �] \}}|| }|s|n| }|| }|| }|| }|| }tj ¡ sU|rUtƒ }|jj|jj  krE|jj  krE|jjkrMn n|jj|v sUJ d|› d�ƒ‚t |¡rit |¡}t |¡}t |¡}|d7 }|
dkrx|j	||
d�}|rŒ| 
d||  ¡ |j||dd� nt|ƒ}| 
d||  ¡ |j|| d� |s©| ¡ dkrµ| | |¡ 
|¡¡ n| |¡ |rÜ| |d|| |  |	  ¡ | dt || t |¡¡ ¡ qt|ƒ}t |d|| |  |	  ¡}| |¡ t dtd|| ƒ ¡}| |¡ qd S )NúUIf capturable=True, params, mus, etas, and state_steps must be on supported devices: Ú.r   r   ©r   éÿÿÿÿ©Úvalue)Ú	enumerater:   Ú_utilsÚis_compilingr   r1   ÚtyperE   Úview_as_realÚaddÚmul_Úaddcmul_r   Úadd_ÚitemÚsubÚcopy_ÚmaximumÚ	ones_likerJ   Úmax)r   rQ   rS   rR   rT   rU   r   r   r   r   r    r"   r#   r$   rV   ÚiÚparamrD   r3   rC   r2   Ústep_tÚcapturable_supported_devicesZ	eta_valuer/   Znew_etaZnew_mur-   r-   r.   Ú_single_tensor_asgd¿   s\   ÿþ
ýÿþ




"
Ér{   c       	            sD  t | ƒdkrd S |rJ dƒ‚tj ¡ s2|r2tdd�‰t‡fdd„t| |||ƒD ƒƒs2J dˆ› d�ƒ‚t | |||||g¡}| 	¡ D ]Þ\\‰}\\}}}}}}}|rWt
|||ƒ |r^t |¡}|d jrqtj|tjd	d
d�d	d� nt |d¡ |
dkr™|rˆtj|||
d� |}ntj|||
d�}tj||ˆd� ntj||ˆd�}tj|||dd� ~t ||¡}t |||¡ ~|rýt |ˆ¡}t |d	¡ t |¡ t ||¡ ~t |ˆ¡}t |ˆ¡ t |d¡ t |ˆ ¡ t |¡ t |ˆ¡ t ||¡ qA‡ ‡‡‡fdd„|D ƒ}‡‡fdd„|D ƒ}t ||¡ t ||¡ qAd S )Nr   z#_foreach ops don't support autogradF)Úsupports_xlac                 3   sV   � | ]&\}}}}|j j|j j  ko|j j  ko|j jkn  o&|j jˆ v V  qd S r\   )r1   rk   )Ú.0r?   r3   r2   r/   )rz   r-   r.   Ú	<genexpr>'  s   € 
2þ

þz%_multi_tensor_asgd.<locals>.<genexpr>rb   rc   g      ð?Úcpu©r1   rd   r   re   rf   c                    s.   g | ]}t jˆd ˆˆ |  ˆ   ˆd�‘qS ©r   r€   )r:   rJ   ©r}   r/   )r   r1   r   r   r-   r.   Ú
<listcomp>ƒ  s     ÿÿz&_multi_tensor_asgd.<locals>.<listcomp>c                    s,   g | ]}t jd td t|ƒˆ ƒ ˆ d�‘qS r�   )r:   rJ   rv   r   r‚   )r1   r   r-   r.   rƒ   ‡  s    ÿÿ)r9   r:   ri   rj   r   ÚallÚzipr   Ú"_group_tensors_by_device_and_dtypeÚitemsr   Ú_foreach_negÚis_cpuÚ_foreach_add_r=   Ú_foreach_addÚ_foreach_addcmul_Ú_foreach_subÚ_foreach_maximum_Ú_foreach_reciprocal_Z_foreach_copy_Ú_foreach_mulÚ_foreach_mul_Z_foreach_pow_)r   rQ   rS   rR   rT   rU   r   r   r   r   r    r"   r#   r$   rV   Úgrouped_tensorsÚ_Úgrouped_paramsÚgrouped_gradsZgrouped_axsZgrouped_musZgrouped_etasÚgrouped_state_stepsZintermediateZnew_musZnew_etasr-   )r   rz   r1   r   r   r   r.   Ú_multi_tensor_asgd  s�   ÿ
ý
üÿö

ÿÿÿ


þþ¤r—   )Úsingle_tensor_fnFr!   c                C   sr   |du rt | |dd�\}}|rtj ¡ rtdƒ‚|r"tj ¡ s"t}nt}|| |||||||||||||	|
d� dS )znFunctional API that performs asgd algorithm computation.

    See :class:`~torch.optim.ASGD` for details.
    NF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)	r   r   r   r   r    r"   r#   r$   rV   )r	   r:   ÚjitÚis_scriptingrH   r—   r{   )r   rQ   rS   rR   rT   rU   r!   r"   r#   r$   rV   r   r   r   r   r    r“   Úfuncr-   r-   r.   r   �  s4   
ÿ
ñ)NFFFF)Útypingr   r   r   r   r:   r   Ú	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r<   r`   r{   r—   r   r-   r-   r-   r.   Ú<module>   sì   < óòñ
ðÿþýüûúø	÷
öõôóòñ
ðLÿþýüûúø	÷
öõôóòñ
ð 
óÿþýüûú	÷
öõôóñðïîí