o
    !ôj‰7  ã                   @   s‚  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	m
Z
mZ d dlmZmZ ddgZG dd„ deƒZd	d
e
› de› de	› d� e_				d"dee dee dee dee dedee dededededededefdd„Zdd„ Zdee dee dee dee dedededededededefdd„Zdee dee dee dee dedededededededefd d!„ZdS )#é    N)ÚTensoré   )	Ú	OptimizerÚ_use_grad_for_differentiableÚ
_get_valueÚ_view_as_realÚ_default_to_fused_or_foreachÚ_get_scalar_dtypeÚ_differentiable_docÚ_foreach_docÚ_maximize_doc)ÚListÚOptionalÚAdagradÚadagradc                       sn   e Zd Z						ddddœdee ded	ef‡ fd
d„Z‡ fdd„Zdd„ Zdd„ Ze	ddd„ƒZ
‡  ZS )r   ç{®Gáz„?r   ç»½×Ùß|Û=NF)ÚmaximizeÚdifferentiableÚforeachr   r   c             
      sþ   d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|ks!t d|› �ƒ‚d|ks,t d|› �ƒ‚d|ks7t d|› �ƒ‚t||||||||	d�}
tƒ  ||
¡ | jD ]/}|d D ](}| j| }tjdtƒ d	�|d
< t 	|¡rnt
||ƒn|}tj||tjd�|d< qSqMd S )Ng        zInvalid learning rate: zInvalid lr_decay value: zInvalid weight_decay value: z)Invalid initial_accumulator_value value: zInvalid epsilon value: )ÚlrÚlr_decayÚepsÚweight_decayÚinitial_accumulator_valuer   r   r   Úparams©ÚdtypeÚstep)Úmemory_formatÚsum)Ú
ValueErrorÚdictÚsuperÚ__init__Úparam_groupsÚstateÚtorchÚtensorr	   Ú
is_complexÚcomplexÚ	full_likeÚpreserve_format)Úselfr   r   r   r   r   r   r   r   r   ÚdefaultsÚgroupÚpr&   Z
init_value©Ú	__class__© úf/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/adagrad.pyr$      sH   ÿø


ÿýÿøÿzAdagrad.__init__c                    sš   t ƒ  |¡ | jD ]}| dd ¡ | dd¡ | dd¡ q	t| j ¡ ƒ}t|ƒdko3t 	|d d ¡}|sI|D ]}tj
t|d ƒtƒ d�|d< q8d S d S )Nr   r   Fr   r   r   r   )r#   Ú__setstate__r%   Ú
setdefaultÚlistr&   ÚvaluesÚlenr'   Ú	is_tensorr(   Úfloatr	   )r-   r&   r/   Zstate_valuesZstep_is_tensorÚsr1   r3   r4   r5   @   s   

ÿþzAdagrad.__setstate__c                 C   s4   | j D ]}|d D ]}| j| }|d  ¡  q	qd S )Nr   r    )r%   r&   Úshare_memory_)r-   r/   r0   r&   r3   r3   r4   Úshare_memoryO   s   

þÿzAdagrad.share_memoryc           
      C   s~   d\}}|d D ]2}|j d ur:||j jO }|t |¡O }| |¡ | |j ¡ | j| }	| |	d ¡ | |	d ¡ q||fS )N)FFr   r    r   )ÚgradÚ	is_sparser'   r)   Úappendr&   )
r-   r/   Úparams_with_gradÚgradsÚ
state_sumsÚstate_stepsÚhas_sparse_gradÚhas_complexr0   r&   r3   r3   r4   Ú_init_groupU   s   


€zAdagrad._init_groupc           
      C   s®   d}|durt  ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]4}g }g }g }g }|  |||||¡\}}	t|||||d |d |d |d ||d |d |d |	d	� q |S )
z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   )	r   r   r   r   rF   r   r   r   rG   )r'   Úenable_gradr%   rH   r   )
r-   ÚclosureÚlossr/   rB   rC   rD   rE   rF   rG   r3   r3   r4   r   c   s6   
ÿ
ózAdagrad.step)r   r   r   r   r   N©N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr$   r5   r>   rH   r   r   Ú__classcell__r3   r3   r1   r4   r      s,    ø
õø
öõ3a[  Implements Adagrad algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)
                \text{ (objective)}, \: \lambda \text{ (weight decay)},                          \\
            &\hspace{12mm}    \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\
            &\textbf{initialize} :  state\_sum_0 \leftarrow 0                             \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \tilde{\gamma}    \leftarrow \gamma / (1 +(t-1) \eta)                  \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm}state\_sum_t  \leftarrow  state\_sum_{t-1} + g^2_t                      \\
            &\hspace{5mm}\theta_t \leftarrow
                \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon}            \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adaptive Subgradient Methods for Online Learning
    and Stochastic Optimization`_.
    aÉ  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lr_decay (float, optional): learning rate decay (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-10)
        z	
        zŽ

    .. _Adaptive Subgradient Methods for Online Learning and Stochastic
        Optimization: http://jmlr.org/papers/v12/duchi11a.html

    Fr   rC   rD   rE   rF   r   r   rG   r   r   r   r   r   c                C   s†   t dd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r%tj ¡ r%tdƒ‚|r/tj ¡ s/t}nt}|| |||||	|
|||||d� dS )	ztFunctional API that performs Adagrad algorithm computation.

    See :class:`~torch.optim.Adagrad` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rL   )Ú
isinstancer'   r   )Ú.0Útr3   r3   r4   Ú	<genexpr>Î   s   € zadagrad.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers©r   r   r   r   rF   r   r   rG   )ÚallÚRuntimeErrorr   r'   ÚjitÚis_scriptingÚ_multi_tensor_adagradÚ_single_tensor_adagrad)r   rC   rD   rE   rF   r   r   rG   r   r   r   r   r   Ú_Úfuncr3   r3   r4   r   ¸   s2   ÿ
ôc                 C   s8   |   ¡ }| ¡ dks| ¡ dkrt | ¡S t |||¡S )Nr   )ÚsizeÚnumelr'   Ú
empty_likeÚsparse_coo_tensor)r?   Úgrad_indicesr8   r`   r3   r3   r4   Ú_make_sparseî   s   
re   c             	   C   sl  t | |||ƒD ]¬\}}}}|d7 }t|ƒ}|	s|n| }|dkr.|jr'tdƒ‚|j||d�}|d|d |   }|jrp| ¡ }| ¡ }| ¡ }| t	||| 
d¡ƒ¡ | |¡}| ¡  ¡  |¡}|jt	|||| ƒ| d� qt |¡}|r†t |¡}t |¡}t |¡}|j||dd� |
r—| ¡ | }n| ¡  |¡}|j||| d� |r³t |¡}t |¡}qd S )Nr   r   z;weight_decay option is not compatible with sparse gradients©Úalphaé   ©Úvalue)Úzipr   r@   rY   ÚaddÚcoalesceÚ_indicesÚ_valuesÚadd_re   ÚpowÚsparse_maskÚsqrt_r'   r)   Úview_as_realÚaddcmul_ÚsqrtÚaddcdiv_Úview_as_complex)r   rC   rD   rE   r   r   r   r   rF   r   r   rG   Úparamr?   Z	state_sumZstep_tr   Zclrrd   Zgrad_valuesÚstdZ
std_valuesr)   r3   r3   r4   r]   õ   sH   ÿ
ÿ





€Ør]   c                   sz  |
rJ dƒ‚t | ƒdkrd S t | |||g¡}| ¡ D ]Ÿ\\}}}}}|o-tdd„ |D ƒƒ}|rAt||||ˆ |ˆ|dd|
|d� q|rIt|||ƒ |	rPt |¡}|d j	rctj
|tjdd	d
�dd� nt 
|d¡ |dkr€|	rxtj
|||d� ntj|||d�}‡ ‡fdd„|D ƒ}tj|||dd� t |¡}t 
||¡ |dks¤|	r­t ||¡ |}nt ||¡}t |||¡ qd S )Nz#_foreach ops don't support autogradr   c                 s   s   � | ]}|j V  qd S rL   )r@   )rS   r?   r3   r3   r4   rU   H  s   € z(_multi_tensor_adagrad.<locals>.<genexpr>TFrW   g      ð?Úcpu)Údevicerf   r   c                    s&   g | ]}ˆ  d t |ƒd  ˆ   ‘qS )r   )r   )rS   r   ©r   r   r3   r4   Ú
<listcomp>r  s   & z)_multi_tensor_adagrad.<locals>.<listcomp>ri   )r9   r   Ú"_group_tensors_by_device_and_dtyper8   Úanyr]   r   r'   Ú_foreach_negÚis_cpuÚ_foreach_add_r(   Ú_foreach_addÚ_foreach_addcmul_Z_foreach_sqrtÚ_foreach_mul_Z_foreach_mulZ_foreach_addcdiv_)r   rC   rD   rE   r   r   r   r   rF   r   r   rG   Zgrouped_tensorlistsÚdevice_paramsÚdevice_gradsZdevice_state_sumsZdevice_state_stepsr^   Zdevice_has_sparse_gradZ	minus_clrrz   Ú	numeratorr3   r}   r4   r\   0  sV   ô


Çr\   )NNFF)r'   r   Ú	optimizerr   r   r   r   r   r	   r
   r   r   Útypingr   r   Ú__all__r   Ú__doc__rP   r;   r   re   r]   r\   r3   r3   r3   r4   Ú<module>   sÄ    , 	÷
öõç3öÿþýüùø	÷
öôóòñ
ð6ÿþýüúùø	÷
öõô
ó;ÿþýüúùø	÷
öõôó