o
    ôjPO  ã                !   @   s.  d dl mZmZ d dlZd dlmZ d dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZ ddgZG d	d„ deƒZd
de
› de› de	› d� e_							d)dee dee dee dee dee dee dee dedee dededededededef dd„Zd d!„ Zdee dee dee dee dee dee dedededededededefd"d#„Zdee dee dee dee dee dee dedededededededefd$d%„Zdee dee dee dee dee dee dedededededededed&dfd'd(„ZdS )*é    )ÚListÚOptionalN)ÚTensor)Ú$_get_fused_kernels_supported_devicesé   )
Ú_default_to_fused_or_foreachÚ_differentiable_docÚ_foreach_docÚ_get_scalar_dtypeÚ
_get_valueÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚAdagradÚadagradc                       s�   e Zd Z						dddddœdeded	ed
edededee dededee f‡ fdd„Z‡ fdd„Zdd„ Z	dd„ Z
eddd„ƒZ‡  ZS )r   ç{®Gáz„?r   ç»½×Ùß|Û=NF)ÚmaximizeÚdifferentiableÚfusedÚparamsÚlrÚlr_decayÚweight_decayÚinitial_accumulator_valueÚepsÚforeachr   r   r   c                   s~  d|kst d|› �ƒ‚d|kst d|› �ƒ‚d|ks!t d|› �ƒ‚d|ks,t d|› �ƒ‚d|ks7t d|› �ƒ‚t||||||||	|
d�	}tƒ  ||¡ |
rx|	rStdƒ‚d	| _tƒ ‰ ˆ  d
¡ t‡ fdd„| j	D ƒƒsrtdˆ › d�ƒ‚|rxtdƒ‚| j	D ]A}|d D ]:}| j
| }|d rštjdt|d d�|jd�ntjdtƒ d�|d< t |¡r®t||ƒn|}tj||tjd�|d< q�q{d S )Ng        zInvalid learning rate: zInvalid lr_decay value: zInvalid weight_decay value: z)Invalid initial_accumulator_value value: zInvalid epsilon value: )	r   r   r   r   r   r   r   r   r   z)`fused` does not support `differentiable`TÚcudac                 3   s4   � | ]}|d  D ]}|j jˆ v ot |¡V  qqdS )r   N)ÚdeviceÚtypeÚtorchÚis_floating_point)Ú.0ÚpgÚp©Úfused_supported_devices© úd/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/adagrad.pyÚ	<genexpr>G   s   € ýþÿz#Adagrad.__init__.<locals>.<genexpr>zX`fused=True` requires all the params to be floating point Tensors of supported devices: Ú.z0`fused` and `foreach` cannot be `True` together.r   r   r)   ©Úis_fused)Údtyper    ©r/   Ústep)Úmemory_formatÚsum)Ú
ValueErrorÚdictÚsuperÚ__init__ÚRuntimeErrorÚ_step_supports_amp_scalingr   ÚremoveÚallÚparam_groupsÚstater"   Úzerosr
   r    ÚtensorÚ
is_complexÚcomplexÚ	full_likeÚpreserve_format)Úselfr   r   r   r   r   r   r   r   r   r   ÚdefaultsÚgroupr&   r=   Z
init_value©Ú	__class__r'   r*   r7      sx   ÿ÷
þÿÿ

ûýùÿýÿðÿzAdagrad.__init__c                    s®   t ƒ  |¡ d }| jD ]}| dd ¡ | dd¡ | dd¡ | dd ¡}qt| j ¡ ƒ}t|ƒdko;t 	|d d ¡}|sS|D ]}tj
t|d ƒt|d�d	�|d< q@d S d S )
Nr   r   Fr   r   r   r1   r-   r0   )r6   Ú__setstate__r<   Ú
setdefaultÚlistr=   ÚvaluesÚlenr"   Ú	is_tensorr?   Úfloatr
   )rD   r=   r   rF   Zstate_valuesZstep_is_tensorÚsrG   r)   r*   rI   h   s$   

ÿÿþzAdagrad.__setstate__c                 C   s4   | j D ]}|d D ]}| j| }|d  ¡  q	qd S )Nr   r3   )r<   r=   Úshare_memory_)rD   rF   r&   r=   r)   r)   r*   Úshare_memory}   s   

þÿzAdagrad.share_memoryc           
      C   s~   d\}}|d D ]2}|j d ur:||j jO }|t |¡O }| |¡ | |j ¡ | j| }	| |	d ¡ | |	d ¡ q||fS )N)FFr   r3   r1   )ÚgradÚ	is_sparser"   r@   Úappendr=   )
rD   rF   Úparams_with_gradÚgradsÚ
state_sumsÚstate_stepsÚhas_sparse_gradÚhas_complexr&   r=   r)   r)   r*   Ú_init_groupƒ   s   


€zAdagrad._init_groupc           
      C   sÈ   d}|durt  ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]A}g }g }g }g }|  |||||¡\}}	t|||||d |d |d |d ||d |d |d |	|d	 t| d
dƒt| ddƒd� q |S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   r   Ú
grad_scaleÚ	found_inf)r   r   r   r   rZ   r   r   r   r[   r   r]   r^   )r"   Úenable_gradr<   r\   r   Úgetattr)
rD   ÚclosureÚlossrF   rV   rW   rX   rY   rZ   r[   r)   r)   r*   r1   ‘   s@   
ÿ

ÿ

ðzAdagrad.step)r   r   r   r   r   N©N)Ú__name__Ú
__module__Ú__qualname__r   rO   r   Úboolr7   rI   rR   r\   r   r1   Ú__classcell__r)   r)   rG   r*   r      sJ    ø
ôþýüûúùø
öõôPa[  Implements Adagrad algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta)
                \text{ (objective)}, \: \lambda \text{ (weight decay)},                          \\
            &\hspace{12mm}    \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\
            &\textbf{initialize} :  state\_sum_0 \leftarrow \tau                          \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \tilde{\gamma}    \leftarrow \gamma / (1 +(t-1) \eta)                  \\
            &\hspace{5mm} \textbf{if} \: \lambda \neq 0                                          \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1}                             \\
            &\hspace{5mm}state\_sum_t  \leftarrow  state\_sum_{t-1} + g^2_t                      \\
            &\hspace{5mm}\theta_t \leftarrow
                \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon}            \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adaptive Subgradient Methods for Online Learning
    and Stochastic Optimization`_.
    aH  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        lr_decay (float, optional): learning rate decay (default: 0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        initial_accumulator_value (float, optional): initial value of the
            sum of squares of gradients (default: 0)
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-10)
        z	
        aÒ  
        fused (bool, optional): whether the fused implementation (CPU only) is used.
            Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16`
            are supported. (default: None). Please note that the fused implementations does not
            support sparse or complex gradients.
    .. _Adaptive Subgradient Methods for Online Learning and Stochastic
        Optimization: http://jmlr.org/papers/v12/duchi11a.html

    Fr   rW   rX   rY   r   r]   r^   rZ   r   r   r[   r   r   r   r   r   c                C   sÔ   t dd„ |D ƒƒstdƒ‚|du r|du rt| |	dd�\}}|du r$d}|du r*d}|r5tj ¡ r5tdƒ‚|r@tj ¡ r@tdƒ‚|rJtj ¡ sJt}n|rTtj ¡ sTt}nt}|| ||||||||||	|
||d	� dS )
ztFunctional API that performs Adagrad algorithm computation.

    See :class:`~torch.optim.Adagrad` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rc   )Ú
isinstancer"   r   )r$   Útr)   r)   r*   r+     s   € zadagrad.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers©
r   r   r   r   rZ   r   r   r[   r]   r^   )	r;   r8   r   r"   ÚjitÚis_scriptingÚ_fused_adagradÚ_multi_tensor_adagradÚ_single_tensor_adagrad)r   rW   rX   rY   r   r]   r^   rZ   r   r   r[   r   r   r   r   r   Ú_Úfuncr)   r)   r*   r   ó   sJ   ÿ
ÿ
òc                 C   s   |   ¡ }t |||¡S rc   )Úsizer"   Úsparse_coo_tensor)rS   Úgrad_indicesrL   rt   r)   r)   r*   Ú_make_sparse=  s   rw   c             	   C   s€  |d u r|d u s
J ‚t | |||ƒD ]¬\}}}}|d7 }t|ƒ}|s#|n| }|dkr8|jr1tdƒ‚|j||d�}|d|d |   }|jrz| ¡ }| ¡ }| ¡ }| t	||| 
d¡ƒ¡ | |¡}| ¡  ¡  |	¡}|jt	|||| ƒ| d� qt |¡}|r�t |¡}t |¡}t |¡}|j||dd� |r¡| ¡ |	 }n| ¡  |	¡}|j||| d� |r½t |¡}t |¡}qd S )Nr   r   z;weight_decay option is not compatible with sparse gradients©Úalphaé   ©Úvalue)Úzipr   rT   r8   ÚaddÚcoalesceÚ_indicesÚ_valuesÚadd_rw   ÚpowÚsparse_maskÚsqrt_r"   r@   Úview_as_realÚaddcmul_ÚsqrtÚaddcdiv_Úview_as_complex)r   rW   rX   rY   r]   r^   r   r   r   r   rZ   r   r   r[   ÚparamrS   Z	state_sumZstep_tr1   Zclrrv   Zgrad_valuesÚstdZ
std_valuesr@   r)   r)   r*   rq   B  sJ   ÿ
ÿ





€Ørq   c                   s’  |rJ dƒ‚|d u r|d u sJ ‚t | ƒdkrd S t | |||g¡}| ¡ D ]¡\\}}}}}|
o7tdd„ |D ƒƒ}|rMt||||ˆ |ˆ|	d|||||d� q%|rUt|||ƒ |r\t |¡}|d j	rotj
|tjddd	�dd
� nt 
|d¡ |dkrŒ|r„tj
|||d
� ntj|||d
�}‡ ‡fdd„|D ƒ}tj|||dd� t |¡}t 
||	¡ |dks°|r¹t ||¡ |}nt ||¡}t |||¡ q%d S )Nz#_foreach ops don't support autogradr   c                 s   s   � | ]}|j V  qd S rc   )rT   )r$   rS   r)   r)   r*   r+      s   € 
ÿz(_multi_tensor_adagrad.<locals>.<genexpr>Trl   g      ð?Úcpu)r    rx   r   c                    s&   g | ]}ˆ  d t |ƒd  ˆ   ‘qS )r   )r   )r$   r1   ©r   r   r)   r*   Ú
<listcomp>Ò  s    ÿz)_multi_tensor_adagrad.<locals>.<listcomp>r{   )rM   r   Ú"_group_tensors_by_device_and_dtyperL   Úanyrq   r   r"   Ú_foreach_negÚis_cpuÚ_foreach_add_r?   Ú_foreach_addÚ_foreach_addcmul_Z_foreach_sqrtÚ_foreach_mul_Z_foreach_mulÚ_foreach_addcdiv_)r   rW   rX   rY   r]   r^   r   r   r   r   rZ   r   r   r[   Zgrouped_tensorlistsÚdevice_paramsÚdevice_gradsÚdevice_state_sumsÚdevice_state_stepsrr   Zdevice_has_sparse_gradZ	minus_clrrŒ   Ú	numeratorr)   rŽ   r*   rp     s|   
ÿûÿò

ÿÿÿ
¸rp   Úreturnc                C   s>  | sd S |
s|rt dƒ‚|rt dƒ‚|d ur|j|ind }|d ur&|j|ind }t | |||g¡}| ¡ D ]g\\}}\\}}}}}d\}}|d ur^|d ur^||vrZ|j|dd�||< || }|d urw|d urw||vrs|j|dd�||< || }t |d¡ tj||||||||	|||d� |d urœt 	||gt
|ƒ ¡ q5d S )Nz5`fused` does not support sparse grad or complex paramz<adagrad with fused=True does not support differentiable=True)NNT)Únon_blockingr   )r   r   r   r   r   r]   r^   )r8   r    r   r�   ÚitemsÚtor"   r”   Z_fused_adagrad_Z_foreach_sub_rM   )r   rW   rX   rY   r]   r^   r   r   r   r   rZ   r   r   r[   Zgrad_scale_dictZfound_inf_dictÚgrouped_tensorsr    rr   r™   rš   r›   rœ   Zdevice_grad_scaleZdevice_found_infr)   r)   r*   ro   å  sf   ÿÿ
ÿøõÿ€ßro   )NNNFNFF)Útypingr   r   r"   r   Útorch.utils._foreach_utilsr   Ú	optimizerr   r   r	   r
   r   r   r   r   r   r   Ú__all__r   Ú__doc__rg   rO   r   rw   rq   rp   ro   r)   r)   r)   r*   Ú<module>   s(  0 *õôóæÿ9óÿþýüûúù
öõôóñðïî
íJÿþýüûúø	÷
öõôóò
ñ=ÿþýüûúø	÷
öõôóò
ñfÿþýüûúø	÷
öõôóòñð