o
    !ôj	,  ã                   @   sd  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	 d dl
mZmZ ddgZG dd„ deƒZd	d
e› de› de› d� e_			ddee dee dee dee dee dededededededefdd„Zdee dee dee dee dededededededefdd„Zdee dee dee dee dededededededefdd„ZdS ) é    N)ÚTensoré   )Ú	OptimizerÚ_use_grad_for_differentiableÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_foreach_docÚ_maximize_docÚ_view_as_real)ÚListÚOptionalÚAdadeltaÚadadeltac                       sd   e Zd Z					ddddœdee d	ed
ef‡ fdd„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   ç      ð?çÍÌÌÌÌÌì?ç�íµ ÷Æ°>r   NF)ÚmaximizeÚdifferentiableÚforeachr   r   c          
   	      sŽ   d|kst d|› �ƒ‚d|  krdksn t d|› �ƒ‚d|ks(t d|› �ƒ‚d|ks3t d|› �ƒ‚t|||||||d�}	tƒ  ||	¡ d S )Ng        zInvalid learning rate: r   zInvalid rho value: zInvalid epsilon value: zInvalid weight_decay value: )ÚlrÚrhoÚepsÚweight_decayr   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)
ÚselfÚparamsr   r   r   r   r   r   r   Údefaults©Ú	__class__© úg/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/adadelta.pyr      s$   ù	zAdadelta.__init__c                    s@   t ƒ  |¡ | jD ]}| dd ¡ | dd¡ | dd¡ q	d S )Nr   r   Fr   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr    r"   r#   r$   ,   s   
ýzAdadelta.__setstate__c           	      C   sÈ   d}|d D ][}|j d u rq|t |¡O }| |¡ |j jr"tdƒ‚| |j ¡ | j| }t|ƒdkrKd|d< tj|tj	d�|d< tj|tj	d�|d< | |d ¡ | |d ¡ |d  d	7  < q|S )
NFr   z*Adadelta does not support sparse gradientsr   Ústep)Úmemory_formatÚ
square_avgÚ	acc_deltar   )
ÚgradÚtorchÚ
is_complexÚappendÚ	is_sparseÚRuntimeErrorr'   ÚlenÚ
zeros_likeÚpreserve_format)	r   r(   Úparams_with_gradÚgradsÚsquare_avgsÚ
acc_deltasÚhas_complexÚpr'   r"   r"   r#   Ú_init_group3   s,   



ÿ
ÿzAdadelta._init_groupc                 C   sÈ   d}|durt  ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]A}g }g }g }g }|d |d |d |d |d |d |d f\}}	}
}}}}|  |||||¡}t||||||	|
|||||d	� q |S )
z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   r   r   )r   r   r   r   r   r   r   r:   )r.   Úenable_gradr%   r<   r   )r   ÚclosureÚlossr(   r6   r7   r8   r9   r   r   r   r   r   r   r   r:   r"   r"   r#   r)   P   sD   
ÿ
ù
ôzAdadelta.step)r   r   r   r   N)N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr   r$   r<   r   r)   Ú__classcell__r"   r"   r    r#   r      s(    ù	öù	÷
ö a  Implements Adadelta algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)},
                \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)},
                \: \lambda \text{ (weight decay)}                                                \\
            &\textbf{initialize} :  v_0  \leftarrow 0 \: \text{ (square avg)},
                \: u_0 \leftarrow 0 \: \text{ (accumulate variables)}                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm} v_t      \leftarrow v_{t-1} \rho + g^2_t (1 - \rho)                    \\
            &\hspace{5mm}\Delta x_t    \leftarrow   \frac{\sqrt{u_{t-1} +
                \epsilon }}{ \sqrt{v_t + \epsilon}  }g_t \hspace{21mm}                           \\
            &\hspace{5mm} u_t  \leftarrow   u_{t-1}  \rho +
                 \Delta x^2_t  (1 - \rho)                                                        \\
            &\hspace{5mm}\theta_t      \leftarrow   \theta_{t-1} - \gamma  \Delta x_t            \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `ADADELTA: An Adaptive Learning Rate Method`_.
    aá  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        rho (float, optional): coefficient used for computing a running average
            of squared gradients (default: 0.9). A higher value of `rho` will
            result in a slower average, which can be helpful for preventing
            oscillations in the learning process.
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-6).
        lr (float, optional): coefficient that scale delta before it is applied
            to the parameters (default: 1.0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zd

    .. _ADADELTA\: An Adaptive Learning Rate Method:
        https://arxiv.org/abs/1212.5701

    Fr   r7   r8   r9   r   r   r:   r   r   r   r   r   c                C   sj   |du rt | |dd�\}}|rtj ¡ rtdƒ‚|r"tj ¡ s"t}nt}|| ||||||	|
|||d� dS )zvFunctional API that performs Adadelta algorithm computation.

    See :class:`~torch.optim.Adadelta` for details.
    NF)Z	use_fusedz6torch.jit.script not supported with foreach optimizers)r   r   r   r   r   r   r:   )r   r.   ÚjitÚis_scriptingr2   Ú_multi_tensor_adadeltaÚ_single_tensor_adadelta)r   r7   r8   r9   r   r   r:   r   r   r   r   r   Ú_Úfuncr"   r"   r#   r   ²   s(   
õc                C   sü   t | |||ƒD ]t\}}}}|s|n| }|dkr|j||d�}t |¡r3t |¡}t |¡}t |¡}| |¡j||d| d� | |¡ ¡ }| |¡ ¡ }|	rT| ¡ }| 	|¡ |¡ | |¡j||d| d� t |¡rst 
|¡}|j|| d� qd S )Nr   ©Úalphar   ©Úvalue)ÚzipÚaddr.   r/   Úview_as_realÚmul_Úaddcmul_Úsqrt_ÚcloneÚdiv_Úview_as_complexÚadd_)r   r7   r8   r9   r   r   r   r   r   r   r:   Úparamr-   r+   r,   ÚstdÚdeltar"   r"   r#   rH   â   s*   ÿ





érH   c                C   s4  |	rJ dƒ‚t | ƒdkrd S t | |||g¡}| ¡ D ]|\\}}}}}|
r,t||||ƒ |r3t |¡}|dkrJ|rBtj|||d� ntj|||d�}t 	||¡ tj
|||d| d� t ||¡}t |¡ t ||¡}t |¡ t ||¡ t 	||¡ tj||| d� t 	||¡ tj
|||d| d� qd S )Nz#_foreach ops don't support autogradr   rK   r   rM   )r3   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr
   r.   Z_foreach_negZ_foreach_add_Z_foreach_addÚ_foreach_mul_Z_foreach_addcmul_Z_foreach_sqrt_Z_foreach_div_)r   r7   r8   r9   r   r   r   r   r   r   r:   Zgrouped_tensorsZdevice_paramsÚdevice_gradsZdevice_square_avgsZdevice_acc_deltasrI   rZ   Zdeltasr"   r"   r#   rG     s2   


ärG   )NFF)r.   r   Ú	optimizerr   r   r   r   r   r	   r
   Útypingr   r   Ú__all__r   Ú__doc__rC   Úfloatr   rH   rG   r"   r"   r"   r#   Ú<module>   s²    $uóòñå9÷ÿþýüùø	÷õôóò
ñ0ÿþýüúùø	÷
öõ
ô)ÿþýüúùø	÷
öõô