o
    ôj»>  ã                   @   s¶  d dl mZmZmZmZ d dlZd dlmZ ddlmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ ddgZG dd„ deƒZd	d
e› de› de› de
› d�	 e_dee dee dee dee dee dedededededededefdd„Zdee dee dee dee dee dedededededededefdd„Zeed�				d"dee dee dee dee dee ded ee dededededededefd!d„ƒZdS )#é    )ÚAnyÚDictÚListÚOptionalN)ÚTensoré   )Ú_capturable_docÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_disable_dynamo_if_unsupportedÚ_foreach_docÚ!_get_capturable_supported_devicesÚ_get_scalar_dtypeÚ_maximize_docÚ_use_grad_for_differentiableÚ_view_as_realÚ	OptimizerÚParamsTÚAdadeltaÚadadeltac                       s´   e Zd Z					dddddœded	ed
edededee dededef‡ fdd„Z‡ fdd„Zde	e
ef dee dee dee dee dee fdd„Zed dd„ƒZ‡  ZS )!r   ç      ð?çÍÌÌÌÌÌì?ç�íµ ÷Æ°>r   NF)Ú
capturableÚmaximizeÚdifferentiableÚparamsÚlrÚrhoÚepsÚweight_decayÚforeachr   r   r   c             
      s�   d|kst d|› �ƒ‚d|  krdksn t d|› �ƒ‚d|ks(t d|› �ƒ‚d|ks3t d|› �ƒ‚t||||||||	d�}
tƒ  ||
¡ d S )Ng        zInvalid learning rate: r   zInvalid rho value: zInvalid epsilon value: zInvalid weight_decay value: )r   r   r   r    r   r   r!   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)Úselfr   r   r   r   r    r!   r   r   r   Údefaults©Ú	__class__© úe/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/optim/adadelta.pyr%      s&   ø
zAdadelta.__init__c                    s¾   t ƒ  |¡ | jD ]S}| dd ¡ | dd¡ | dd¡ | dd¡ |d D ]4}| j |g ¡}t|ƒdkr[t |d ¡s[t	|d ƒ}|d rQtj
|tƒ |jd	�ntj
|tƒ d
�|d< q'q	d S )Nr!   r   Fr   r   r   r   Ústep©ÚdtypeÚdevice©r.   )r$   Ú__setstate__Úparam_groupsÚ
setdefaultÚstateÚgetÚlenÚtorchÚ	is_tensorÚfloatÚtensorr   r/   )r&   r4   ÚgroupÚpZp_stateZstep_valr(   r*   r+   r1   <   s&   
ý
ÿû€üûzAdadelta.__setstate__r;   Úparams_with_gradÚgradsÚsquare_avgsÚ
acc_deltasÚstate_stepsc           
      C   sî   d}|d D ]n}|j d u rq|t |¡O }| |¡ |j jr"tdƒ‚| |j ¡ | j| }	t|	ƒdkr_|d rAtjdt	ƒ |j
d�ntjdt	ƒ d�|	d	< tj|tjd
�|	d< tj|tjd
�|	d< | |	d ¡ | |	d ¡ | |	d	 ¡ q|S )NFr   z*Adadelta does not support sparse gradientsr   r   r*   r-   r0   r,   )Úmemory_formatÚ
square_avgÚ	acc_delta)Úgradr7   Ú
is_complexÚappendÚ	is_sparseÚRuntimeErrorr4   r6   Úzerosr   r/   Ú
zeros_likeÚpreserve_format)
r&   r;   r=   r>   r?   r@   rA   Úhas_complexr<   r4   r*   r*   r+   Ú_init_groupO   s2   	


ÿý
ÿ
ÿzAdadelta._init_groupc                 C   sâ   |   ¡  d}|dur!t ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]J}g }g }g }g }g }|d |d |d |d |d |d |d |d	 f\}	}
}}}}}}|  ||||||¡}t||||||	|
|||||||d
� q$|S )z°Perform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r    r!   r   r   r   )	r   r   r   r    r!   r   r   r   rM   )Ú _cuda_graph_capture_health_checkr7   Úenable_gradr2   rN   r   )r&   ÚclosureÚlossr;   r=   r>   r?   r@   rA   r   r   r   r    r!   r   r   r   rM   r*   r*   r+   r,   z   sd   
ÿ
ø÷ÿòzAdadelta.step)r   r   r   r   N©N)Ú__name__Ú
__module__Ú__qualname__r   r9   r   Úboolr%   r1   r   Ústrr   r   r   rN   r   r,   Ú__classcell__r*   r*   r(   r+   r      sZ    ù	õþýüûúù	÷
öõ"
þýüûú
ù+a  Implements Adadelta algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)},
                \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)},
                \: \lambda \text{ (weight decay)}                                                \\
            &\textbf{initialize} :  v_0  \leftarrow 0 \: \text{ (square avg)},
                \: u_0 \leftarrow 0 \: \text{ (accumulate variables)}                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm}if \: \lambda \neq 0                                                    \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm} v_t      \leftarrow v_{t-1} \rho + g^2_t (1 - \rho)                    \\
            &\hspace{5mm}\Delta x_t    \leftarrow   \frac{\sqrt{u_{t-1} +
                \epsilon }}{ \sqrt{v_t + \epsilon}  }g_t \hspace{21mm}                           \\
            &\hspace{5mm} u_t  \leftarrow   u_{t-1}  \rho +
                 \Delta x^2_t  (1 - \rho)                                                        \\
            &\hspace{5mm}\theta_t      \leftarrow   \theta_{t-1} - \gamma  \Delta x_t            \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `ADADELTA: An Adaptive Learning Rate Method`_.
    aá  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        rho (float, optional): coefficient used for computing a running average
            of squared gradients (default: 0.9). A higher value of `rho` will
            result in a slower average, which can be helpful for preventing
            oscillations in the learning process.
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-6).
        lr (float, optional): coefficient that scale delta before it is applied
            to the parameters (default: 1.0)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        z	
        zd

    .. _ADADELTA\: An Adaptive Learning Rate Method:
        https://arxiv.org/abs/1212.5701

    r   r>   r?   r@   rA   r   r   r   r    r   r   r   rM   c                   sL  t j ¡ s"|r"tdd�‰ t‡ fdd„t| |ƒD ƒƒs"J dˆ › d�ƒ‚t| ||||ƒD ]y\}}}}}|d7 }|	s9|n| }|dkrG|j||d	�}t  |¡r[t  |¡}t  |¡}t  |¡}| 	|¡j
||d| d
� | |¡ ¡ }| |¡ ¡ }|
r|| ¡ }| |¡ 	|¡ | 	|¡j
||d| d
� t  |¡r›t  |¡}|j|| d	� q*d S )NF©Úsupports_xlac                 3   ó0   � | ]\}}|j j|j jko|j jˆ v V  qd S rS   ©r/   Útype©Ú.0r<   r,   ©Úcapturable_supported_devicesr*   r+   Ú	<genexpr>  ó   € þ

þz*_single_tensor_adadelta.<locals>.<genexpr>úIIf capturable=True, params and state_steps must be on supported devices: Ú.r   r   ©Úalpha©Úvalue)r7   Ú_utilsÚis_compilingr   ÚallÚzipÚaddrF   Úview_as_realÚmul_Úaddcmul_Úsqrt_ÚcloneÚdiv_Úview_as_complexÚadd_)r   r>   r?   r@   rA   r   r   r   r    r   r   r   rM   ÚparamrE   rC   rD   r,   ÚstdÚdeltar*   ra   r+   Ú_single_tensor_adadeltañ   s>   ÿ
ý
ü
ÿ





èr{   c                   sÚ  |
rJ dƒ‚t j ¡ s(|r(tdd�‰ t‡ fdd„t| |ƒD ƒƒs(J dˆ › d�ƒ‚t| ƒdkr0d S t | ||||g¡}| 	¡ D ]¬\\}}}}}}|rPt
||||ƒ |d jrct j|t jd	d
d�d	d� nt  |d¡ |	rpt  |¡}|dkr‡|	rt j|||d� nt j|||d�}t  ||¡ t j|||d| d� t  ||¡}t  |¡ t  ||¡}t  |¡ t  ||¡ t  ||¡ t  ||¡ t j|||d| d� |rát|t jƒrát  || ¡ t  ||¡ q>t j||| d� q>d S )Nz#_foreach ops don't support autogradFrZ   c                 3   r\   rS   r]   r_   ra   r*   r+   rc   >  rd   z)_multi_tensor_adadelta.<locals>.<genexpr>re   rf   r   r   Úcpu)r/   rg   r   ri   )r7   rk   rl   r   rm   rn   r6   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr   Úis_cpuÚ_foreach_add_r:   Z_foreach_negZ_foreach_addÚ_foreach_mul_Z_foreach_addcmul_Z_foreach_sqrt_Z_foreach_div_Ú
isinstancer   )r   r>   r?   r@   rA   r   r   r   r    r   r   r   rM   Úgrouped_tensorsZdevice_paramsÚdevice_gradsZdevice_square_avgsZdevice_acc_deltasZdevice_state_stepsÚ_ry   Zdeltasr*   ra   r+   Ú_multi_tensor_adadelta'  sr   ÿ
ý
üÿ	úÿ
ÿ
ÿÿ

Çr†   )Úsingle_tensor_fnFr!   c	                C   s’   t j ¡ stdd„ |D ƒƒstdƒ‚|du rt| |dd�\}}|r*t j ¡ r*tdƒ‚|r4t j ¡ s4t}nt	}|| |||||	|
||||||d� dS )	zvFunctional API that performs Adadelta algorithm computation.

    See :class:`~torch.optim.Adadelta` for details.
    c                 s   s   � | ]	}t |tjƒV  qd S rS   )r‚   r7   r   )r`   Útr*   r*   r+   rc   ¡  s   € 
ÿzadadelta.<locals>.<genexpr>zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsNF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)r   r   r   r    r   r   r   rM   )
r7   rk   rl   rm   rI   r	   ÚjitÚis_scriptingr†   r{   )r   r>   r?   r@   rA   r   r!   r   rM   r   r   r   r    r   r…   Úfuncr*   r*   r+   r   †  s<   ÿÿ
ÿ
ó)FNFF)Útypingr   r   r   r   r7   r   Ú	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   Ú__all__r   Ú__doc__r9   rW   r{   r†   r   r*   r*   r*   r+   Ú<module>   sÖ   8 $óòñðäÿ6ÿþýüûùø	÷
öõôó
ò6ÿþýüûùø	÷
öõôó
ò_	õÿþýüûø	÷
öõóòñðï