o
    !ôj0  ã                   @   sf  d dl Z d dl mZ ddlmZmZmZmZmZmZm	Z	 d dl
mZmZ ddgZG dd„ deƒZd	d
e› de› de› d� e_				ddee dee dee dee dee dededededededefdd„Zdee dee dee dee dededededededefdd„Zdee dee dee dee dededededededefdd„ZdS ) é    N)ÚTensoré   )Ú	OptimizerÚ_use_grad_for_differentiableÚ_default_to_fused_or_foreachÚ_differentiable_docÚ_foreach_docÚ_maximize_docÚ_view_as_real)ÚListÚOptionalÚRpropÚrpropc                       sb   e Zd Z			dddddœdee ded	ef‡ fd
d„Z‡ fdd„Zdd„ Zeddd„ƒZ	‡  Z
S )r   ç{®Gáz„?©g      à?g333333ó?©g�íµ ÷Æ°>é2   NF)ÚforeachÚmaximizeÚdifferentiabler   r   r   c          	         s€   d|kst d|› �ƒ‚d|d   k rd  k r|d k s-n t d|d › d|d › �ƒ‚t||||||d�}tƒ  ||¡ d S )	Ng        zInvalid learning rate: r   g      ð?r   zInvalid eta values: z, )ÚlrÚetasÚ
step_sizesr   r   r   )Ú
ValueErrorÚdictÚsuperÚ__init__)	ÚselfÚparamsr   r   r   r   r   r   Údefaults©Ú	__class__© úd/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/torch/optim/rprop.pyr      s   (úzRprop.__init__c                    s@   t ƒ  |¡ | jD ]}| dd ¡ | dd¡ | dd¡ q	d S )Nr   r   Fr   )r   Ú__setstate__Úparam_groupsÚ
setdefault)r   ÚstateÚgroupr    r"   r#   r$   %   s   
ýzRprop.__setstate__c           
      C   sò   d}|d D ]p}|j d u rq|t |¡O }| |¡ |j }|jr$tdƒ‚| |¡ | j| }	t|	ƒdkr`d|	d< tj|tj	d�|	d< |j
jrVt |t|d |d ƒ¡|	d	< n
t ||d ¡|	d	< | |	d ¡ | |	d	 ¡ |	d  d
7  < q|S )NFr   z'Rprop does not support sparse gradientsr   Ústep©Úmemory_formatÚprevr   Ú	step_sizer   )ÚgradÚtorchÚ
is_complexÚappendÚ	is_sparseÚRuntimeErrorr'   ÚlenÚ
zeros_likeÚpreserve_formatÚdtypeÚ	full_likeÚcomplex)
r   r(   r   ÚgradsÚprevsr   Úhas_complexÚpr.   r'   r"   r"   r#   Ú_init_group,   s0   




ÿÿzRprop._init_groupc                 C   s¸   d}|durt  ¡ � |ƒ }W d  ƒ n1 sw   Y  | jD ]9}g }g }g }g }|d \}}	|d \}
}|d }|d }|  |||||¡}t|||||
|||	|||d |d� q |S )z±Performs a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr   r   r   r   r   )Ústep_size_minÚstep_size_maxÚetaminusÚetaplusr   r   r   r<   )r/   Úenable_gradr%   r>   r   )r   ÚclosureÚlossr(   r   r:   r;   r   rA   rB   r?   r@   r   r   r<   r"   r"   r#   r)   O   s<   
ÿ
ôz
Rprop.step)r   r   r   )N)Ú__name__Ú
__module__Ú__qualname__r   Úboolr   r$   r>   r   r)   Ú__classcell__r"   r"   r    r#   r   
   s&    û÷ùø	÷#aÁ
  Implements the resilient backpropagation algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \theta_0 \in \mathbf{R}^d \text{ (params)},f(\theta)
                \text{ (objective)},                                                             \\
            &\hspace{13mm}      \eta_{+/-} \text{ (etaplus, etaminus)}, \Gamma_{max/min}
                \text{ (step sizes)}                                                             \\
            &\textbf{initialize} :   g^0_{prev} \leftarrow 0,
                \: \eta_0 \leftarrow \text{lr (learning rate)}                                   \\
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\
            &\hspace{5mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})           \\
            &\hspace{5mm} \textbf{for} \text{  } i = 0, 1, \ldots, d-1 \: \mathbf{do}            \\
            &\hspace{10mm}  \textbf{if} \:   g^i_{prev} g^i_t  > 0                               \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{min}(\eta^i_{t-1} \eta_{+},
                \Gamma_{max})                                                                    \\
            &\hspace{10mm}  \textbf{else if}  \:  g^i_{prev} g^i_t < 0                           \\
            &\hspace{15mm}  \eta^i_t \leftarrow \mathrm{max}(\eta^i_{t-1} \eta_{-},
                \Gamma_{min})                                                                    \\
            &\hspace{15mm}  g^i_t \leftarrow 0                                                   \\
            &\hspace{10mm}  \textbf{else}  \:                                                    \\
            &\hspace{15mm}  \eta^i_t \leftarrow \eta^i_{t-1}                                     \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \eta_t \mathrm{sign}(g_t)             \\
            &\hspace{5mm}g_{prev} \leftarrow  g_t                                                \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to the paper
    `A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm
    <http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.21.1417>`_.
    aò  
    Args:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-2)
        etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that
            are multiplicative increase and decrease factors
            (default: (0.5, 1.2))
        step_sizes (Tuple[float, float], optional): a pair of minimal and
            maximal allowed step sizes (default: (1e-6, 50))
        z	
        z

    Fr   r:   r;   r   r   r   r   r<   r?   r@   rA   rB   c                C   sj   |du rt | |dd�\}}|rtj ¡ rtdƒ‚|r"tj ¡ s"t}nt}|| |||||	|
||||d� dS )zpFunctional API that performs rprop algorithm computation.

    See :class:`~torch.optim.Rprop` for details.
    NF)Ú	use_fusedz6torch.jit.script not supported with foreach optimizers)r?   r@   rA   rB   r   r   r<   )r   r/   ÚjitÚis_scriptingr3   Ú_multi_tensor_rpropÚ_single_tensor_rprop)r   r:   r;   r   r   r   r   r<   r?   r@   rA   rB   Ú_Úfuncr"   r"   r#   r   ¬   s(   
õc                C   s
  t | ƒD ]~\}}|| }|s|n| }|| }|| }t |¡r4t |¡}t |¡}t |¡}t |¡}|	r@| | ¡ ¡ ¡ }n| |¡ ¡ }||| d¡< ||| d¡< d|| 	d¡< | 
|¡ ||¡ |jtjd�}d|| 	|¡< |j| ¡ |dd� | |¡ qd S )Nr   r   r*   éÿÿÿÿ©Úvalue)Ú	enumerater/   r0   Úview_as_realÚmulÚcloneÚsignÚgtÚltÚeqÚmul_Úclamp_r6   Úaddcmul_Úcopy_)r   r:   r;   r   r?   r@   rA   rB   r   r   r<   ÚiÚparamr.   r,   r-   rY   r"   r"   r#   rO   Ü   s,   




ãrO   c                C   sH  t | ƒdkrd S |	rJ dƒ‚t | |||g¡}| ¡ D ]†\\}}}}}|
r,t||||ƒ t ||¡}|r9t |¡ t ||¡ |rFt |¡ |}t 	|¡ |D ]}||| 
d¡< ||| d¡< d|| d¡< qOt ||¡ |D ]}| ||¡ qot|ƒ}tt |ƒƒD ]}d|| ||  |¡< q‚~dd„ |D ƒ}tj|||dd� qd S )Nr   z#_foreach ops don't support autogradr   c                 S   s   g | ]}|  ¡ ‘qS r"   )rY   )Ú.0r.   r"   r"   r#   Ú
<listcomp>F  s    z'_multi_tensor_rprop.<locals>.<listcomp>rR   rS   )r4   r   Ú"_group_tensors_by_device_and_dtypeÚvaluesr
   r/   Ú_foreach_mulÚ_foreach_neg_Ú_foreach_copy_Z_foreach_sign_rZ   r[   r\   Ú_foreach_mul_r^   ÚlistÚrangeÚ_foreach_addcmul_)r   r:   r;   r   r?   r@   rA   rB   r   r   r<   Úgrouped_tensorsÚgrouped_paramsÚgrouped_gradsZgrouped_prevsZgrouped_step_sizesrP   ZsignsrY   r-   ra   Z
grad_signsr"   r"   r#   rN     s:   


ÙrN   )NFFF)r/   r   Ú	optimizerr   r   r   r   r   r	   r
   Útypingr   r   Ú__all__r   Ú__doc__rI   Úfloatr   rO   rN   r"   r"   r"   r#   Ú<module>   s´    $p"
öõôÞ9öÿþýüùø	÷
öôóò
ñ0ÿþýüúùø	÷
öõ
ô/ÿþýüúùø	÷
öõô