o
    pôj	!  ã                   @   sŠ   d dl Z d dlZddlmZmZ ddlmZmZmZm	Z	 e	ƒ r.d dl
m  mZ d dlmZ dd„ ZG dd„ dejjƒZd	efd
d„ZdS )é    Né   )ÚAcceleratorStateÚGradientState)ÚDistributedTypeÚ
honor_typeÚis_lomo_availableÚis_torch_xla_availablec                    sh   t | ttfƒrt| ‡ fdd„| D ƒƒS t | tƒr't| ƒ‡ fdd„|  ¡ D ƒƒS t | tjƒr2|  	ˆ ¡S | S )Nc                 3   s   � | ]}t |ˆ ƒV  qd S ©N©Úmove_to_device)Ú.0Út©Údevice© úe/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/optimizer.pyÚ	<genexpr>   s   € z!move_to_device.<locals>.<genexpr>c                    s   i | ]
\}}|t |ˆ ƒ“qS r   r
   ©r   ÚkÚvr   r   r   Ú
<dictcomp>    s    z"move_to_device.<locals>.<dictcomp>)
Ú
isinstanceÚlistÚtupler   ÚdictÚtypeÚitemsÚtorchÚTensorÚto)Ústater   r   r   r   r      s   

r   c                   @   sÈ   e Zd ZdZd%dd„Zedd„ ƒZejdd„ ƒZed	d
„ ƒZejdd
„ ƒZedd„ ƒZ	e	jdd„ ƒZ	dd„ Z
dd„ Zdd„ Zd&dd„Zdd„ Zdd„ Zd&dd„Zdd„ Zedd „ ƒZd!d"„ Zd#d$„ ZdS )'ÚAcceleratedOptimizera®  
    Internal wrapper around a torch optimizer.

    Conditionally will perform `step` and `zero_grad` if gradients should be synchronized when performing gradient
    accumulation.

    Args:
        optimizer (`torch.optim.optimizer.Optimizer`):
            The optimizer to wrap.
        device_placement (`bool`, *optional*, defaults to `True`):
            Whether or not the optimizer should handle device placement. If so, it will place the state dictionary of
            `optimizer` on the right device.
        scaler (`torch.amp.GradScaler` or `torch.cuda.amp.GradScaler`, *optional*):
            The scaler to use in the step function if training with mixed precision.
    TNc                 C   s¢   || _ || _tƒ | _tƒ | _|| _d| _| jd ur)d| _| j j	| _
t| | j j	ƒ| _|rO| j  ¡ }| jjtjkr@t || jj¡ nt|| jjƒ}| j  |¡ d S d S ©NF)Ú	optimizerÚscalerr   Úaccelerator_stater   Úgradient_stateÚdevice_placementÚ_is_overflowÚ_accelerate_step_calledÚstepÚ_optimizer_original_step_methodÚpatch_optimizer_stepÚ_optimizer_patched_step_methodÚ
state_dictÚdistributed_typer   ÚXLAÚxmÚsend_cpu_data_to_devicer   r   Úload_state_dict)Úselfr#   r'   r$   r.   r   r   r   Ú__init__7   s"   


úzAcceleratedOptimizer.__init__c                 C   ó   | j jS r	   ©r#   r    ©r4   r   r   r   r    M   ó   zAcceleratedOptimizer.statec                 C   ó   || j _d S r	   r7   ©r4   r    r   r   r   r    Q   ó   c                 C   r6   r	   ©r#   Úparam_groupsr8   r   r   r   r>   U   r9   z!AcceleratedOptimizer.param_groupsc                 C   r:   r	   r=   )r4   r>   r   r   r   r>   Y   r<   c                 C   r6   r	   ©r#   Údefaultsr8   r   r   r   r@   ]   r9   zAcceleratedOptimizer.defaultsc                 C   r:   r	   r?   )r4   r@   r   r   r   r@   a   r<   c                 C   s   | j  |¡ d S r	   )r#   Úadd_param_group)r4   Úparam_groupr   r   r   rA   e   s   z$AcceleratedOptimizer.add_param_groupc                 C   s4   | j jtjkr| jrt || j j¡ | j 	|¡ d S r	   )
r%   r/   r   r0   r'   r1   r2   r   r#   r3   )r4   r.   r   r   r   r3   h   s   z$AcceleratedOptimizer.load_state_dictc                 C   s
   | j  ¡ S r	   )r#   r.   r8   r   r   r   r.   m   s   
zAcceleratedOptimizer.state_dictc                 C   s`   | j jr.dt | jj¡jv }|r|d u rd}| jj|d� d S |d ur'tdƒ‚| j ¡  d S d S )NÚset_to_noneT)rC   zJ`set_to_none` for Optimizer.zero_grad` is not supported by this optimizer.)r&   Úsync_gradientsÚinspectÚ	signaturer#   Ú	zero_gradÚ
parametersÚ
ValueError)r4   rC   Z
accept_argr   r   r   rG   p   s   ÷zAcceleratedOptimizer.zero_gradc                 C   ój   t | jdƒrt| jjƒr| j ¡  dS t | jdƒr/t | jjdƒr1t| jjjƒr3| jj ¡  dS dS dS dS )z`
        Sets the optimizer to "train" mode. Useful for optimizers like `schedule_free`
        Útrainr#   N)Úhasattrr#   ÚcallablerK   r8   r   r   r   rK   |   ó   
ÿþýúzAcceleratedOptimizer.trainc                 C   rJ   )z_
        Sets the optimizer to "eval" mode. Useful for optimizers like `schedule_free`
        Úevalr#   N)rL   r#   rM   rO   r8   r   r   r   rO   Š   rN   zAcceleratedOptimizer.evalc                 C   sø   t ƒ rddlm}m} | jjs,| jjtj	kr,t
 | j¡}t
jd|dt ¡  d� d| j_t ƒ r9t| j||fƒr9d S | jjrm| jd urg| j| j_| j | j|¡ | j ¡  | js[d| _nd| _| j| j_d| _n| j |¡ | jjtj	krzd| j_d S d S )Nr   )ÚAdaLomoÚLomoÚsumg      ð?)ÚscaleTF)r   Ú
lomo_optimrP   rQ   r&   Úis_xla_gradients_syncedr%   r/   r   r0   r1   Ú_fetch_gradientsr#   Ú
all_reduceÚxrÚ
world_sizer   rD   r$   r-   r*   Úupdater)   r(   r+   )r4   ÚclosurerP   rQ   Ú	gradientsr   r   r   r*   ˜   s2   ÿ



ÿzAcceleratedOptimizer.stepc                    s,   | j jD ]}‡ fdd„|d D ƒ|d< qd S )Nc                    s   g | ]}ˆ   ||¡‘qS r   )Úget)r   Úp©Úparameters_mapr   r   Ú
<listcomp>À   s    z;AcceleratedOptimizer._switch_parameters.<locals>.<listcomp>Úparamsr=   )r4   r`   rB   r   r_   r   Ú_switch_parameters¾   s   ÿz'AcceleratedOptimizer._switch_parametersc                 C   s   | j S )z.Whether or not the optimizer step was skipped.)r(   r8   r   r   r   Ústep_was_skippedÂ   s   z%AcceleratedOptimizer.step_was_skippedc                    s    g d¢‰ ‡ fdd„| j  ¡ D ƒS )N)r)   r+   r-   c                    s   i | ]\}}|ˆ vr||“qS r   r   r   ©Z_ignored_keysr   r   r   Í   s    z5AcceleratedOptimizer.__getstate__.<locals>.<dictcomp>)Ú__dict__r   r8   r   re   r   Ú__getstate__Ç   s   z!AcceleratedOptimizer.__getstate__c                 C   s>   | j  |¡ | jd urd| _| jj| _t| | jjƒ| _d S d S r"   )	rf   rZ   r$   r)   r#   r*   r+   r,   r-   r;   r   r   r   Ú__setstate__Ï   s   

ýz!AcceleratedOptimizer.__setstate__)TNr	   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r5   Úpropertyr    Úsetterr>   r@   rA   r3   r.   rG   rK   rO   r*   rc   rd   rg   rh   r   r   r   r   r!   &   s6    








&
r!   Úaccelerated_optimizerc                    s   ‡ ‡fdd„}|S )Nc                     s   dˆ _ ˆ| i |¤ŽS )NT)r)   )ÚargsÚkwargs©ro   Úmethodr   r   Úpatched_stepØ   s   z*patch_optimizer_step.<locals>.patched_stepr   )ro   rs   rt   r   rr   r   r,   ×   s   r,   )rE   r   r    r   r   Úutilsr   r   r   r   Útorch_xla.core.xla_modelÚcoreÚ	xla_modelr1   Ztorch_xla.runtimeÚruntimerX   r   ÚoptimÚ	Optimizerr!   r,   r   r   r   r   Ú<module>   s   
 2