o
    Gôj¤T  ã                   @   s  d Z ddlZddlmZ ddlZddlZddlm	Z	 ddl
mZ ddlmZmZmZ eƒ r7ddlZddlmZ e e¡Zdd	„ Zeƒ rMeƒ rMdd
lmZ nddlmZ G dd„ deƒZG dd„ deƒZdadd„ Zdd„ Zdd„ Zdd„ Z dd„ Z!dd„ Z"d"dd„Z#d#d d!„Z$dS )$z
Integration with Deepspeed
é    N)Úpartialmethodé   )Údep_version_check)Úis_accelerate_availableÚis_torch_availableÚlogging)Únnc                  C   s@   t j d¡d u} | rzt d¡}W dS  tjy   Y dS w d S )NÚ	deepspeedTF)Ú	importlibÚutilÚ	find_specÚimportlib_metadataÚmetadataÚPackageNotFoundError)Úpackage_existsÚ_© r   úv/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/transformers/integrations/deepspeed.pyÚis_deepspeed_available$   s   
ÿür   )ÚHfDeepSpeedConfig)Úobjectc                       s    e Zd ZdZ‡ fdd„Z‡  ZS )r   aJ  
    This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage.

    A `weakref` of this object is stored in the module's globals to be able to access the config from areas where
    things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore
    it's important that this object remains alive while the program is still running.

    [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration
    with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic
    the DeepSpeed configuration is not modified in any way.

    Args:
        config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict.

    c                    s(   t | ƒ tdƒ tdƒ tƒ  |¡ d S )NÚ
accelerater	   )Úset_hf_deepspeed_configr   ÚsuperÚ__init__©ÚselfZconfig_file_or_dict©Ú	__class__r   r   r   J   s   zHfDeepSpeedConfig.__init__)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__classcell__r   r   r   r   r   9   s    r   c                       sX   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd
d„Zeedd�Z	ddd„Z
dd„ Z‡  ZS )ÚHfTrainerDeepSpeedConfigzž
    The `HfTrainerDeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has the
    same lifespan as the latter.
    c                    s   t ƒ  |¡ d | _g | _d S ©N)r   r   Ú_dtypeÚ
mismatchesr   r   r   r   r   X   s   
z!HfTrainerDeepSpeedConfig.__init__c                 C   s   | j d u r	tdƒ‚| j S )Nz8trainer_config_process() wasn't called yet to tell dtype)r&   Ú
ValueError©r   r   r   r   Údtype]   s   
zHfTrainerDeepSpeedConfig.dtypec                 C   s   |   |¡}|d u rdS |dkS )NFÚauto)Ú	get_value)r   Úds_key_longÚvalr   r   r   Úis_autob   s   
z HfTrainerDeepSpeedConfig.is_autoNTc              
   C   s†   |   |¡\}}|du rdS | |¡dkr|||< dS |sdS | |¡}|dur?||krA| j d|› d|› d|› d|› �¡ dS dS dS )a¶  
        A utility method that massages the config file and can optionally verify that the values match.

        1. Replace "auto" values with `TrainingArguments` value.

        2. If it wasn't "auto" and `must_match` is true, then check that DS config matches Trainer
        config values and if mismatched add the entry to `self.mismatched` - will assert during
        `trainer_config_finalize` for one or more mismatches.

        Nr+   z- ds ú=z vs hf )Zfind_config_nodeÚgetr'   Úappend)r   r-   Zhf_valZhf_keyÚ
must_matchÚconfigZds_keyÚds_valr   r   r   Ú
fill_matchi   s   
(ÿz#HfTrainerDeepSpeedConfig.fill_matchF)r3   c                 C   s–  |j |j |j }|  d|jd| ¡ |  d|jd¡ |  d|d| ¡ |  d|jd¡ |  d|jd	¡ |  d
|j|jgd¡ |  d|jd¡ |  d|j	d¡ |  
dd¡ |  d|jd	¡ |jsc|jrm|jdkrjdnd}nd}|jr„| j di ¡| jd< |j| jd d< |  d|js�|jo�|dkd¡ |  d|dkd¡ |  d|jd¡ |  d|jp¬|jd¡ |  d¡rºtj| _dS |  d¡rÅtj| _dS tj| _dS ) zŠ
        Adjust the config with `TrainingArguments` values. This stage is run during `TrainingArguments` object
        creation.
        Ztrain_micro_batch_size_per_gpuÚper_device_train_batch_sizeÚgradient_accumulation_stepsÚtrain_batch_sizeztrain_batch_size (calculated)Zgradient_clippingÚmax_grad_normzoptimizer.params.lrÚlearning_ratezoptimizer.params.betaszadam_beta1+adam_beta2zoptimizer.params.epsÚadam_epsilonzoptimizer.params.weight_decayÚweight_decayzscheduler.params.warmup_min_lrr   zscheduler.params.warmup_max_lrZapexÚampNÚ
checkpointZuse_node_local_storagezfp16.enabledz%fp16|fp16_full_eval+fp16_backend(amp)zamp.enabledzfp16+fp16_backend(apex)zamp.opt_levelÚfp16_opt_levelzbf16.enabledzbf16|bf16_full_eval)Ú
world_sizer7   r8   r6   r:   r;   Z
adam_beta1Z
adam_beta2r<   r=   Ú	fill_onlyÚfp16Zfp16_full_evalÚfp16_backendZsave_on_each_noder4   r1   r@   Úbf16Zbf16_full_evalÚis_trueÚtorchÚbfloat16r&   Zis_falseÚfloat32Úfloat16)r   ÚargsZauto_find_batch_sizer9   rD   r   r   r   Útrainer_config_process…   s`   üýü
ýý

z/HfTrainerDeepSpeedConfig.trainer_config_processc                    sF  g d¢}‡ fdd„|D ƒ}t |ƒdkr{t|jdƒr|jj}n=t|jdƒr+t|jjƒ}n0t|jdƒr>t|jjdƒr>|jjj}nt|jdƒrSt|jjdƒrSt|jjjƒ}ntd|› d	�ƒ‚ˆ  d
|| ¡ ˆ  	¡ r{ˆ  dt
d| | ƒ¡ ˆ  dd| ¡ ˆ  d|d¡ ˆ  d| |¡d¡ t ˆ jƒdkr¡d ˆ j¡}td|› d�ƒ‚dS )z�
        This stage is run after we have the model and know num_training_steps.

        Now we can complete the configuration process.
        )ú$zero_optimization.reduce_bucket_sizeú-zero_optimization.stage3_prefetch_bucket_sizeú4zero_optimization.stage3_param_persistence_thresholdc                    s   g | ]	}ˆ   |¡r|‘qS r   )r/   )Ú.0Úxr)   r   r   Ú
<listcomp>Ý   ó    zDHfTrainerDeepSpeedConfig.trainer_config_finalize.<locals>.<listcomp>r   Úhidden_sizeÚhidden_sizesÚtext_configz½The model's config file has neither `hidden_size` nor `hidden_sizes` entry, therefore it's not possible to automatically fill out the following `auto` entries in the DeepSpeed config file: zb. You can fix that by replacing `auto` values for these keys with an integer value of your choice.rM   rN   gÍÌÌÌÌÌì?rO   é
   z scheduler.params.total_num_stepsznum_training_steps (calculated)z!scheduler.params.warmup_num_stepsZwarmup_stepsÚ
z]Please correct the following DeepSpeed config values that mismatch TrainingArguments values:
zF
The easiest method is to set these DeepSpeed config values to 'auto'.N)ÚlenÚhasattrr4   rT   ÚmaxrU   rV   r(   rB   Úis_zero3Úintr6   Zget_warmup_stepsr'   Újoin)r   rK   ÚmodelÚnum_training_stepsZhidden_size_based_keysZhidden_size_auto_keysrT   r'   r   r)   r   Útrainer_config_finalizeÏ   sX   	
þÿþþýýÿÿþz0HfTrainerDeepSpeedConfig.trainer_config_finalize)NT©F)r   r    r!   r"   r   r*   r/   r6   r   rB   rL   ra   r#   r   r   r   r   r$   R   s    

Jr$   c                 C   s   t  | ¡ad S r%   )ÚweakrefÚrefÚ_hf_deepspeed_config_weak_ref)Zhf_deepspeed_config_objr   r   r   r     s   r   c                   C   s   d a d S r%   )re   r   r   r   r   Úunset_hf_deepspeed_config  s   rf   c                   C   s    t d urt ƒ d urt ƒ  ¡ S dS )NF)re   r\   r   r   r   r   Úis_deepspeed_zero3_enabled$  s   
rg   c                   C   s   t d urt ƒ d urt ƒ jS d S r%   )re   r4   r   r   r   r   Údeepspeed_config+  s   rh   c                    sR   t |ddƒ‰| ¡ }ˆdurˆ|_g ‰ d	dtjf‡ ‡‡fdd„‰ˆ| |dd� ˆ S )
zÏ
    Loads state dict into a model specifically for Zero3, since DeepSpeed does not support the `transformers`
    tensor parallelism API.

    Nearly identical code to PyTorch's `_load_from_state_dict`
    Ú	_metadataNÚ FÚmodulec           
         s  ˆd u ri n	ˆ  ˆd d… i ¡}||d< |ˆ|dg g ˆf}tƒ rst‡fdd„|D ƒƒdkrsdd l}t| jˆd d… dd�ƒ‰ ‡ fd	d„| ¡ D ƒ}t|ƒdkrs|jj|dd
�� t	j
 ¡ dkrd| j|Ž  W d   ƒ n1 snw   Y  | j ¡ D ]\}}	|	d ur‹ˆ|	|ˆ| d |ƒ qxd S )NéÿÿÿÿÚassign_to_params_buffersTc                    s   g | ]	}|  ˆ ¡r|‘qS r   )Ú
startswith)rP   Úkey)Úprefixr   r   rR   J  rS   zC_load_state_dict_into_zero3_model.<locals>.load.<locals>.<listcomp>r   F)rp   Úrecursec                    s   g | ]
}|ˆ v rˆ | ‘qS r   r   )rP   Úk)Únamed_parametersr   r   rR   P  s    )Úmodifier_rankÚ.)r1   rg   rY   r	   Údictrs   ÚkeysÚzeroÚGatheredParametersrG   ÚdistributedÚget_rankÚ_load_from_state_dictÚ_modulesÚitems)
rk   Ú
state_dictrp   rm   Úlocal_metadatarK   r	   Zparams_to_gatherÚnameÚchild©Ú
error_msgsÚloadr   )rs   rp   r   r…   C  s$     
€þ€þz/_load_state_dict_into_zero3_model.<locals>.load)rm   )rj   F)ÚgetattrÚcopyri   r   ÚModule)Úmodel_to_loadr   r   rƒ   r   Ú!_load_state_dict_into_zero3_model2  s   rŠ   c                    s¼   ddl m}m} |j}d}d|v r|jrtdƒ‚||d�}n| ¡ r't d¡ ˆ 	¡ }d|d	< d}	d
|v r=||ƒ}	||	fS t
||ƒrS‡ ‡fdd„}
|||
d�}	||	fS ˆjˆ |d�}	||	fS )zY
    A convenience wrapper that deals with optimizer and lr scheduler configuration.
    r   )Ú
DummyOptimÚDummySchedulerNÚ	optimizerz|--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. Only one optimizer can be configured.)Úparamsz¢Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the custom optimizer has both CPU and GPU implementation (except LAMB)TZzero_allow_untested_optimizerÚ	schedulerc                    s"   t   ˆ¡}d |_|jˆ | d�}|S )N©r`   r�   )r‡   Úlr_schedulerÚcreate_scheduler)r�   Ztrainer_copyr‘   ©r`   Útrainerr   r   Ú_lr_scheduler_callable�  s   
ÿz5deepspeed_optim_sched.<locals>._lr_scheduler_callable)Zlr_scheduler_callabler�   )Úaccelerate.utilsr‹   rŒ   r4   Z	adafactorr(   Z
is_offloadÚloggerÚinfoÚcreate_optimizerÚ
isinstancer’   )r”   Úhf_deepspeed_configrK   r`   Úmodel_parametersr‹   rŒ   r4   r�   r‘   r•   r   r“   r   Údeepspeed_optim_schedb  s2   
ÿÿ
ïþr�   Fc                 C   sî   ddl m} | j}| j}| jjjj}| |||¡ | 	| 
¡ ¡ |r>| ¡ s*tdƒ‚| d¡ | d¡ d\}}d}	||fS d| _|j di ¡ d	d¡}
|
d
kr^ddl}|j||
| ¡ d�}ttdd„ | ¡ ƒƒ}	t| ||||	ƒ\}}||fS )a  
    Init DeepSpeed, after updating the DeepSpeed configuration with any relevant Trainer's args.

    If `resume_from_checkpoint` was passed then an attempt to resume from a previously saved checkpoint will be made.

    Args:
        trainer: Trainer object
        num_training_steps: per single gpu
        resume_from_checkpoint: path to a checkpoint if to resume from after normal DeepSpeedEngine load
        inference: launch in inference mode (no optimizer and no lr scheduler)
        auto_find_batch_size: whether to ignore the `train_micro_batch_size_per_gpu` argument as it's being
            set automatically by the auto batch size finder

    Returns: optimizer, lr_scheduler

    We may use `deepspeed_init` more than once during the life of Trainer, when we do - it's a temp hack based on:
    https://github.com/deepspeedai/DeepSpeed/issues/1394#issuecomment-937405374 until Deepspeed fixes a bug where it
    can't resume from a checkpoint after it did some stepping https://github.com/deepspeedai/DeepSpeed/issues/1612

    r   )r—   zMZeRO inference only makes sense with ZeRO Stage 3 - please adjust your configr�   r‘   )NNNÚtensor_parallelZautotp_sizeé   )r_   Útp_sizer*   c                 S   s   | j S r%   )Úrequires_grad)Úpr   r   r   Ú<lambda>Ò  s    z deepspeed_init.<locals>.<lambda>)Zdeepspeed.utilsr—   r_   rK   ÚacceleratorÚstateZdeepspeed_pluginZhf_ds_configra   ÚsetLevelZget_process_log_levelr\   r(   Zdel_config_sub_treer�   r4   r1   r	   Ztp_model_initr*   ÚlistÚfilterÚ
parametersr�   )r”   r`   Ú	inferenceZ	ds_loggerr_   rK   r›   r�   r‘   rœ   r    r	   r   r   r   Údeepspeed_initŸ  s0   

ò
ÿr«   Tc                 C   sv   dd l }t|  |› d�¡ƒ}t|ƒdkr4t d|› �¡ | j||ddd�\}}|d u r2td|› �ƒ‚d S td|› �ƒ‚)Nr   z/global_step*zAttempting to resume from T)Úload_module_strictZload_optimizer_statesZload_lr_scheduler_statesz-[deepspeed] failed to resume from checkpoint z!Can't find a valid checkpoint at )ÚglobÚsortedrY   r—   r˜   Zload_checkpointr(   )Zdeepspeed_engineÚcheckpoint_pathr¬   r­   Zdeepspeed_checkpoint_dirsZ	load_pathr   r   r   r   Údeepspeed_load_checkpointÝ  s   
üÿr°   rb   )T)%r"   r‡   Úimportlib.metadatar   r   Úimportlib.utilr
   rc   Ú	functoolsr   Údependency_versions_checkr   Úutilsr   r   r   rG   r   Ú
get_loggerr   r—   r   Zaccelerate.utils.deepspeedr   ZDeepSpeedConfigÚbuiltinsr   r$   re   r   rf   rg   rh   rŠ   r�   r«   r°   r   r   r   r   Ú<module>   s8   
 B0
=>