o
    §ôjÀJ  ã                   @   s  d Z ddlZddlmZ ddlZddlZddlm	Z	 ddl
mZ ddlmZmZmZmZ eƒ r3ddlZe e¡Zdd„ Zeƒ rIeƒ rIdd	lmZ ndd
lmZ G dd„ deƒZG dd„ deƒZdadd„ Zdd„ Zdd„ Zdd„ Z dd„ Z!ddd„Z"d dd„Z#dS )!z
Integration with Deepspeed
é    N)Úpartialmethodé   )Údep_version_check)Úis_accelerate_availableÚis_torch_availableÚis_torch_mlu_availableÚloggingc                  C   sV   t j d¡d u} | r)ztƒ rt d¡}W dS t d¡}W dS  tjy(   Y dS w d S )NÚ	deepspeedzdeepspeed-mluTF)Ú	importlibÚutilÚ	find_specr   Úimportlib_metadataÚmetadataÚPackageNotFoundError)Úpackage_existsÚ_© r   út/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/transformers/integrations/deepspeed.pyÚis_deepspeed_available#   s   

ÿùr   )ÚHfDeepSpeedConfig)Úobjectc                       s    e Zd ZdZ‡ fdd„Z‡  ZS )r   aJ  
    This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage.

    A `weakref` of this object is stored in the module's globals to be able to access the config from areas where
    things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore
    it's important that this object remains alive while the program is still running.

    [`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration
    with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic
    the DeepSpeed configuration is not modified in any way.

    Args:
        config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict.

    c                    s(   t | ƒ tdƒ tdƒ tƒ  |¡ d S )NÚ
accelerater	   )Úset_hf_deepspeed_configr   ÚsuperÚ__init__©ÚselfZconfig_file_or_dict©Ú	__class__r   r   r   L   s   zHfDeepSpeedConfig.__init__)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ú__classcell__r   r   r   r   r   ;   s    r   c                       sX   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd
d„Zeedd�Z	ddd„Z
dd„ Z‡  ZS )ÚHfTrainerDeepSpeedConfigzž
    The `HfTrainerDeepSpeedConfig` object is meant to be created during `TrainingArguments` object creation and has the
    same lifespan as the latter.
    c                    s   t ƒ  |¡ d | _g | _d S ©N)r   r   Ú_dtypeÚ
mismatchesr   r   r   r   r   Z   s   
z!HfTrainerDeepSpeedConfig.__init__c                 C   s   | j d u r	tdƒ‚| j S )Nz8trainer_config_process() wasn't called yet to tell dtype)r&   Ú
ValueError©r   r   r   r   Údtype_   s   
zHfTrainerDeepSpeedConfig.dtypec                 C   s   |   |¡}|d u rdS |dkS )NFÚauto)Ú	get_value)r   Úds_key_longÚvalr   r   r   Úis_autod   s   
z HfTrainerDeepSpeedConfig.is_autoNTc              
   C   s†   |   |¡\}}|du rdS | |¡dkr|||< dS |sdS | |¡}|dur?||krA| j d|› d|› d|› d|› �¡ dS dS dS )a¶  
        A utility method that massages the config file and can optionally verify that the values match.

        1. Replace "auto" values with `TrainingArguments` value.

        2. If it wasn't "auto" and `must_match` is true, then check that DS config matches Trainer
        config values and if mismatched add the entry to `self.mismatched` - will assert during
        `trainer_config_finalize` for one or more mismatches.

        Nr+   z- ds ú=z vs hf )Zfind_config_nodeÚgetr'   Úappend)r   r-   Zhf_valZhf_keyÚ
must_matchÚconfigZds_keyÚds_valr   r   r   Ú
fill_matchk   s   
(ÿz#HfTrainerDeepSpeedConfig.fill_matchF)r3   c                 C   s–  |j |j |j }|  d|jd| ¡ |  d|jd¡ |  d|d| ¡ |  d|jd¡ |  d|jd	¡ |  d
|j|jgd¡ |  d|jd¡ |  d|j	d¡ |  
dd¡ |  d|jd	¡ |jsc|jrm|jdkrjdnd}nd}|jr„| j di ¡| jd< |j| jd d< |  d|js�|jo�|dkd¡ |  d|dkd¡ |  d|jd¡ |  d|jp¬|jd¡ |  d¡rºtj| _dS |  d¡rÅtj| _dS tj| _dS ) zŠ
        Adjust the config with `TrainingArguments` values. This stage is run during `TrainingArguments` object
        creation.
        Ztrain_micro_batch_size_per_gpuÚper_device_train_batch_sizeÚgradient_accumulation_stepsÚtrain_batch_sizeztrain_batch_size (calculated)Zgradient_clippingÚmax_grad_normzoptimizer.params.lrÚlearning_ratezoptimizer.params.betaszadam_beta1+adam_beta2zoptimizer.params.epsÚadam_epsilonzoptimizer.params.weight_decayÚweight_decayzscheduler.params.warmup_min_lrr   zscheduler.params.warmup_max_lrZapexÚampNÚ
checkpointZuse_node_local_storagezfp16.enabledz%fp16|fp16_full_eval+fp16_backend(amp)zamp.enabledzfp16+fp16_backend(apex)zamp.opt_levelÚfp16_opt_levelzbf16.enabledzbf16|bf16_full_eval)Ú
world_sizer7   r8   r6   r:   r;   Z
adam_beta1Z
adam_beta2r<   r=   Ú	fill_onlyÚfp16Zfp16_full_evalÚfp16_backendZsave_on_each_noder4   r1   r@   Úbf16Zbf16_full_evalÚis_trueÚtorchÚbfloat16r&   Úis_falseÚfloat32Úfloat16)r   ÚargsZauto_find_batch_sizer9   rD   r   r   r   Útrainer_config_process‡   s`   üýü
ýý

z/HfTrainerDeepSpeedConfig.trainer_config_processc                    sF  g d¢}‡ fdd„|D ƒ}t |ƒdkr{t|jdƒr|jj}n=t|jdƒr+t|jjƒ}n0t|jdƒr>t|jjdƒr>|jjj}nt|jdƒrSt|jjdƒrSt|jjjƒ}ntd|› d	�ƒ‚ˆ  d
|| ¡ ˆ  	¡ r{ˆ  dt
d| | ƒ¡ ˆ  dd| ¡ ˆ  d|d¡ ˆ  d| |¡d¡ t ˆ jƒdkr¡d ˆ j¡}td|› d�ƒ‚dS )z�
        This stage is run after we have the model and know num_training_steps.

        Now we can complete the configuration process.
        )ú$zero_optimization.reduce_bucket_sizeú-zero_optimization.stage3_prefetch_bucket_sizeú4zero_optimization.stage3_param_persistence_thresholdc                    s   g | ]	}ˆ   |¡r|‘qS r   )r/   )Ú.0Úxr)   r   r   Ú
<listcomp>ß   s    zDHfTrainerDeepSpeedConfig.trainer_config_finalize.<locals>.<listcomp>r   Úhidden_sizeÚhidden_sizesÚtext_configz½The model's config file has neither `hidden_size` nor `hidden_sizes` entry, therefore it's not possible to automatically fill out the following `auto` entries in the DeepSpeed config file: zb. You can fix that by replacing `auto` values for these keys with an integer value of your choice.rN   rO   gÍÌÌÌÌÌì?rP   é
   z scheduler.params.total_num_stepsznum_training_steps (calculated)z!scheduler.params.warmup_num_stepsZwarmup_stepsÚ
z]Please correct the following DeepSpeed config values that mismatch TrainingArguments values:
zF
The easiest method is to set these DeepSpeed config values to 'auto'.N)ÚlenÚhasattrr4   rT   ÚmaxrU   rV   r(   rB   Úis_zero3Úintr6   Zget_warmup_stepsr'   Újoin)r   rL   ÚmodelÚnum_training_stepsZhidden_size_based_keysZhidden_size_auto_keysrT   r'   r   r)   r   Útrainer_config_finalizeÑ   sX   	
þÿþþýýÿÿþz0HfTrainerDeepSpeedConfig.trainer_config_finalize)NT©F)r   r    r!   r"   r   r*   r/   r6   r   rB   rM   ra   r#   r   r   r   r   r$   T   s    

Jr$   c                 C   s   t  | ¡ad S r%   )ÚweakrefÚrefÚ_hf_deepspeed_config_weak_ref)Zhf_deepspeed_config_objr   r   r   r     s   r   c                   C   s   d a d S r%   )re   r   r   r   r   Úunset_hf_deepspeed_config   s   rf   c                   C   s    t d urt ƒ d urt ƒ  ¡ S dS )NF)re   r\   r   r   r   r   Úis_deepspeed_zero3_enabled&  s   
rg   c                   C   s   t d urt ƒ d urt ƒ jS d S r%   )re   r4   r   r   r   r   Údeepspeed_config-  s   rh   c                    s¼   ddl m}m} |j}d}d|v r|jrtdƒ‚||d�}n| ¡ r't d¡ ˆ 	¡ }d|d	< d}	d
|v r=||ƒ}	||	fS t
||ƒrS‡ ‡fdd„}
|||
d�}	||	fS ˆjˆ |d�}	||	fS )zY
    A convenience wrapper that deals with optimizer and lr scheduler configuration.
    r   )Ú
DummyOptimÚDummySchedulerNÚ	optimizerz|--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. Only one optimizer can be configured.)Úparamsz¢Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the custom optimizer has both CPU and GPU implementation (except LAMB)TZzero_allow_untested_optimizerÚ	schedulerc                    s"   t   ˆ¡}d |_|jˆ | d�}|S )N©r`   rk   )ÚcopyÚlr_schedulerÚcreate_scheduler)rk   Ztrainer_copyrp   ©r`   Útrainerr   r   Ú_lr_scheduler_callable_  s   
ÿz5deepspeed_optim_sched.<locals>._lr_scheduler_callable)Zlr_scheduler_callablern   )Zaccelerate.utilsri   rj   r4   Z	adafactorr(   Z
is_offloadÚloggerÚinfoÚcreate_optimizerÚ
isinstancerq   )rs   Úhf_deepspeed_configrL   r`   Úmodel_parametersri   rj   r4   rk   rp   rt   r   rr   r   Údeepspeed_optim_sched4  s2   
ÿÿ
ïþr{   Fc           
      C   s´   ddl m} | j}| j}| jjjj}| |||¡ | 	| 
¡ ¡ |r>| ¡ s*tdƒ‚| d¡ | d¡ d\}}d}	||fS d| _ttdd	„ | ¡ ƒƒ}	t| ||||	ƒ\}}||fS )
a  
    Init DeepSpeed, after updating the DeepSpeed configuration with any relevant Trainer's args.

    If `resume_from_checkpoint` was passed then an attempt to resume from a previously saved checkpoint will be made.

    Args:
        trainer: Trainer object
        num_training_steps: per single gpu
        resume_from_checkpoint: path to a checkpoint if to resume from after normal DeepSpeedEngine load
        inference: launch in inference mode (no optimizer and no lr scheduler)
        auto_find_batch_size: whether to ignore the `train_micro_batch_size_per_gpu` argument as it's being
            set automatically by the auto batch size finder

    Returns: optimizer, lr_scheduler

    We may use `deepspeed_init` more than once during the life of Trainer, when we do - it's a temp hack based on:
    https://github.com/microsoft/DeepSpeed/issues/1394#issuecomment-937405374 until Deepspeed fixes a bug where it
    can't resume from a checkpoint after it did some stepping https://github.com/microsoft/DeepSpeed/issues/1612

    r   )ru   zMZeRO inference only makes sense with ZeRO Stage 3 - please adjust your configrk   rp   )NNNc                 S   s   | j S r%   )Úrequires_grad)Úpr   r   r   Ú<lambda>Ÿ  s    z deepspeed_init.<locals>.<lambda>)Zdeepspeed.utilsru   r_   rL   ÚacceleratorÚstateZdeepspeed_pluginZhf_ds_configra   ÚsetLevelZget_process_log_levelr\   r(   Zdel_config_sub_treerk   ÚlistÚfilterÚ
parametersr{   )
rs   r`   Ú	inferenceZ	ds_loggerr_   rL   ry   rk   rp   rz   r   r   r   Údeepspeed_initq  s(   

÷
ÿr†   Tc                 C   sv   dd l }t|  |› d�¡ƒ}t|ƒdkr4t d|› �¡ | j||ddd�\}}|d u r2td|› �ƒ‚d S td|› �ƒ‚)Nr   z/global_step*zAttempting to resume from T)Úload_module_strictZload_optimizer_statesZload_lr_scheduler_statesz-[deepspeed] failed to resume from checkpoint z!Can't find a valid checkpoint at )ÚglobÚsortedrY   ru   rv   Úload_checkpointr(   )Zdeepspeed_engineÚcheckpoint_pathr‡   rˆ   Zdeepspeed_checkpoint_dirsÚ	load_pathr   r   r   r   Údeepspeed_load_checkpointª  s   
üÿr�   rb   )T)$r"   ro   Úimportlib.metadatar   r   Úimportlib.utilr
   rc   Ú	functoolsr   Údependency_versions_checkr   Úutilsr   r   r   r   rG   Ú
get_loggerr   ru   r   Zaccelerate.utils.deepspeedr   ZDeepSpeedConfigÚbuiltinsr   r$   re   r   rf   rg   rh   r{   r†   r�   r   r   r   r   Ú<module>   s4   
 B
=9