o
    pôjÅ  ã                   @   sÜ  d dl Z d dlZd dlZd dlmZ d dlmZ d dlZd dlm	  m
Z d dlmZmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZmZ eƒ rúd dlmZmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z'm(Z( d dl)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5m6Z6m7Z7m8Z8 d dl9m:Z:m;Z;m<Z<m=Z=m>Z> d dl?m@Z@mAZAmBZB d dlCmDZD d dlEmFZF d dlGmHZHmIZImJZJmKZKmLZLmMZM d dlNmOZO d dlPmQZQmRZRmSZSmTZTmUZUmVZV d dlWmXZXmYZYmZZZ dGd!d"„Z[d#d$„ Z\G d%d&„ d&ƒZ]d'd(„ Z^d)d*„ Z_G d+d,„ d,eƒZ`d-d.„ ZaG d/d0„ d0ƒZbG d1d2„ d2eƒZcd3d4„ ZdG d5d6„ d6eƒZeG d7d8„ d8eeƒZfG d9d:„ d:eeƒZgG d;d<„ d<eeƒZhd=d>„ ZidHd?d@„ZjG dAdB„ dBej	jkƒZldCdD„ ZmdEdF„ ZndS )Ié    N)ÚABC)Úpartial)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚAcceleratedOptimizer)ÚAcceleratedScheduleré   )Úis_megatron_lm_available)Úrecursively_applyÚsend_to_device)ÚmpuÚtensor_parallel)ÚDistributedDataParallel)Úfinalize_model_grads)Ú	ModelType)Úget_num_microbatches)Úget_megatron_optimizer)Úget_tensor_model_parallel_groupÚ"get_tensor_model_parallel_src_rank)Úget_forward_backward_func)Úget_model_config)Úbuild_train_valid_test_datasets)Ú	BertModelÚT5Model)ÚClassification)Úget_argsÚget_tensorboard_writerÚget_tokenizerÚprint_rank_last)Ú_add_data_argsÚ_add_validation_argsÚ!core_transformer_config_from_argsÚ
parse_argsÚvalidate_args)Úload_args_from_checkpointÚload_checkpointÚsave_checkpoint)Úset_global_variables)Úgpt_builder)Ú_compile_dependenciesÚ_init_autoresumeÚ_initialize_distributedÚ_set_random_seedÚset_jit_fusion_optionsÚwrite_args_to_tensorboard)Ú_vocab_size_with_padding)Ú%build_train_valid_test_data_iteratorsÚget_optimizer_param_schedulerÚnum_floating_point_operationsÚsetup_model_and_optimizerÚ
train_stepÚtraining_log)Ú)average_losses_across_data_parallel_groupÚcalc_params_l2_normÚget_ltor_masks_and_position_idsTc           	   	   C   sð   t ƒ }|jrdnd}|jdkrtd|j› d|› d�ƒ tdƒ t|ƒ}|jdkrK|jr?|jr0d	nd}t|||jd
| |d�}|S t||j	d	| |d�}|S |jdkr^d|_
t|| |ddd�}|S |jdkrpt|dd
| |||d�}|S td|j› �ƒ‚)zBuild the model.zpre-trainingzfine-tuningr   z	Building z model in the z mode.z»The Megatron LM model weights are initialized at random in `accelerator.prepare`. Please use `accelerator.load_checkpoint` to load a pre-trained checkpoint matching the distributed setup.Úbertr   T)ÚconfigÚnum_tokentypesZadd_binary_headÚparallel_outputÚpre_processÚpost_process)r<   Únum_classesr=   r?   r@   ÚgptFN)Zvp_stager<   Út5)r<   r=   r>   r?   r@   Úadd_encoderÚadd_decoderúUnsupported model type: )r   Úpretraining_flagÚrankÚprintÚmodel_type_namer#   Úbert_binary_headr   r   Ú
num_labelsZuse_legacy_modelsr*   r   Ú
ValueError)	r?   r@   rD   rE   ÚargsÚmoder<   r=   Úmodel© rQ   úm/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/utils/megatron_lm.pyÚmodel_provider_funcU   sX   
ÿ
ú éû
ð
ôùÿrS   c                 C   s¼   |   d¡ tƒ }| jjjd ur3| jjjd u rtdƒ‚| jjj}| jj |¡}t| |ƒ}t| |d d�}n!t	j
}|jdkr>t	j}t}| jjjd urL| jjj}t||ƒ\}}}t|ƒ|_|||fS )Nz#Preparing model optimizer schedulerzaYou must provide a `custom_model_provider_function` when using a `custom_prepare_model_function`.)Ú	schedulerrC   )rI   r   ÚstateÚmegatron_lm_pluginÚcustom_prepare_model_functionÚcustom_model_provider_functionrM   Úprepare_optimizerÚprepare_schedulerr   Zencoder_or_decoderrJ   Zencoder_and_decoderrS   r5   ÚlenÚ	model_len)ÚacceleratorrN   Zcustom_model_provider_funcrP   Ú	optimizerrT   Ú
model_typeZmodel_provider_func_rQ   rQ   rR   Ú!prepare_model_optimizer_scheduler†   s.   
ÿ




þ

r`   c                   @   s0   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
S )ÚMegatronLMDummyDataLoaderzº
    Dummy dataloader presents model parameters or param groups, this is primarily used to follow conventional training

    Args:
        **dataset_kwargs: Megatron data arguments.
    c                 K   sH   t  ¡ }t|ƒ}t|ƒ}| ¡ }t|d ƒ| _| j |¡ d| jd< d S )Nr   TÚmegatron_dataset_flag)ÚargparseÚArgumentParserr!   r"   Úparse_known_argsÚvarsÚdataset_argsÚupdate)ÚselfZdataset_kwargsÚparserÚ	data_argsrQ   rQ   rR   Ú__init__ª   s   z"MegatronLMDummyDataLoader.__init__c              
   C   s^   t ƒ }| j ¡ D ]$\}}t||dƒ}||kr&td|› d|› d|› d|› �ƒ t|||ƒ qd S )NÚ z<WARNING: MegatronLMDummyDataLoader overriding arguments for ú:ú with )r   rg   ÚitemsÚgetattrrI   Úsetattr)ri   rN   ÚkeyÚvalueÚ	old_valuerQ   rQ   rR   Úset_megatron_data_args³   s   ÿúz0MegatronLMDummyDataLoader.set_megatron_data_argsc                 C   s¨   dd„ }|j jjd ur|j jjS z9tƒ }|jdkr%ddlm} d|_|W S |jdkr6ddlm} d|_|W S |jdkrGddl	m} d|_|W S W |S  t
yS   Y |S w )	Nc                 S   s¾   t ƒ }t|jttfƒr|jn|jg|j| |jdœ}|jdkr)| |j	|j
dœ¡ n'|jdkr7| d|j	i¡ n|jdkrH| |j|jddœ¡ ntd|j› �ƒ‚td
i |¤Ž\}}}|||fS )z&Build train, valid, and test datasets.)Zdata_prefixZsplits_stringZtrain_valid_test_num_samplesÚseedr;   )Úmax_seq_lengthZbinary_headrB   rx   rC   )rx   Zmax_seq_length_decÚdataset_typerF   NrQ   )r   Ú
isinstanceÚ	data_pathÚlistÚtupleÚsplitrw   rJ   rh   Ú
seq_lengthrK   Úencoder_seq_lengthÚdecoder_seq_lengthrM   r   )Ztrain_val_test_num_samplesrN   rg   Ztrain_dsZvalid_dsZtest_dsrQ   rQ   rR   Ú"train_valid_test_datasets_provider¾   s6   ü
þÿ
ÿÿ
ýÿ
zlMegatronLMDummyDataLoader.get_train_valid_test_datasets_provider.<locals>.train_valid_test_datasets_providerr;   r   )r‚   TrB   rC   )rU   rV   Ú*custom_megatron_datasets_provider_functionr   rJ   Úpretrain_bertr‚   Úis_distributedÚpretrain_gptÚpretrain_t5ÚImportError)ri   r]   r‚   rN   rQ   rQ   rR   Ú&get_train_valid_test_datasets_provider½   s.   #



üþþz@MegatronLMDummyDataLoader.get_train_valid_test_datasets_providerc           	      C   s’   t ƒ }|  |¡}|jd ur=g }g }g }tt|ddƒƒD ] }t |¡ t|ƒ}| |d ¡ | |d ¡ | |d ¡ qnt|ƒ\}}}|||fS )Nr\   r   r
   r   )	r   r‰   Z$virtual_pipeline_model_parallel_sizeÚrangerq   r   Z(set_virtual_pipeline_model_parallel_rankr2   Úappend)	ri   r]   rN   Z!train_valid_test_dataset_providerÚtrain_data_iteratorÚvalid_data_iteratorÚtest_data_iteratorÚiÚ	iteratorsrQ   rQ   rR   r2   ù   s"   


û
ÿ
z?MegatronLMDummyDataLoader.build_train_valid_test_data_iteratorsN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rl   rv   r‰   r2   rQ   rQ   rQ   rR   ra   ¢   s    	
<ra   c                 C   sR   G dd„ dƒ}|d u }t j|t j| jd�}t jj|tƒ tƒ d� |s'|r'|ƒ S |S )Nc                   @   s   e Zd Zdd„ Zdd„ ZdS )z?_handle_megatron_data_iterator.<locals>.DummyMegatronDataloaderc                 S   s   | S ©NrQ   ©ri   rQ   rQ   rR   Ú__iter__  ó   zH_handle_megatron_data_iterator.<locals>.DummyMegatronDataloader.__iter__c                 S   s   i S r•   rQ   r–   rQ   rQ   rR   Ú__next__  r˜   zH_handle_megatron_data_iterator.<locals>.DummyMegatronDataloader.__next__N)r‘   r’   r“   r—   r™   rQ   rQ   rQ   rR   ÚDummyMegatronDataloader  s    rš   ©ÚdtypeÚdevice©Úgroup)ÚtorchÚtensorÚboolr�   ÚdistributedÚ	broadcastr   r   )r]   Údata_iteratorrš   Zis_data_iterator_emptyZis_src_data_iterator_emptyrQ   rQ   rR   Ú_handle_megatron_data_iterator  s   
ÿr¦   c           	   
      sh  |   d¡ tƒ }|jspddlm‰ m} |j|j }‡ ‡fdd„ˆ D ƒ}|d d u rHt|d t	j
jjƒr9||d _n|d= |d= |d= ||d	 _n|d	= ||d< t	j
jjˆjfi |¤Ž‰|ˆ| jt ¡ t ¡ d
d| j ¡ | jd�S |jd ur|j\|_|_|_nd\|_|_|_|j|j |_ˆ | ¡\}}}|j|j |_t| |d�}t| |d�}t| |d�}|||fS )NzPreparing dataloaderr   )Ú_PYTORCH_DATALOADER_KWARGSÚprepare_data_loaderc                    s   i | ]}|t ˆ|ˆ | ƒ“qS rQ   )rq   )Ú.0Úk©r§   Ú
dataloaderrQ   rR   Ú
<dictcomp>(  s    z'prepare_data_loader.<locals>.<dictcomp>Ú
batch_sizeÚsamplerÚshuffleÚbatch_samplerFT)Únum_processesÚprocess_indexÚsplit_batchesÚput_on_deviceÚ	rng_typesÚdispatch_batches)r   r   r   )r]   r¥   )rI   r   rb   Údata_loaderr§   r¨   Úmicro_batch_sizeÚnum_micro_batchesrz   r    ÚutilsÚdataÚBatchSamplerr®   Ú
DataLoaderÚdatasetr�   r   Úget_data_parallel_world_sizeZget_data_parallel_rankr¶   Úcopyr·   Úconsumed_samplesÚconsumed_train_samplesÚconsumed_valid_samplesZconsumed_test_samplesr2   r¦   )	r]   r¬   rN   r¨   r¹   ÚkwargsrŒ   r�   rŽ   rQ   r«   rR   r¨   !  s`   
ø
üüÿÿ
r¨   c                       s:   e Zd Z‡ fdd„Zd
dd„Zdd„ Zedd	„ ƒZ‡  ZS )ÚMegatronLMOptimizerWrapperc                    s   t ƒ j|dd d� d S )NF)Údevice_placementÚscaler©Úsuperrl   )ri   r^   ©Ú	__class__rQ   rR   rl   d  s   z#MegatronLMOptimizerWrapper.__init__Nc                 C   ó   d S r•   rQ   )ri   Úset_to_nonerQ   rQ   rR   Ú	zero_gradg  r˜   z$MegatronLMOptimizerWrapper.zero_gradc                 C   rÍ   r•   rQ   r–   rQ   rQ   rR   Ústepj  r˜   zMegatronLMOptimizerWrapper.stepc                 C   s   | j jS )zTWhether or not the optimizer step was done, or skipped because of gradient overflow.)r^   Úskipped_iterr–   rQ   rQ   rR   Ústep_was_skippedm  s   z+MegatronLMOptimizerWrapper.step_was_skippedr•   )	r‘   r’   r“   rl   rÏ   rÐ   ÚpropertyrÒ   Ú__classcell__rQ   rQ   rË   rR   rÆ   c  s    
rÆ   c                 C   s$   |   d¡ tƒ }t||j|j|jƒS )NzPreparing optimizer)rI   r   r   Úno_wd_decay_condÚscale_lr_condÚlr_mult)r]   rP   rN   rQ   rQ   rR   rY   s  s   
rY   c                   @   s   e Zd ZdZddd„ZdS )ÚMegatronLMDummySchedulera  
    Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training
    loop when scheduler config is specified in the deepspeed config file.

    Args:
        optimizer (`torch.optim.optimizer.Optimizer`):
            The optimizer to wrap.
        total_num_steps (int):
            Total number of steps.
        warmup_num_steps (int):
            Number of steps for warmup.
        **kwargs (additional keyword arguments, *optional*):
            Other arguments.
    Nr   c                 K   s   || _ || _|| _|| _d S r•   )r^   Útotal_num_stepsÚwarmup_num_stepsrÅ   )ri   r^   rÙ   rÚ   rÅ   rQ   rQ   rR   rl   Š  s   
z!MegatronLMDummyScheduler.__init__©Nr   )r‘   r’   r“   r”   rl   rQ   rQ   rQ   rR   rØ   z  s    rØ   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚMegatronLMSchedulerWrapperc                    s   t ƒ  ||¡ d S r•   rÉ   )ri   rT   Ú
optimizersrË   rQ   rR   rl   ’  s   z#MegatronLMSchedulerWrapper.__init__c                 O   rÍ   r•   rQ   )ri   rN   rÅ   rQ   rQ   rR   rÐ   •  r˜   zMegatronLMSchedulerWrapper.step)r‘   r’   r“   rl   rÐ   rÔ   rQ   rQ   rË   rR   rÜ   ‘  s    rÜ   c                 C   s   |   d¡ t|ƒ}|S )NzPreparing scheduler)rI   r3   )r]   r^   rT   rQ   rQ   rR   rZ   ™  s   
rZ   c                       ó8   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd	„ Z‡  ZS )
ÚAbstractTrainStepz;Abstract class for batching, forward pass and loss handler.c                    s   t ƒ  ¡  || _d S r•   )rÊ   rl   Úname)ri   rà   rË   rQ   rR   rl   ¢  s   

zAbstractTrainStep.__init__c                 C   rÍ   r•   rQ   )ri   r]   rb   rQ   rQ   rR   Úget_batch_func¦  r˜   z AbstractTrainStep.get_batch_funcc                 C   rÍ   r•   rQ   r–   rQ   rQ   rR   Úget_forward_step_func©  r˜   z'AbstractTrainStep.get_forward_step_funcc                 C   rÍ   r•   rQ   )ri   r]   rQ   rQ   rR   Úget_loss_func¬  r˜   zAbstractTrainStep.get_loss_func)	r‘   r’   r“   r”   rl   rá   râ   rã   rÔ   rQ   rQ   rË   rR   rß   Ÿ  s    rß   c                       rÞ   )
ÚBertTrainStepzg
    Bert train step class.

    Args:
        args (`argparse.Namespace`): Megatron-LM arguments.
    c                    sh   t ƒ  d¡ |  ||j¡| _|  ||j|j¡| _|  	|j|j
¡| _|js)d | _d S ddlm} || _d S )Nrä   r   )ÚSequenceClassifierOutput)rÊ   rl   rá   rb   Ú	get_batchrã   rG   rL   Ú	loss_funcrâ   rK   Úforward_stepÚmodel_return_dictÚmodel_output_classÚtransformers.modeling_outputsrå   )ri   r]   rN   rå   rË   rQ   rR   rl   ¸  s   

zBertTrainStep.__init__c                 C   óX   dd„ }dd„ }|j jjd ur|j jjS |r*z	ddlm} |W S  ty)   Y |S w |S )Nc                 S   sŠ   g d¢}t j}| durt| ƒ}nd}t |||¡}|d  ¡ }|d  ¡ }|d  ¡ }|d  ¡ }|d  ¡ }	|d  ¡ }
|||||	|
fS )	úBuild the batch.)ÚtextÚtypesÚlabelsÚ	is_randomÚ	loss_maskÚpadding_maskNrî   rï   rñ   rò   rð   ró   ©r    Úint64Únextr   Úbroadcast_dataÚlongÚfloat)r¥   ÚkeysÚdatatyper¼   Údata_bÚtokensrï   Úsentence_orderrò   Ú	lm_labelsró   rQ   rQ   rR   Úget_batch_megatronÅ  s   
z8BertTrainStep.get_batch_func.<locals>.get_batch_megatronc                 S   s¦   t | ƒ}t|tj ¡ ƒ}|d  ¡ }|d  ¡ }d|v r#|d  ¡ }nd}d|v r:|d  ¡ }|d dk tj¡}nd}d}d|v rI|d  ¡ }nd}||||||fS )rí   Ú	input_idsÚattention_maskÚtoken_type_idsNrð   éœÿÿÿZnext_sentence_label)rö   r   r    ÚcudaÚcurrent_devicerø   Útorù   )r¥   r¼   rý   ró   rï   rÿ   rò   rþ   rQ   rQ   rR   Úget_batch_transformerÝ  s    z;BertTrainStep.get_batch_func.<locals>.get_batch_transformerr   ©ræ   )rU   rV   Úcustom_get_batch_functionr„   ræ   rˆ   ©ri   r]   rb   r   r  ræ   rQ   rQ   rR   rá   Ä  s   
þzBertTrainStep.get_batch_funcc                    s:   dd„ }‡ ‡fdd„}|j jjd ur|j jjS |r|S |S )Nc           	      S   s´   |\}}|  ¡ }|   ¡ } t | d¡|  d¡ ¡|  ¡  }|d urKtj| dd¡  ¡ | d¡dd�}|  ¡ }|| }t||gƒ}||d |d dœfS |}t|gƒ}|d|d ifS )Néÿÿÿÿr   )Úignore_indexr   r
   )úlm losszsop lossr  )rù   r    ÚsumÚviewÚreshapeÚFÚcross_entropyr8   )	rò   rþ   Úoutput_tensorÚlm_loss_Z
sop_logitsÚlm_lossZsop_lossÚlossÚaveraged_lossesrQ   rQ   rR   Úloss_func_pretrain  s   ""
z7BertTrainStep.get_loss_func.<locals>.loss_func_pretrainc                    sŒ   ˆ dkrt ƒ }|| d¡|  d¡ƒ}n&ˆjdkr1| jtjtjfv r1tƒ }|| dˆ ¡|  d¡ƒ}ntƒ }||| ƒ}t	|gƒ}|d|d ifS )Nr
   r  r  r   )
r   r  rL   rœ   r    rø   Úintr   r   r8   )rð   ÚlogitsZloss_fctr  r  ©rL   ri   rQ   rR   Úloss_func_finetune  s   

z7BertTrainStep.get_loss_func.<locals>.loss_func_finetune©rU   rV   Úcustom_loss_function)ri   r]   rG   rL   r  r  rQ   r  rR   rã     s   
zBertTrainStep.get_loss_funcc                    s   ‡ ‡‡fdd„}|S )Nc           
         sb   ˆ  | ¡\}}}}}}ˆ sd}ˆr"|||||d�}|tˆj||ƒfS ||||d�}	|	tˆj|ƒfS )úForward step.N©Útokentype_idsrÿ   )r"  ©ræ   r   rç   )
r¥   rP   rý   rï   rþ   rò   rð   ró   r  r  ©rK   rG   ri   rQ   rR   rè   .  s   z9BertTrainStep.get_forward_step_func.<locals>.forward_steprQ   )ri   rG   rK   rè   rQ   r$  rR   râ   -  s   z#BertTrainStep.get_forward_step_func©	r‘   r’   r“   r”   rl   rá   rã   râ   rÔ   rQ   rQ   rË   rR   rä   °  s    @)rä   c                       rÞ   )
ÚGPTTrainStepzf
    GPT train step class.

    Args:
        args (`argparse.Namespace`): Megatron-LM arguments.
    c                    s˜   t ƒ  d¡ |  ||j¡| _|  |¡| _|  ¡ | _|j	d ur%t
ƒ }|j| _|j| _|j| _|j| _|j| _|j| _|jsAd | _d S ddlm} || _d S )Nr&  r   )Ú!CausalLMOutputWithCrossAttentions)rÊ   rl   rá   rb   ræ   rã   rç   râ   rè   Ú
vocab_filer   ZeodÚ	eod_tokenÚeos_token_idÚ	pad_tokenÚreset_position_idsÚreset_attention_maskÚeod_mask_lossré   rê   rë   r'  )ri   r]   rN   Ú	tokenizerr'  rË   rQ   rR   rl   F  s    



zGPTTrainStep.__init__c                    s`   ‡ fdd„}‡ fdd„}|j jjd ur|j jjS |r.z	ddlm} |W S  ty-   Y |S w |S )Nc              	      s    dg}t j}| durt| ƒ}nd}t |||¡}|d  ¡ }|dd…dd…f  ¡ }|dd…dd…f  ¡ }t|ˆ jˆ jˆ j	ˆ j
ˆ jdd�\}}	}
|||	||
fS )zGenerate a batchrî   Nr
   r  T©r)  r+  r,  r-  r.  Zpad_mask_loss)r    rõ   rö   r   r÷   rø   Ú
contiguousr:   r)  r,  r-  r.  )r¥   rú   rû   r¼   rü   Útokens_rð   rý   r  rò   Úposition_idsr–   rQ   rR   r   [  s&   
ù	z7GPTTrainStep.get_batch_func.<locals>.get_batch_megatronc           	   	      sÊ   t | ƒ}d|d i}t|tj ¡ ƒ}|d  ¡ }tj|jd df|j|j	d�ˆ j
 }tj||gdd�}|d d …dd …f  ¡ }|d d …d d…f  ¡ }t|ˆ j
ˆ j
ˆ jˆ jˆ jdd�\}}}|||||fS )	Nr  r   r
   r›   ©Údimr  Tr0  )rö   r   r    r  r  rø   ÚzerosÚshaperœ   r�   r)  Úconcatr1  r:   r,  r-  r.  )	r¥   r¼   r2  Úpaddingrð   rý   r  rò   r3  r–   rQ   rR   r  y  s$   $ù	z:GPTTrainStep.get_batch_func.<locals>.get_batch_transformerr   r	  )rU   rV   r
  r†   ræ   rˆ   r  rQ   r–   rR   rá   Z  s   
þzGPTTrainStep.get_batch_funcc                    s.   t ƒ ‰ ‡ fdd„}|jjjd ur|jjjS |S )Nc                    s  ˆ j r|\}}n|}| ¡ }|  d¡ ¡ } ˆ jdkrDt t | d¡|  ¡ d¡|  ¡  d¡g¡}tjj|t	 
¡ d� |d |d  }nt | d¡|  ¡|  ¡  }ˆ jrrtj ¡ }| ¡ rrJ d|› dtj ¡ › dt ¡ d › �ƒ‚t|gƒ}d|d i}ˆ j r‡| d	|i¡ ||fS )
Nr  r
   rž   r   zRank z7: found NaN in local forward loss calculation. Device: z, node: r  r  )Úreturn_logitsrù   r  Úcontext_parallel_sizer    Úcatr  r£   Ú
all_reducer   Zget_context_parallel_groupZcheck_for_nan_in_loss_and_gradÚget_rankÚisnanr  r  ÚosÚunamer8   rh   )rò   r  Úlossesr  r  Úglobal_rankZaveraged_lossZoutput_dict©rN   rQ   rR   rç      s0   

.

ÿ
ÿÿ
z-GPTTrainStep.get_loss_func.<locals>.loss_func)r   rU   rV   r  ©ri   r]   rç   rQ   rD  rR   rã   �  s
   
zGPTTrainStep.get_loss_funcc                    ó   ‡ fdd„}|S )Nc                    s4   ˆ   | ¡\}}}}}|||||d�}|tˆ j|ƒfS )r   )rð   r#  )r¥   rP   rý   rð   rò   r  r3  r  r–   rQ   rR   rè   Ã  s   z8GPTTrainStep.get_forward_step_func.<locals>.forward_steprQ   ©ri   rè   rQ   r–   rR   râ   Â  s   z"GPTTrainStep.get_forward_step_funcr%  rQ   rQ   rË   rR   r&  >  s    C%r&  c                       s\   e Zd ZdZ‡ fdd„Zedd„ ƒZedd„ ƒZedd	„ ƒZd
d„ Z	dd„ Z
dd„ Z‡  ZS )ÚT5TrainStepze
    T5 train step class.

    Args:
        args (`argparse.Namespace`): Megatron-LM arguments.
    c                    sX   t ƒ  d¡ |  ||j¡| _|  |¡| _|  ¡ | _|j	s!d | _
d S ddlm} || _
d S )NrH  r   )ÚSeq2SeqLMOutput)rÊ   rl   rá   rb   ræ   rã   rç   râ   rè   ré   rê   rë   rI  )ri   r]   rN   rI  rË   rQ   rR   rl   Ö  s   


zT5TrainStep.__init__c                 C   s(   |   d¡}|   d¡}|| }|dk }|S )Nr
   r   ç      à?)Ú	unsqueeze)r  Úattention_mask_b1sÚattention_mask_bs1Úattention_mask_bssÚextended_attention_maskrQ   rQ   rR   Úattn_mask_postprocessâ  s
   

z!T5TrainStep.attn_mask_postprocessc                 C   s&   t  t jd| | f|d�¡}|dk }|S ©Nr
   ©r�   rJ  )r    ÚtrilÚones)r   r�   r  rQ   rQ   rR   Úget_decoder_maskï  s   zT5TrainStep.get_decoder_maskc           	      C   s<   | j \}}|  d¡}tj||df|d�}|| }|dk }|S rQ  )r7  rK  r    rT  )	r  Zdec_seq_lengthr�   r®   Ú_rL  rM  rN  rO  rQ   rQ   rR   Úget_enc_dec_maskõ  s   

zT5TrainStep.get_enc_dec_maskc                 C   rì   )Nc                 S   s˜   g d¢}t j}| durt| ƒ}nd}t |||¡}|d  ¡ }|d  ¡ }|d  ¡ }|d  ¡ }|d dk }	|d	 dk }
|d
 dk }|||||	|
|fS )rí   )Útext_encÚtext_decrð   rò   Úenc_maskÚdec_maskÚenc_dec_maskNrX  rY  rð   rò   rZ  rJ  r[  r\  rô   )r¥   rú   rû   r¼   rü   Ú
tokens_encÚ
tokens_decrð   rò   rZ  r[  r\  rQ   rQ   rR   r     s   
z6T5TrainStep.get_batch_func.<locals>.get_batch_megatronc           	      S   sú   t | ƒ}t|tj ¡ ƒ}|d  ¡ }|d  ¡ }|dk tj¡}d|v r+|d  ¡ }n'|j|j	|j
tjd�}|ddd…f  ¡ |dd	d…f< d
|d< | |dkd
¡ t |d  ¡ ¡}t |j	d	 |j
¡}t |d  ¡ |j	d	 |j
¡}|||||||fS )rí   r  rð   r  Údecoder_input_ids)r�   rœ   .Nr  r
   r   ).r   r  )rö   r   r    r  r  rø   r  rù   Ú	new_zerosr7  r�   ÚcloneÚmasked_fill_rH  rP  rU  rW  )	r¥   r¼   r]  rð   rò   r^  rZ  r[  r\  rQ   rQ   rR   r    s"    ÿz9T5TrainStep.get_batch_func.<locals>.get_batch_transformerr   r	  )rU   rV   r
  r‡   ræ   rˆ   r  rQ   rQ   rR   rá     s   
þzT5TrainStep.get_batch_funcc                 C   s$   dd„ }|j jjd ur|j jjS |S )Nc                 S   sH   |  ¡ }t | d¡|  d¡ ¡|  ¡  }|}t|gƒ}|d|d ifS )Nr  r  r   )rù   r    r  r  r  r8   )rò   r  r  r  r  r  rQ   rQ   rR   rç   A  s
   "
z,T5TrainStep.get_loss_func.<locals>.loss_funcr  rE  rQ   rQ   rR   rã   @  s   	
zT5TrainStep.get_loss_funcc                    rF  )Nc           
   	      s>   ˆ   | ¡\}}}}}}}||||||d|d�}	|	tˆ j|ƒfS )r   Nr!  r#  )
r¥   rP   r]  r^  rò   rÿ   rZ  r[  r\  r  r–   rQ   rR   rè   O  s   ÿÿz7T5TrainStep.get_forward_step_func.<locals>.forward_steprQ   rG  rQ   r–   rR   râ   N  s   z!T5TrainStep.get_forward_step_func)r‘   r’   r“   r”   rl   ÚstaticmethodrP  rU  rW  rá   rã   râ   rÔ   rQ   rQ   rË   rR   rH  Î  s    


?rH  c                  C   s@   t ƒ } td d d ƒ | jdkrtd| j› d�ƒ t| j| jƒ d S )Nr   z> setting random seeds to z ...)r   r-   rH   rI   rw   r.   Zdata_parallel_random_initrD  rQ   rQ   rR   Úfinish_mpu_init_  s
   
rd  c              
   C   sD  |d u ri }|   d¡ tj ¡ sJ dƒ‚t|dd�}| ¡ D ],\}}t||d ƒd urD|jdkrDt d|› dt||ƒ› d|› d|› �dd	� t|||ƒ q|j	sT| 
d
d¡ra|jd us]J dƒ‚t|ƒ t|ƒ t|dd� tƒ  tƒ  tƒ  tƒ  tƒ }t|dd ƒd u r‰t|j|ƒ|_|jdkrš|jrš|jdkršd|_nd|_d|_d S )NzInitializing Megatron-LMzMegatron requires CUDA.T)Zignore_unknown_argsr   z*WARNING: overriding default arguments for rn   ro   )ÚflushÚuse_checkpoint_argsFz/--use-checkpoints-args requires --load argument)Zbuild_tokenizerÚpadded_vocab_sizer;   r   )rI   r    r  Úis_availabler$   rp   rq   rH   rr   rf  ÚgetÚloadr&   r%   r)   rd  r,   r+   r/   r   r1   Úorig_vocab_sizerg  rJ   rG   rL   rK   Ú	iteration)r]   Zextra_args_providerÚargs_defaultsrN   rs   rt   rQ   rQ   rR   Ú
initializel  s:   

 þ
rn  c                       sp   e Zd ZdZ‡ fdd„Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Z‡  ZS )ÚMegatronEnginezþ
    Megatron-LM model wrapper

    Args:
        accelerator (:class:`~accelerate.Accelerator`): The accelerator object to use.
        model: Megatron-LM model
        optimizer: Megatron-LM optimizer
        lr_scheduler: Megatron-LM lr scheduler
    c                    sö   t ƒ  ¡  || _|d | _|| _|| _tƒ }|jjj	d ur-|jjj	|fi |jjj
¤Ž| _n,|jdkr9t||ƒ| _n |jdkrEt||ƒ| _n|jdkrQt||ƒ| _ntd|j› �ƒ‚d| j_i | _i | _d| _d| _d| _d | _|jd urytƒ  d S d S )Nr   r;   rB   rC   rF   FT)rÊ   rl   ÚmoduleÚ
base_modelr^   rT   r   rU   rV   Úcustom_train_step_classÚcustom_train_step_kwargsÚtrain_step_handlerrJ   rä   r&  rH  rM   rÑ   Útotal_loss_dictÚeval_total_loss_dictrl  Úreport_memory_flagÚ$num_floating_point_operations_so_farÚmodule_configÚtensorboard_dirr0   )ri   r]   rP   r^   rT   rN   rË   rQ   rR   rl   ©  s:   

ÿ
ÿ




ÿzMegatronEngine.__init__c                    sò   t ƒ }tˆ jd ƒ}ˆ jj|_tˆ jd tƒrR|jrR|j	d u s#J dƒ‚dd„ ˆ jD ƒ|_	t
ˆ jƒdkr9|j	d |_	|jrRdd„ ˆ jD ƒ|_t
ˆ jƒdkrR|jd |_|jrt|jrt‡ fdd„tt
ˆ jƒƒD ƒ|_t
ˆ jƒdkrt|jd |_t|_|S )Nr   z‡When overlap_grad_reduce is True, config.no_sync_func must be None; a custom no_sync_func is not supported when overlapping grad-reducec                 S   ó   g | ]}|j ‘qS rQ   )Úno_sync©r©   Zmodel_chunkrQ   rQ   rR   Ú
<listcomp>Ò  ó    z4MegatronEngine.get_module_config.<locals>.<listcomp>r
   c                 S   r{  rQ   )Zstart_grad_syncr}  rQ   rQ   rR   r~  Ö  r  c                    s   g | ]	‰ ‡ ‡fd d„‘qS )c                    s   ˆj  ˆ | ¡S r•   )r^   Zfinish_param_sync)Úx)Úmodel_indexri   rQ   rR   Ú<lambda>Û  s    z=MegatronEngine.get_module_config.<locals>.<listcomp>.<lambda>rQ   )r©   r–   )r�  rR   r~  Ú  s    ÿ)r   r   rp  r^   Z
scale_lossZgrad_scale_funcrz   ÚLocalDDPZoverlap_grad_reduceZno_sync_funcr[   Zdelay_grad_reduceZgrad_sync_funcZoverlap_param_gatherZdelay_param_gatherrŠ   Zparam_sync_funcr   Zfinalize_model_grads_func)ri   rN   r<   rQ   r–   rR   Úget_module_configÈ  s,   
ÿ
ÿz MegatronEngine.get_module_configc                 C   s4   | j D ]}| ¡  q| jd u r|  ¡ | _|  ¡  d S r•   )rp  Útrainry  r„  Úlog_eval_results©ri   Zmodel_modulerQ   rQ   rR   r…  â  s
   



zMegatronEngine.trainc                 C   s0   | j D ]}| ¡  q| jd u r|  ¡ | _d S d S r•   )rp  Úevalry  r„  r‡  rQ   rQ   rR   rˆ  ë  s
   


ÿzMegatronEngine.evalc                    sÂ   t ƒ ‰ g ‰t|ƒdkr,ˆ jdkr)tdˆ jƒD ]‰ˆ ‡ ‡fdd„| ¡ D ƒ¡ qn|g‰t| jƒdkrSt|ƒdkrI‡fdd„tt| jƒƒD ƒ}|S d gt| jƒ }|S t|ƒdkr]tˆƒnd }|S )Nr   r
   c                    s.   i | ]\}}||ˆˆ j  ˆd  ˆ j  … “qS )r
   )r¹   )r©   rª   Úv)rN   r�   rQ   rR   r­   ù  s    ÿÿz:MegatronEngine.get_batch_data_iterator.<locals>.<dictcomp>c                    s   g | ]}t ˆ ƒ‘qS rQ   )Úiter©r©   rV  )Údata_chunksrQ   rR   r~    ó    z:MegatronEngine.get_batch_data_iterator.<locals>.<listcomp>)r   r[   rº   rŠ   r‹   rp   rp  rŠ  )ri   Ú
batch_dataÚbatch_data_iteratorrQ   )rN   rŒ  r�   rR   Úget_batch_data_iteratorò  s,   
þÿÿÿÿüýÿz&MegatronEngine.get_batch_data_iteratorc              	   K   sT   |   |¡}t| jj|| j| j| j| jtƒ d�\}}}}}}}|dk| j_	||||fS )z€
        Training step for Megatron-LM

        Args:
            batch_data (:obj:`dict`): The batch data to train on.
        )Úforward_step_funcr¥   rP   r^   Zopt_param_schedulerr<   Úforward_backward_funcr
   )
r�  r6   rt  rè   rp  r^   rT   ry  r   rÑ   )ri   rŽ  r�  Úloss_reducedrÑ   rV  Ú	grad_normÚnum_zeros_in_gradrQ   rQ   rR   r6     s   
ù
zMegatronEngine.train_stepc              	      sØ   t ƒ }|  |¡}tƒ }|| jj|| jtƒ |j|jdd�}|j	dkr&t
j ¡  | jt ¡ |j tƒ  7  _tjdd�rji }|d D ]&‰ ‡ fdd„|D ƒ}t|d jƒdkr`t|ƒt|ƒ |ˆ < qAt
 |¡|ˆ < qA|S i S )z…
        Evaluation step for Megatron-LM

        Args:
            batch_data (:obj:`dict`): The batch data to evaluate on.
        T)r‘  r¥   rP   Znum_microbatchesr   r¹   Zforward_onlyr
   )Zignore_virtualr   c                    s   g | ]}|ˆ  ‘qS rQ   rQ   )r©   r€  ©rs   rQ   rR   r~  C  r�  z,MegatronEngine.eval_step.<locals>.<listcomp>)r   r�  r   rt  rè   rp  r   r   r¹   Zempty_unused_memory_levelr    r  Úempty_cacherÄ   r   rÀ   Zis_pipeline_last_stager[   r7  r  r8  )ri   rŽ  rN   r�  r’  Z
loss_dictsr“  Zlosses_reduced_for_keyrQ   r–  rR   Ú	eval_step#  s4   
ù


ÿzMegatronEngine.eval_stepc                 K   s¸  t ƒ }| jd jre| jd
i |¤Ž\}}}}|  jd7  _t ¡ |j tƒ  }| j	|7  _	|  j
t||ƒ7  _
|jd urd| j ¡  ¡ }d }	|jrMt| jƒ}	t|| j| jjd d | j|| j|||	|ƒ
| _n?| jd
i |¤Ž}|jd ur¤|D ]/}
| j |
tj dg¡¡||
  | j|
< | j |
d tj dg¡¡tj dg¡ | j|
d < qttjdtj ¡ d�}|D ]}
t||
 j ƒdkrÁ|||
 7 }q°d }d|v rÌ|d }| j!j"d urÚ| j!j"||d	�S |S )Nr   r
   Úlrg        Ú
_num_itersg      ð?rR  r  )r  r  rQ   )#r   rp  Útrainingr6   rl  r   rÀ   r¹   r   rÃ   rx  r4   rz  r^   Zget_loss_scaleÚitemZlog_params_normr9   rP   r7   ru  Úparam_groupsrw  r˜  rv  ri  r    r  ÚFloatTensorr¡   r  r[   r7  rt  rê   )ri   rŽ  rN   Z	loss_dictrÑ   r”  r•  r®   Z
loss_scaleZparams_normrs   r  r  rQ   rQ   rR   ÚforwardK  s\   

ö€
ÿÿþ€zMegatronEngine.forwardc                 C   s  t ƒ }|jd u s| jdkrd S t ƒ }tƒ }d| j› d�}| jD ]R}| d¡r'q| j| | j|d   }||› d|› d�7 }t td| 	¡ ƒ¡}|j
rT||› d|› d�7 }|rq| |› d�| 	¡ | j¡ |j
rq| |› d	�|| j¡ qt|ƒd
 }td| ƒ t|ƒ td| ƒ i | _d S )Nr   zvalidation loss at iteration z | rš  z value: é   z PPL: z validationz validation pplr
   ú-)r   rz  rl  r   rv  ÚendswithÚmathÚexpÚminrœ  rG   Ú
add_scalarr[   r    )ri   rN   ÚwriterÚstringrs   rt   ZpplÚlengthrQ   rQ   rR   r†  Š  s0   

€
zMegatronEngine.log_eval_resultsc                 C   sH   |   ¡  tƒ }||_tj ¡  t| j| j| j	| j
| jd� tj ¡  d S )N)rx  )r†  r   Úsaver    r£   Úbarrierr(   rl  rp  r^   rT   rx  )ri   Ú
output_dirrN   rQ   rQ   rR   r(   ¤  s   
ûzMegatronEngine.save_checkpointc                 C   st   t ƒ }||_d|_d|_tj ¡  t| j| j	| j
ƒ\}}tj ¡  || _|| _|jr6| jdkr8| j	 ¡  d S d S d S rÛ   )r   rj  rÃ   rÄ   r    r£   r«  r'   rp  r^   rT   rl  rx  Úfp16Zreload_model_params)ri   Ú	input_dirrN   rl  rx  rQ   rQ   rR   r'   ²  s   

ÿzMegatronEngine.load_checkpoint)r‘   r’   r“   r”   rl   r„  r…  rˆ  r�  r6   r˜  rŸ  r†  r(   r'   rÔ   rQ   rQ   rË   rR   ro  ž  s    
	(?ro  c                 C   s   t | ƒS )z”
    Average losses across data parallel group.

    Args:
        losses (List[Tensor]): List of losses to average across data parallel group.
    )r8   )rB  rQ   rQ   rR   Ú%avg_losses_across_data_parallel_groupÁ  s   r¯  c                 C   s   dd„ }t || dd�S )zò
    Recursively gather tensor in a nested list/tuple/dictionary of tensors from data parallel ranks.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to gather across data parallel ranks.

    c                    s^   ˆ j dkrˆ  ¡ d  ‰ ‡ fdd„ttjjt ¡ d�ƒD ƒ}tjj|ˆ t ¡ d� tj	|dd�S )Nr   c                    s   g | ]}t  ˆ ¡‘qS rQ   )r    Ú
empty_liker‹  ©r¡   rQ   rR   r~  Ù  s    ÿÿzOgather_across_data_parallel_groups.<locals>._gpu_gather_one.<locals>.<listcomp>rž   r4  )
Úndimra  rŠ   r    r£   Úget_world_sizer   Zget_data_parallel_groupÚ
all_gatherr<  )r¡   Úoutput_tensorsrQ   r±  rR   Ú_gpu_gather_oneÖ  s   

þz;gather_across_data_parallel_groups.<locals>._gpu_gather_oneT)Úerror_on_other_type)r   )r¡   r¶  rQ   rQ   rR   Ú"gather_across_data_parallel_groupsÌ  s   

r¸  )TTTT)NN)orc   r£  r@  Úabcr   Ú	functoolsr   r    Ztorch.nn.functionalÚnnÚ
functionalr  Útorch.nnr   r   r   r^   r   rT   r	   Úimportsr   Ú
operationsr   r   Zmegatron.corer   r   Zmegatron.core.distributedr   rƒ  r   Zmegatron.core.enumsr   Z)megatron.core.num_microbatches_calculatorr   Zmegatron.core.optimizerr   Zmegatron.core.parallel_stater   r   Zmegatron.core.pipeline_parallelr   Zmegatron.core.utilsr   Z"megatron.legacy.data.dataset_utilsr   Zmegatron.legacy.modelr   r   Z$megatron.legacy.model.classificationr   Zmegatron.trainingr   r   r   r    Úmegatron.training.argumentsr!   r"   r#   r$   r%   Zmegatron.training.checkpointingr&   r'   r(   Zmegatron.training.global_varsr)   Zmegatron.training.gpt_buildersr*   Zmegatron.training.initializer+   r,   r-   r.   r/   r0   Z%megatron.training.tokenizer.tokenizerr1   Zmegatron.training.trainingr2   r3   r4   r5   r6   r7   Zmegatron.training.utilsr8   r9   r:   rS   r`   ra   r¦   r¨   rÆ   rY   rØ   rÜ   rZ   rß   rä   r&  rH  rd  rn  ÚModulero  r¯  r¸  rQ   rQ   rQ   rR   Ú<module>   st     
1mB   
2  %