o
    pôj–—  ã                   @   s@  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZddlmZ ddlmZmZmZmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZmZ dd
l m!Z! ddl"m#Z# ddl$m%Z%m&Z& g fdd„Z'dd„ Z(de j)de*e+e+f fdd„Z,de j)de-e.e+ e*e+e+f f fdd„Z/de j)de*e+e+f fdd„Z0de j)de*de-e.e+ e*e+e+f f fdd„Z1de j)de-e.e+ e*e+e+f f fdd„Z2	d-de j)de*e+e+f d e3de-e j)e*e+e+f f fd!d"„Z4d#e.e+ de*e+e+f fd$d%„Z5d&ede j)de-e j)e*e+e
f f fd'd(„Z6d)d*„ Z7G d+d,„ d,ƒZ8dS ).é    N©Úliteral_eval©Úwhich)ÚAnyé   )ÚSageMakerConfig)ÚDynamoBackendÚPrecisionTypeÚis_fp8_availableÚis_hpu_availableÚis_mlu_availableÚis_musa_availableÚis_neuron_availableÚis_npu_availableÚis_sdaa_availableÚis_torch_xla_availableÚis_xpu_available)ÚDEEPSPEED_MULTINODE_LAUNCHERS)Úget_free_portÚis_port_in_useÚmerge_dicts)Úcompare_versionsé   )Ústr_to_bool)ÚDistributedTypeÚSageMakerDistributedTypec                 C   sD   |  |¡\}}t| ƒ ¡ D ]\}}|t|ƒ ¡ v rt|||ƒ q|S )z4
    Filters out all `accelerate` specific args
    )Úparse_known_argsÚvarsÚitemsÚkeysÚsetattr)ÚargsÚparserZdefault_argsÚnew_argsÚ_ÚkeyÚvalue© r(   úh/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/utils/launch.pyÚ_filter_args/   s   €r*   c                  C   s\   dd„ dD ƒ} t | ƒdkrtdƒ‚| d }t |dg¡}d|v r'|dd	d
dfS |dd	ddfS )a  
    Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
    are: OpenMPI, Intel MPI, or MVAPICH.

    Returns: Program name and arg names for hostfile, num processes, and processes per node
    c                 S   s   g | ]}t |ƒr|‘qS r(   r   )Ú.0Úxr(   r(   r)   Ú
<listcomp>B   s    z$_get_mpirun_args.<locals>.<listcomp>)ZmpirunZmpiexecr   z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPIú
--hostfilez-nz
--npernodez	--bind-toz-fz-ppnÚ )ÚlenÚOSErrorÚ
subprocessÚcheck_output)Zmpi_appsZmpi_appZmpirun_versionr(   r(   r)   Ú_get_mpirun_args:   s   r4   r"   Úcurrent_envc                 C   s–   d}t | ƒD ]B}| d¡rHt| |ƒ}|durH|dkr9t|d ƒ||d < t|d ƒ||d < t|d	 ƒ||d
 < qtt| |ƒƒ||› | ¡ › �< q|S )z.
    Setup the FP8 environment variables.
    ÚACCELERATE_Zfp8_NZfp8_override_linear_precisionr   ZFP8_OVERRIDE_FPROPr   ZFP8_OVERRIDE_DGRADr   ZFP8_OVERRIDE_WGRAD)r   Ú
startswithÚgetattrÚstrÚupper)r"   r5   ÚprefixÚargr'   r(   r(   r)   Úsetup_fp8_envR   s   

€r=   Úreturnc              	   C   sb  g }| j r| jrtdƒ‚t| ddƒ}| j}| jdurPtƒ \}}}}}t| ddƒ}	|r2|r2t|| ƒnd}
|||| j||
g7 }|rH||t|ƒg7 }|rP|||	g7 }| j sa| t	j
¡ | jra| d¡ | | j¡ | | j¡ tj ¡ }t| jpx| jƒ|d< | jrƒd	|d
< | jdkrÑ| jdurÑtƒ r–| j|d< n;tƒ rŸ| j|d< n2tƒ r¨| j|d< n)tƒ r±| j|d< n tƒ rº| j|d< ntƒ rÃ| j|d< ntƒ rÌ| j|d< n| j|d< |dkrç| jdusÞJ dƒ‚| jdusçJ dƒ‚|durï|dksô|dk�r| jdurü| jnd|d< | jdu�rt| jƒnd|d< t|d ƒ�r d|d< tdƒ|d< z	t| j  !¡ ƒ}W n t�yA   td| j  !¡ › dt "¡ › d �ƒ‚w t|ƒ|d!< | j  !¡ d"k�r]t#ƒ �sXt$d#ƒ‚t%| |ƒ}z	t&| j' (¡ ƒ}W n t�y~   td$| j' (¡ › dt& "¡ › d �ƒ‚w |j)|d%< | j*|d&< t| j+ƒ|d'< t| j,ƒ|d(< t| j-ƒ|d)< t| j.ƒ|d*< | j/�r­d|d+< ||fS ),zz
    Prepares and returns the command list and an environment with the correct simple launcher environment variables.
    ú0--module and --no_python cannot be used togetherÚnum_processesNzbind-toÚsocketÚ1z-mÚACCELERATE_USE_CPUÚtrueÚACCELERATE_DEBUG_MODEÚallÚZE_AFFINITY_MASKÚMLU_VISIBLE_DEVICESÚSDAA_VISIBLE_DEVICESÚMUSA_VISIBLE_DEVICESÚASCEND_RT_VISIBLE_DEVICESÚHABANA_VISIBLE_MODULESÚNEURON_RT_VISIBLE_CORESÚCUDA_VISIBLE_DEVICESr   zFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.z	127.0.0.1ÚMASTER_ADDRÚ29500ÚMASTER_PORTzgranularity=fine,compact,1,0ZKMP_AFFINITYZKMP_BLOCKTIMEúUnknown mixed_precision mode: ú. Choose between Ú.ÚACCELERATE_MIXED_PRECISIONÚfp8úrFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.úUnknown dynamo backend: ÚACCELERATE_DYNAMO_BACKENDÚACCELERATE_DYNAMO_MODEÚACCELERATE_DYNAMO_USE_FULLGRAPHÚACCELERATE_DYNAMO_USE_DYNAMICÚ*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONÚOMP_NUM_THREADSÚACCELERATE_CPU_AFFINITY)0Ú	no_pythonÚmoduleÚ
ValueErrorr8   Únum_machinesÚmpirun_hostfiler4   r9   ÚappendÚsysÚ
executableÚtraining_scriptÚextendÚtraining_script_argsÚosÚenvironÚcopyÚcpuÚuse_cpuÚdebugÚgpu_idsr   r   r   r   r   r   r   Úmain_process_ipÚmain_process_portr   r
   Úmixed_precisionÚlowerÚlistr   ÚRuntimeErrorr=   r	   Údynamo_backendr:   r'   Údynamo_modeÚdynamo_use_fullgraphÚdynamo_use_dynamicÚdynamo_use_regional_compilationÚnum_cpu_threads_per_processÚenable_cpu_affinity)r"   Úcmdr@   rc   Zmpi_app_nameZhostfile_argZnum_proc_argZproc_per_node_argZbind_to_argZbind_toÚnproc_per_noder5   rt   rx   r(   r(   r)   Úprepare_simple_launcher_cmd_envd   s²   
û


ÿÿÿÿÿ
ÿÿ

r�   c              	   C   sž  | j dkr
tƒ | _ n| j du rd| _ | j}| j}| j}| j }|dkrNt|| ƒ| _t|ƒ| _t| j	ƒ| _
t| ddƒrEt|ƒ| _t|ƒ| _n|› d|› �| _nt|ƒ| _|dur\t|ƒ| _|dkpft| j	ƒdk}|r†t|ƒr†|dkr~d| _t d	|› d
�¡ ntd|› d�ƒ‚| jr�| jr�tdƒ‚| jr—d| _n| jr�d| _tj ¡ }| jr©d|d< t| ddƒ}|dkrô| jdurôtƒ rÀ||d< n4tƒ rÈ||d< n,tƒ rÐ||d< n$tƒ rØ||d< nt ƒ rà||d< nt!ƒ rè||d< nt"ƒ rð||d< n||d< | j# $¡ }zt%|ƒ}W n t�y   td|› dt% &¡ › d�ƒ‚w t|ƒ|d< | j# $¡ dk�r0t'ƒ �s+t(dƒ‚t)| |ƒ}z	t*| j+ ,¡ ƒ}	W n t�yQ   td | j+ ,¡ › dt* &¡ › d�ƒ‚w |	j-|d!< | j.|d"< t| j/ƒ|d#< t| j0ƒ|d$< t| j1ƒ|d%< | j2�r.d|d&< | j3�r…| j4�s…td'ƒ‚t5| d(ƒ�r�t| j6ƒnd)|d*< t| j7ƒ|d+< t| j8ƒ $¡ |d,< t| j9ƒ $¡ |d-< t| j:ƒ|d.< | j;du�rÁt| j;ƒ|d/< | j<du�rÎt| j<ƒ|d0< | j=du�rÛt| j=ƒ|d1< | j>du�rèt| j>ƒ|d2< t| j?ƒ $¡ |d3< t| j@ƒ $¡ |d4< t| j3ƒ $¡ |d5< t| j4ƒ $¡ |d6< t| jAƒ $¡ |d7< t| jBƒ $¡ |d8< t| d9dƒdu�r.t| jCƒ|d:< | jD�rµd;}
d|d<< t| jEƒ||
d= < t| jFƒ||
d> < | jGdu�rYt| jGƒ||
d? < | jHdu�rht| jHƒ||
d@ < | jIdu�rwt| jIƒ||
dA < | jJdu�r†t| jJƒ||
dB < | jKdu�r•t| jKƒ||
dC < | jLdu�r¤t| jLƒ||
dD < | jMdu�r³t| jMƒ||
dE < t| jNƒ||
dF < t| jOƒ||
dG < | jPdu�rÔt| jPƒ||
dH < | jQdu�rãt| jQƒ||
dI < | jRdu�ròt| jRƒ||
dJ < | jSdu�rt| jSƒ||
dK < | jTdu�rt| jTƒ||
dL < | jUdu�rt| jUƒ||
dM < | jVdu�r.t| jVƒ||
dN < | jWdu�r=t| jWƒ||
dO < | jXdu�rLt| jXƒ||
dP < | jYdu�r[t| jYƒ||
dQ < | jZdu�rjt| jZƒ||
dR < | j[du�ryt| j[ƒ||
dS < | j\du�rˆt| j\ƒ||
dT < | j]du�r—t| j]ƒ||
dU < | j^du�r¦t| j^ƒ||
dV < | j_du�rµt| j_ƒ||
dW < t| j`ƒ|dX< | ja�rÄd)|dY< | jb�rÍtc| |ƒ}|S )Zz_
    Prepares and returns an environment with the correct multi-GPU environment variables.
    r   Né<s  r   Úsame_networkFú:TúPort `á…  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.ú3Tried to launch distributed communication on port `á$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.r?   rD   rE   rq   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   ÚACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`Úfsdp_versionrB   ZFSDP_VERSIONÚFSDP_SHARDING_STRATEGYZFSDP_RESHARD_AFTER_FORWARDÚFSDP_OFFLOAD_PARAMSZFSDP_MIN_NUM_PARAMSÚFSDP_AUTO_WRAP_POLICYZFSDP_TRANSFORMER_CLS_TO_WRAPÚFSDP_BACKWARD_PREFETCHÚFSDP_STATE_DICT_TYPEZFSDP_FORWARD_PREFETCHZFSDP_USE_ORIG_PARAMSÚFSDP_CPU_RAM_EFFICIENT_LOADINGZFSDP_SYNC_MODULE_STATESZFSDP_ACTIVATION_CHECKPOINTINGZ%FSDP_ACTIVATION_CHECKPOINTING_OFFLOADÚfsdp_ignored_modulesZFSDP_IGNORED_MODULESÚMEGATRON_LM_ÚACCELERATE_USE_MEGATRON_LMÚ	TP_DEGREEÚUSE_CUSTOM_FSDPÚNO_LOAD_OPTIMÚEOD_MASK_LOSSÚNO_SAVE_OPTIMÚOPTIMIZER_CPU_OFFLOADÚUSE_PRECISION_AWARE_OPTIMIZERÚOVERLAP_CPU_OPTIMIZER_D2H_H2DÚ DECODER_LAST_PIPELINE_NUM_LAYERSÚ	PP_DEGREEÚGRADIENT_CLIPPINGÚNUM_MICRO_BATCHESÚSEQUENCE_PARALLELISMÚRECOMPUTE_ACTIVATIONSÚUSE_DISTRIBUTED_OPTIMIZERÚRECOMPUTE_GRANULARITYÚRECOMPUTE_METHODÚRECOMPUTE_NUM_LAYERSÚATTENTION_BACKENDÚEXPERT_MODEL_PARALLEL_SIZEÚCONTEXT_PARALLEL_SIZEÚATTENTION_DROPOUTÚHIDDEN_DROPOUTÚATTENTION_SOFTMAX_IN_FP32ÚEXPERT_TENSOR_PARALLEL_SIZEÚCALCULATE_PER_TOKEN_LOSSÚUSE_ROTARY_POSITION_EMBEDDINGSr^   r_   )drs   r   r@   rc   rr   r9   r€   ÚnnodesÚintÚmachine_rankÚ	node_rankr8   Úmaster_addrÚmaster_portÚrdzv_endpointr   Ú
standaloneÚwarningsÚwarnÚConnectionErrorra   r`   rb   rk   rl   rm   rp   rq   r   r   r   r   r   r   r   rt   ru   r
   rv   r   rw   r=   r	   rx   r:   r'   ry   rz   r{   r|   Úuse_fsdpÚfsdp_cpu_ram_efficient_loadingÚfsdp_sync_module_statesÚhasattrrŠ   Zfsdp_sharding_strategyÚfsdp_reshard_after_forwardÚfsdp_offload_paramsÚfsdp_min_num_paramsÚfsdp_auto_wrap_policyÚ"fsdp_transformer_layer_cls_to_wrapÚfsdp_backward_prefetchÚfsdp_state_dict_typeÚfsdp_forward_prefetchÚfsdp_use_orig_paramsÚfsdp_activation_checkpointingZ%fsdp_activation_checkpointing_offloadr‘   Úuse_megatron_lmZmegatron_lm_tp_degreeZmegatron_lm_use_custom_fsdpZmegatron_lm_no_load_optimZmegatron_lm_eod_mask_lossZmegatron_lm_no_save_optimZ!megatron_lm_optimizer_cpu_offloadZ)megatron_lm_use_precision_aware_optimizerZ)megatron_lm_overlap_cpu_optimizer_d2h_h2dZ,megatron_lm_decoder_last_pipeline_num_layersZmegatron_lm_pp_degreeZmegatron_lm_gradient_clippingZmegatron_lm_num_micro_batchesZ megatron_lm_sequence_parallelismZ!megatron_lm_recompute_activationsZ%megatron_lm_use_distributed_optimizerZ!megatron_lm_recompute_granularityZmegatron_lm_recompute_methodZ megatron_lm_recompute_num_layersZmegatron_lm_attention_backendZ&megatron_lm_expert_model_parallel_sizeZ!megatron_lm_context_parallel_sizeZmegatron_lm_attention_dropoutZmegatron_lm_hidden_dropoutZ%megatron_lm_attention_softmax_in_fp32Z'megatron_lm_expert_tensor_parallel_sizeZ$megatron_lm_calculate_per_token_lossZ*megatron_lm_use_rotary_position_embeddingsr}   r~   Úuse_parallelism_configÚ%prepare_extend_env_parallelism_config)r"   r@   rc   rr   rs   Úneed_port_checkr5   rq   rt   rx   r;   r(   r(   r)   Úprepare_multi_gpu_envÉ   sP  







ÿ
ÿ








ÿÿ
ÿÿ

ÿÿ
rÌ   c                 C   sê   d}d|d< t | jƒ||d < t | jƒ||d < t | jƒ||d < t | jƒ||d < t | jƒ||d < t | jƒ||d	 < t | jƒ||d
 < | jdkrSt | jƒ||d < | jdkrst | j	ƒ||d < t | j
ƒ||d < t | jƒ||d < |S )zV
    Extends `current_env` with context parallelism env vars if any have been set
    ZPARALLELISM_CONFIG_rD   Ú!ACCELERATE_USE_PARALLELISM_CONFIGZDP_REPLICATE_SIZEZDP_SHARD_SIZEZTP_SIZEZCP_SIZEZ
CP_BACKENDZSP_SIZEZ
SP_BACKENDr   ZCP_COMM_STRATEGYZSP_SEQ_LENGTHZSP_SEQ_LENGTH_IS_VARIABLEZSP_ATTN_IMPLEMENTATION)r9   Z$parallelism_config_dp_replicate_sizeZ parallelism_config_dp_shard_sizeZparallelism_config_tp_sizeZparallelism_config_cp_sizeZparallelism_config_cp_backendZparallelism_config_sp_sizeZparallelism_config_sp_backendZ#parallelism_config_cp_comm_strategyZ parallelism_config_sp_seq_lengthZ,parallelism_config_sp_seq_length_is_variableZ)parallelism_config_sp_attn_implementation)r"   r5   r;   r(   r(   r)   rÊ   “  s    

rÊ   c           
   	   C   sZ  | j dkr
tƒ | _ n| j du rd| _ | j}| j}| j}| j }d}| jdu r*td | _|dkrÑ| jtd krÑdg}| dt| j	ƒg¡ | jdkr]t
ddd	ƒrQtd
ƒ‚| dt| jƒdg¡ n| ddt| jƒg¡ | jdurx| dt| jƒg¡ n| jdurˆ| dt| jƒg¡ n| dt| j| j ƒg¡ |r | dt|ƒg¡ | dt|ƒg¡ | jr³| jr³tdƒ‚| jr¼| d¡ n| jrÄ| d¡ | | j¡ | | j¡ nI|dk�r| jtd k�rt|| ƒ| _t|ƒ| _t| jƒ| _t| ddƒ�rt|ƒ| _t|ƒ| _n|› d|› �| _nt|ƒ| _|du�rt|ƒ| _|dk�p%t| jƒdk}|�rHt|ƒ�rH|dk�r@d| _t d|› d�¡ nt d|› d�ƒ‚| j�rT| j�rTtdƒ‚| j�r\d| _n| j�rcd| _t!j" #¡ }| j$�rpd|d < t| d!d"ƒ}|d"k�rÄ| j%du�rÄt&ƒ �rŠ||d#< n:t'ƒ �r“||d$< n1t(ƒ �rœ||d%< n(t)ƒ �r¥||d&< nt*ƒ �r®||d'< nt+ƒ �r·||d(< nt,ƒ �rÀ||d)< n||d*< z	t-| j. /¡ ƒ}	W n t�yå   td+| j. /¡ › d,t- 0¡ › d-�ƒ‚w t1d.t!j2 3d-¡ƒ|d.< t|	ƒ|d/< | j. /¡ d0k�rt4ƒ �st5d1ƒ‚t6| |ƒ}t| j7ƒ /¡ |d2< d|d3< | j8du�r&t| j8ƒ|d4< | j9du�r3t| j9ƒ|d5< | j:du�rBt| j:ƒ /¡ |d6< | j;du�rQt| j;ƒ /¡ |d7< | j<du�r`t| j<ƒ /¡ |d8< | j=du�rot| j=ƒ /¡ |d9< | j>du�r~t| j>ƒ /¡ |d:< | j?du�r‹t| j?ƒ|d;< | j@�r“d<|d=< | jAdu�r t| jAƒ|d>< | jB�r©tC| |ƒ}||fS )?zt
    Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
    r   Nr‚   r   Ú	deepspeedr.   Únosshú<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portr?   z--modulez--no_pythonrƒ   Fr„   Tr…   r†   r‡   rˆ   rD   rE   rq   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rR   rS   rT   Ú
PYTHONPATHrU   rV   rW   ÚACCELERATE_CONFIG_DS_FIELDSÚACCELERATE_USE_DEEPSPEEDÚACCELERATE_DEEPSPEED_ZERO_STAGEÚ&ACCELERATE_GRADIENT_ACCUMULATION_STEPSÚACCELERATE_GRADIENT_CLIPPINGÚ-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICEÚ)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEÚACCELERATE_DEEPSPEED_ZERO3_INITÚ+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODELÚ ACCELERATE_DEEPSPEED_CONFIG_FILErB   r_   Ú(ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Drs   r   r@   rc   rr   Údeepspeed_multinode_launcherr   ri   r9   Údeepspeed_hostfiler   rb   r±   Údeepspeed_exclusion_filterÚdeepspeed_inclusion_filterra   r`   re   rh   rj   r€   r¯   r°   r²   r8   r³   r´   rµ   r   r¶   r·   r¸   r¹   rk   rl   rm   rp   rq   r   r   r   r   r   r   r   r
   rt   ru   rv   Úenv_var_path_addÚpathÚabspathr   rw   r=   Ú'deepspeed_fields_from_accelerate_configÚ
zero_stageÚgradient_accumulation_stepsÚgradient_clippingÚoffload_optimizer_deviceÚoffload_param_deviceÚzero3_init_flagÚzero3_save_16bit_modelÚdeepspeed_config_filer~   Údeepspeed_moe_layer_cls_namesrÉ   rÊ   )
r"   r@   rc   rr   rs   r   rË   r5   rq   rt   r(   r(   r)   Úprepare_deepspeed_cmd_env®  s  






þÿ
þÿ







ÿ
ÿ







ÿÿÿ

rî   FÚpodc                 C   sV   | j dkrtdd�r| jrd|d< nd|d< | jrd|d< |r'| j| _| j| _| |fS )	zY
    Prepares and returns an environment with the correct TPU environment variables.
    Úbf16T)Úcheck_is_tpurB   ÚXLA_DOWNCAST_BF16ÚXLA_USE_BF16rD   rE   )rt   r   Údowncast_bf16rp   Útpu_vmÚvmÚtpu_nameÚtpu)r"   r5   rï   r(   r(   r)   Úprepare_tpuR  s   
rù   Únargsc                 C   sÂ   t | ƒdk ri S dd„ }t ¡ }| | ¡\}}t|ƒD ]8\}}| d¡rSd }|d t |ƒk r<||d   d¡r;tdƒ‚ntdƒ‚|d u rL|j||d� q|j||d� qd	d
„ | | ¡j	 
¡ D ƒS )Nr   c                 S   s<   zt | ƒ} | d | krt| ƒW S | W S  ty   |  Y S w )Nr   )Úfloatr°   rb   )Úsr(   r(   r)   Ú_infer_typek  s   
ÿz+_convert_nargs_to_dict.<locals>._infer_type)ú-z--r   ul   SageMaker doesnâ€™t support argparse actions for `store_true` or `store_false`. Please define explicit types)Útype)Úactionc                 S   s&   i | ]\}}||d v rt |ƒn|“qS ))ÚTrueÚFalser   )r+   r&   r'   r(   r(   r)   Ú
<dictcomp>Š  s    ÿÿz*_convert_nargs_to_dict.<locals>.<dictcomp>)r0   ÚargparseÚArgumentParserr   Ú	enumerater7   rb   Úadd_argumentÚ
parse_argsÚ__dict__r   )rú   rý   r#   r%   ÚunknownÚindexÚargumentr   r(   r(   r)   Ú_convert_nargs_to_dictf  s0   

ÿþÿ€þr  Úsagemaker_configc                 C   sR  t dƒ | jtjd< | jd ur| jtjd< n|jd ur-|jd ur-|jtjd< |jtjd< ntdƒ‚tj 	|j
¡}|s<d}tj |j
¡}| d¡sPtd	|› d
�ƒ‚t dƒ t|jƒ}z	t|j ¡ ƒ}W n tyy   td|j ¡ › dt ¡ › d�ƒ‚w z	t|j ¡ ƒ}W n tyš   td|j ¡ › dt ¡ › d�ƒ‚w dt|ƒ|j|jt|jƒt|jƒt|jƒ| jjdœ}|j ¡ dkrÈtƒ sÃtdƒ‚t ||ƒ}d }| jt!j"krØddddiii}d }	| j#d u�r%t d| j#› d�ƒ i }	t$| j#ƒ�%}
t%|
ƒD ]\}}|dkrþqõ| &d¡}|d  '¡ |	|d < qõW d   ƒ n	1 �sw   Y  t d|	› �ƒ d }| j(d u�r{t d| j(› d�ƒ g }t$| j(ƒ�.}
t%|
ƒD ]!\}}|dk�rM�qB| &d¡}|d |d  '¡ dœ}| )|¡ �qBW d   ƒ n	1 �sow   Y  t d|› �ƒ t d ƒ | j*||| j+| j,| j-| j.| j/| j0| j1d!||||d"œ}| j2d u�r¥t3| j2|ƒ}||	fS )#Nz(Configuring Amazon SageMaker environmentÚAWS_DEFAULT_REGIONÚAWS_PROFILEÚAWS_ACCESS_KEY_IDÚAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerT   z.pyz8Your training script should be a python script and not "ú"z'Converting Arguments to HyperparametersrR   rS   rX   rD   )ÚACCELERATE_USE_SAGEMAKERrU   rY   rZ   r[   r\   r]   Ú%ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPErV   rW   ÚsmdistributedZdataparallelÚenabledTzLoading SageMaker Inputs from z filer   ú	r   zLoaded SageMaker Inputs: zLoading SageMaker Metrics from )ÚNameÚRegexzLoaded SageMaker Metrics: zCreating EstimatorF)Ú	image_uriÚentry_pointÚ
source_dirÚroleÚtransformers_versionÚpytorch_versionÚ
py_versionÚbase_job_nameZinstance_countÚinstance_typeZdebugger_hook_configÚdistributionÚhyperparametersÚenvironmentZmetric_definitions)4ÚprintÚregionrk   rl   ÚprofileÚaws_access_key_idÚaws_secret_access_keyr1   râ   Údirnamerh   ÚbasenameÚendswithrb   r  rj   r
   rt   ru   rv   r	   rx   r:   r9   r'   ry   rz   r{   r|   Údistributed_typer   rw   r=   r   ÚDATA_PARALLELÚsagemaker_inputs_fileÚopenr  ÚsplitÚstripÚsagemaker_metrics_filere   r  Úiam_role_namer  r   r!  r"  rc   Úec2_instance_typeÚadditional_argsr   )r  r"   r  r  r%  rt   rx   r&  r$  Zsagemaker_inputsÚfileÚiÚlineÚlZsagemaker_metricsÚmetric_dictr(   r(   r)   Úprepare_sagemager_args_inputs�  sÂ   


ÿÿÿÿø
ÿ

üÿ


þøÿ
ñr>  c                 C   s6   dd„ t j | d¡ d¡D ƒ}| t|ƒ¡ d |¡S )z�
    Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
    caller to set it in os.environ.
    c                 S   s   g | ]
}t |ƒd kr|‘qS )r   )r0   )r+   Úpr(   r(   r)   r-     s    z$env_var_path_add.<locals>.<listcomp>r/   r„   )rk   rl   Úgetr3  re   r9   Újoin)Zenv_var_nameZpath_to_addÚpathsr(   r(   r)   rá     s   
rá   c                   @   s"   e Zd ZdZd	dd„Zdd„ ZdS )
ÚPrepareForLaunchai  
    Prepare a function that will launched in a distributed setup.

    Args:
        launcher (`Callable`):
            The function to launch.
        distributed_type ([`~state.DistributedType`]):
            The distributed type to prepare for.
        debug (`bool`, *optional*, defaults to `False`):
            Whether or not this is a debug launch.
    ÚNOFc                 C   s   || _ t|ƒ| _|| _d S )N)Úlauncherr   r/  rp   )ÚselfrE  r/  rp   r(   r(   r)   Ú__init__  s   

zPrepareForLaunch.__init__c                 G   sÌ   | j r!ttj d¡ƒ}tj d¡}tjjd|tj ||¡|d� n7| j	t
jt
jt
jt
jt
jt
jt
jfv rXt|ƒtjd< ttj dd¡ƒ}ttj dd	¡ƒ}t|| | ƒtjd
< tdƒtjd< | j|Ž  d S )NÚ
WORLD_SIZEZACCELERATE_DEBUG_RDV_FILEÚgloo)ÚrankÚstoreÚ
world_sizeÚ
LOCAL_RANKZNPROCr   Z	NODE_RANKr   ÚRANKÚFORK_LAUNCHED)rp   r°   rk   rl   r@  ÚtorchÚdistributedÚinit_process_groupÚ	FileStorer/  r   Ú	MULTI_GPUÚ	MULTI_MLUÚ
MULTI_MUSAÚ	MULTI_NPUÚ	MULTI_XPUÚ	MULTI_CPUÚMULTI_NEURONr9   rE  )rF  r  r"   rL  Zrdv_fileZnprocr²   r(   r(   r)   Ú__call__"  s0   üù
zPrepareForLaunch.__call__N)rD  F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rG  r[  r(   r(   r(   r)   rC    s    
rC  )F)9r  rk   r2   rf   r·   Úastr   Úshutilr   Útypingr   rP  Zcommands.config.config_argsr   Úutilsr	   r
   r   r   r   r   r   r   r   r   r   Úutils.constantsr   Úutils.otherr   r   r   Zutils.versionsr   r/   r   Údataclassesr   r   r*   r4   Ú	NamespaceÚdictr9   r=   Útuplerv   r�   rÌ   rÊ   rî   Úboolrù   r  r>  rá   rC  r(   r(   r(   r)   Ú<module>   sf   4(e Kÿÿ
þ( &ÿÿ
ÿÿ
þ*ÿÿ
þv
