o
    pôj¥Â  ã                	   @   s|  d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZmZ d dlZdd	lmZ d
dlmZmZmZmZ d
dlmZ d
dlmZmZ d
dlm Z m!Z!m"Z" d
dl#m$Z$ ee%ƒZ&dd„ Z'dd„ Z(dJdd„Z)dJdd„Z*dd„ Z+dKdd„Z,dKdd„Z-dLd d!„Z.	dKd"d#„Z/dMd$e0d%e0d&e1fd'd(„Z2	dNd$e0d)e0d&e1d*e1fd+d,„Z3d-ej4j5d.ej6fd/d0„Z7dOd-ej4j5d1e8d2e1fd3d4„Z9d5ej:j;d6e8fd7d8„Z<d-ej4j5fd9d:„Z=d-ej4j5d;ej4j5dB fd<d=„Z>d-ej4j5d;ej4j5fd>d?„Z?d-ej4j5d;eej4j5ge1f dB fd@dA„Z@dBdC„ ZAdDe8d;e8fdEdF„ZBdGee	ej4j5 e0f d;eCej4jD fdHdI„ZEdS )Pé    N)Údefaultdict)ÚIterable)Únullcontext©Úreplace)ÚPath)ÚCallableÚUnioné   )Ú
get_loggeré   )ÚFSDP_MODEL_NAMEÚOPTIMIZER_NAMEÚSAFE_WEIGHTS_NAMEÚWEIGHTS_NAME)Úget_module_class_from_name)Úget_non_persistent_buffersÚis_peft_model)Úget_module_children_bottom_upÚis_compiled_moduleÚsave)Úis_torch_versionc                   C   s"   dt jvr
dt jd< dt jd< dS )z[
    Enables RAM efficient loading of Hugging Face models for FSDP in the environment.
    ÚACCELERATE_USE_FSDPÚTrueÚFSDP_CPU_RAM_EFFICIENT_LOADINGN©ÚosÚenviron© r   r   úl/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/utils/fsdp_utils.pyÚ!enable_fsdp_ram_efficient_loading(   s   

r    c                   C   s   dt jd< dS )z\
    Disables RAM efficient loading of Hugging Face models for FSDP in the environment.
    ÚFalser   Nr   r   r   r   r   Ú"disable_fsdp_ram_efficient_loading2   s   r"   Fc                 C   s|   |r*t | ƒr*|d ur|jrddlm} || t|dd�d�S ddlm} || | jd�S |d ur:ddlm} || |d�S |  ¡ S )Nr   )Úget_model_state_dictT)Zignore_frozen_params©Úoptions)Úget_peft_model_state_dict©Zadapter_name)	r   Úfull_state_dictÚ'torch.distributed.checkpoint.state_dictr#   r   Úpeftr&   Úactive_adapterÚ
state_dict)ÚmodelÚadapter_onlyÚ
sd_optionsr#   r&   r   r   r   Ú_get_model_state_dict9   s   r0   c                 C   s„   |r,t | ƒr,|d ur|jrddlm} || |t|dd�d�S ddlm} || || jd�S |d ur=ddlm} || ||d�S |  |¡S )Nr   )Úset_model_state_dictF)Ústrictr$   )Úset_peft_model_state_dictr'   )	r   r(   r)   r1   r   r*   r3   r+   Úload_state_dict)r-   r,   r.   r/   r1   r3   r   r   r   Ú_set_model_state_dictM   s   
r5   c                 C   sT   d }| j dkr(ddlm} ddlm} || j|jkt| jddƒt| jddƒd�}|S )	Nr
   r   )ÚStateDictOptions©ÚStateDictTypeÚoffload_to_cpuFÚ
rank0_only)r(   Úcpu_offloadÚbroadcast_from_rank0)	Úfsdp_versionr)   r6   Ú2torch.distributed.fsdp.fully_sharded_data_parallelr8   Ústate_dict_typeÚFULL_STATE_DICTÚgetattrÚstate_dict_config)Úfsdp_pluginr/   r6   r8   r   r   r   Ú_prepare_sd_optionsa   s   

ýrD   Tc                 C   sÌ  ddl m} ddl m} tj|dd� | j|jkr&|jdk}	|	| j_	|	| j_
| jdkr6| || j| j| j¡ntƒ }
t| ƒ}|
�� t|||d�}| j|jkr‚|dkrWt› d�nt› d	|› d�}tj ||¡}|jdkr�t d
|› �¡ t ||¡ t d|› �¡ n¹| j|jkr¿|dkr•t› d|j› d�nt› d	|› d|j› d�}tj ||¡}t d
|› �¡ t ||¡ t d|› �¡ nŒ| j|jk�rC|�r
ddlm  m} ddlm} tj |t› d	|› �¡}tj|dd� t d
|› �¡ |jd|i| |¡|ƒ d� t d|› �¡ nIt› d	|› d|j› d�}tj ||¡}t d
|› �¡ t ||¡ t d|› �¡ W d  ƒ dS W d  ƒ dS W d  ƒ dS W d  ƒ dS W d  ƒ dS 1 �s_w   Y  dS )aŠ  
    Save an FSDP model checkpoint.

    When ``state_dict_type`` is ``SHARDED_STATE_DICT``, the ``use_dcp`` parameter controls the saving strategy:

    - ``use_dcp=True`` (default): Uses ``torch.distributed.checkpoint`` (DCP) to save sharded model state.
      This is the standard approach and works well for typical training setups.
    - ``use_dcp=False``: Uses per-rank ``torch.save`` instead of DCP. Each rank saves its own shard
      independently to a separate file, avoiding cross-rank communication entirely. This is significantly
      faster at large scale (e.g., 2800+ GPUs) where DCP's all-gather-based saving can cause timeouts
      due to TCP congestion from Gloo-based collective communication.

    .. note::
        Checkpoints saved with ``use_dcp=False`` are saved as per-rank files
        (``{prefix}_rank{local_rank}.bin``) and must be loaded with ``use_dcp=False`` as well.
    r   ©ÚFullyShardedDataParallelr7   T©Úexist_okr   ©r.   r/   ú.binÚ_zSaving model to zModel saved to Ú_rankN©ÚDefaultSavePlannerr-   ©r,   Zstorage_writerÚplanner) r>   rF   r8   r   Úmakedirsr?   r@   Únum_processesrB   r9   r:   r=   Úoptim_state_dict_configr   rD   r0   r   ÚpathÚjoinÚprocess_indexÚloggerÚinfoÚtorchr   ÚLOCAL_STATE_DICTÚSHARDED_STATE_DICTÚtorch.distributed.checkpointÚdistributedÚ
checkpointÚ,torch.distributed.checkpoint.default_plannerrN   ÚFileSystemWriter)rC   Úacceleratorr-   Ú
output_dirÚmodel_indexr.   Úuse_dcpÚFSDPr8   Úis_multi_processÚctxr/   r,   Úweights_nameZoutput_model_fileÚdist_cprN   Úckpt_dirr   r   r   Úsave_fsdp_modelr   sx   

ýÿû"
€ÿýýÙ€ ìí!$ßrk   c                 C   s  ddl m} ddl m} | ¡  | j|jkr#|jdk}	|	| j_|	| j_	| j
dkr3| || j| j| j¡ntƒ }
t| ƒ}|
��< | j|jkr£t|ƒ|urg|jdkrg|jsg| js^| j
dkr^tdƒ‚	 W d  ƒ dS |dkrpt› d�nt› d|› d�}tj ||¡}t d	|› �¡ |j p�|j}|r˜tj|d
d�}ni }t d|› �¡ nÇ| j|jkrá|dkr¶t› d|j› d�nt› d|› d|j› d�}tj ||¡}t d	|› �¡ tj|d
d�}t d|› �¡ n‰| j|jk�rj|�r@ddlm  m } ddl!m"} t#|ƒj$t› d|› �k�rt#|ƒt› d|› � nt#|ƒ}t d	|› �¡ dt%|||d�i}|j|| &|¡|ƒ d� |d }t d|› �¡ n*t› d|› d|j› d�}tj ||¡}t d	|› �¡ tj|d
d�}t d|› �¡ t'||||d�}W d  ƒ n	1 �s}w   Y  | ¡  |S )a¥  
    Load an FSDP model checkpoint.

    When ``state_dict_type`` is ``SHARDED_STATE_DICT``, the ``use_dcp`` parameter controls the loading strategy:

    - ``use_dcp=True`` (default): Uses ``torch.distributed.checkpoint`` (DCP) to load sharded model state.
      This is the standard approach and works well for typical training setups.
    - ``use_dcp=False``: Uses per-rank ``torch.load`` instead of DCP. Each rank loads its own shard
      independently from a separate file, avoiding cross-rank communication entirely. This is significantly
      faster at large scale (e.g., 2800+ GPUs) where DCP's broadcast-based loading can cause timeouts
      due to TCP congestion from Gloo-based collective communication.

    .. note::
        Checkpoints saved with ``use_dcp=False`` must be loaded with ``use_dcp=False`` as well, since the
        file format differs (per-rank ``.bin`` files vs. DCP's sharded directory format).
    r   rE   r7   r   zzSet the `sync_module_states` flag to `True` so that model states are synced across processes when initializing FSDP objectNrJ   rK   zLoading model from T©Úweights_onlyzModel loaded from rL   ©ÚDefaultLoadPlannerr-   rI   ©r,   Ústorage_readerrP   )(r>   rF   r8   Úwait_for_everyoner?   r@   rR   rB   r9   r:   r=   rS   r   rD   ÚtyperV   Úis_fsdp2Úsync_module_statesÚ
ValueErrorr   r   rT   rU   rW   rX   Úis_main_processrY   ÚloadrZ   r[   r\   r]   r^   r_   ro   r   Únamer0   ÚFileSystemReaderr5   )rC   ra   r-   Ú	input_dirrc   r.   rd   re   r8   rf   rg   r/   rh   Zinput_model_fileÚ
load_modelr,   ri   ro   rj   Zload_resultr   r   r   Úload_fsdp_modelÂ   s„   

ýÿûÿø"	ÿýÿýýÉ8r}   c                 C   s  ddl m} ddl m} tj|dd� | jdkr#| || j| j| j¡nt	ƒ }	t
| ƒ}
|	�Ø | jdkr@ddlm} ||||
d	�}n| ||¡}| j|jkr€|jdkr|dkrZt› d
�nt› d|› d
�}tj ||¡}t d|› �¡ t ||¡ t d|› �¡ nt|rÃddlm  m} ddlm} tj |t› d|› �¡}tj|dd� t d|› �¡ |jd|i| |¡|ƒ d� t d|› �¡ n9t› d|› d|j› d
�}tj ||¡}t d|› �¡ t ||¡ t d|› �¡ W d  ƒ dS W d  ƒ dS W d  ƒ dS 1 �sw   Y  dS )aÎ  
    Save an FSDP optimizer state checkpoint.

    When ``state_dict_type`` is not ``FULL_STATE_DICT``, the ``use_dcp`` parameter controls the saving strategy:

    - ``use_dcp=True`` (default): Uses ``torch.distributed.checkpoint`` (DCP) to save the optimizer state.
    - ``use_dcp=False``: Uses per-rank ``torch.save`` instead of DCP, avoiding cross-rank communication.
      This is useful at large scale (e.g., 2800+ GPUs) where DCP can cause timeouts due to TCP congestion
      from Gloo-based collective communication.

    .. note::
        Checkpoints saved with ``use_dcp=False`` are saved as per-rank files
        (``{prefix}_rank{local_rank}.bin``) and must be loaded with ``use_dcp=False`` as well.
    r   rE   r7   TrG   r   r
   ©Úget_optimizer_state_dictr$   rJ   rK   zSaving Optimizer state to zOptimizer state saved in NrM   Ú	optimizerrO   rL   )r>   rF   r8   r   rQ   r=   r?   rB   rS   r   rD   r)   r   Úoptim_state_dictr@   rV   r   rT   rU   rW   rX   rY   r   r\   r]   r^   r_   rN   r`   )rC   ra   r€   r-   rb   Úoptimizer_indexrd   re   r8   rg   r/   r   Úoptim_stateZoptim_state_nameZoutput_optimizer_fileri   rN   rj   r   r   r   Úsave_fsdp_optimizer#  s\   
ýÿû

 ÿ€ýÜ€ $âr„   c                 C   sn  ddl m} ddl m}	 | ¡  | jdkr | || j| j| j¡ntƒ }
t	| ƒ}|
�ÿ | j|	j
krkd}|jdks;| jjsj|dkrDt› d�nt› d|› d�}tj ||¡}t d|› �¡ tj|d	d
�}t d|› �¡ n”|rÕddlm  m} ddlm} t|ƒjt› d|› �kr“t|ƒt› d|› � nt|ƒ}t d|› �¡ | jdkr²ddlm} ||||d�}n| ||¡}d|i}|j|| |¡|ƒ d� |d }t d|› �¡ n*t› d|› d|j› d�}tj ||¡}t d|› �¡ tj|d	d
�}t d|› �¡ | jdk�r|j |||d�}| !|¡ nddlm"} |||||d� W d  ƒ n	1 �s,w   Y  | ¡  dS )aç  
    Load an FSDP optimizer state checkpoint.

    When ``state_dict_type`` is not ``FULL_STATE_DICT``, the ``use_dcp`` parameter controls the loading strategy:

    - ``use_dcp=True`` (default): Uses ``torch.distributed.checkpoint`` (DCP) to load the optimizer state.
    - ``use_dcp=False``: Uses per-rank ``torch.load`` instead of DCP, avoiding cross-rank communication.
      This is useful at large scale (e.g., 2800+ GPUs) where DCP can cause timeouts due to TCP congestion
      from Gloo-based collective communication.

    .. note::
        Checkpoints saved with ``use_dcp=False`` must be loaded with ``use_dcp=False`` as well, since the
        file format differs (per-rank ``.bin`` files vs. DCP's sharded directory format).
    r   rE   r7   r   NrJ   rK   zLoading Optimizer state from Trl   zOptimizer state loaded from rn   zLoading Optimizer from r
   r~   r$   r€   rp   zOptimizer loaded from rL   )r-   Úoptimr�   )Úset_optimizer_state_dict)#r>   rF   r8   rr   r=   r?   rB   rS   r   rD   r@   rV   r:   r   r   rT   rU   rW   rX   rY   rx   r\   r]   r^   r_   ro   r   ry   r)   r   r�   rz   Zoptim_state_dict_to_loadr4   r†   )rC   ra   r€   r-   r{   r‚   r.   rd   re   r8   rg   r/   rƒ   Úoptimizer_nameZinput_optimizer_fileri   ro   rj   r   Zflattened_osdr†   r   r   r   Úload_fsdp_optimizeri  sl   
ýÿû ÿ€ÿý
ý€Ï3rˆ   Úcheckpoint_dirÚ	save_pathÚsafe_serializationc                 C   s¤   ddl m  m} ddlm  m  m} i }t|ƒ}|jdd� |j|| | ¡| 	¡ dd� |r5|t
 n|t }t| ¡ ƒdkrI|t|ƒd  }t|||d� |S )z­
    Passthrough to `torch.distributed.checkpoint.format_utils.dcp_to_torch_save`

    Will save under `save_path` as either `model.safetensors` or `pytorch_model.bin`.
    r   NTrG   )rq   rP   Zno_distr   )r‹   )r\   r]   r^   Z)torch.distributed.checkpoint.format_utilsZformat_utilsr   ÚmkdirZ_load_state_dictrz   Z_EmptyStateDictLoadPlannerr   r   ÚlenÚkeysÚlistr   )r‰   rŠ   r‹   ri   Zdist_cp_format_utilsr,   r   r   r   Ú)_distributed_checkpoint_to_merged_weights½  s    ür�   Úoutput_pathÚremove_checkpoint_dirc           
      C   s2  t | ƒ} ddlm} tddƒstdƒ‚|  ¡ sh| d  ¡ }| d  ¡ }d| › d	�}|rD|rD|d
7 }|d| › d| › d�7 }|d7 }t|ƒ‚|rV|d7 }|d| › d�7 }t|ƒ‚|rd|d7 }|d| › d�7 }t|ƒ‚|ƒ }|jr“t d| › �¡ t	| ||ƒ}	t d|	› �¡ |r“t d| › �¡ t
 | ¡ | ¡  dS )a?  
    Merge the weights from sharded FSDP model checkpoints into a single combined checkpoint. Should be used if
    `SHARDED_STATE_DICT` was used for the model. Weights will be saved to `{output_path}/model.safetensors` if
    `safe_serialization` else `pytorch_model.bin`.

    Note: this is a CPU-bound process.

    Args:
        checkpoint_dir (`str`):
            The directory containing the FSDP checkpoints (can be either the model or optimizer).
        output_path (`str`):
            The path to save the merged checkpoint.
        safe_serialization (`bool`, *optional*, defaults to `True`):
            Whether to save the merged weights with safetensors (recommended).
        remove_checkpoint_dir (`bool`, *optional*, defaults to `False`):
            Whether to remove the checkpoint directory after merging.
    r   )ÚPartialStateú>=z2.3.0z/`merge_fsdp_weights` requires PyTorch >= 2.3.0`Zpytorch_model_fsdp_0Zoptimizer_0zTried to load from z) but couldn't find a valid metadata file.zE However, potential model and optimizer checkpoint directories exist.zPlease pass in either z/pytorch_model_fsdp_0 or z/optimizer_0zinstead.z8 However, a potential model checkpoint directory exists.zPlease try passing in z/pytorch_model_fsdp_0 instead.z< However, a potential optimizer checkpoint directory exists.z/optimizer_0 instead.zMerging FSDP weights from z.Successfully merged FSDP weights and saved to z"Removing old checkpoint directory N)r   Úaccelerate.stater“   r   rv   Úexistsrw   rW   rX   r�   ÚshutilÚrmtreerr   )
r‰   r‘   r‹   r’   r“   Zmodel_path_existsZoptimizer_path_existsÚerrÚstaterŠ   r   r   r   Úmerge_fsdp_weightsÙ  s<   
úý
r›   r-   Údevicec           	         s„   t |dd ƒ}|s
ˆS i ‰ |D ]%}| d¡}d |d d… ¡|d }}| |¡}t ||ƒ}d ˆ t|ƒ< qdtjjf‡ ‡fdd„}|S )NZ_tied_weights_keysÚ.éÿÿÿÿÚmodulec                    s�   t tƒ}| jdd�D ]\}}t|ƒˆ v r|t|ƒ  |¡ q
ˆ| ƒ} | ¡ D ]\}}|D ]}ˆ | }|d u r>t| |ƒˆ |< q,t| ||ƒ q,q&| S )NF)Úrecurse)r   r�   Únamed_parametersÚidÚappendÚitemsrA   Úsetattr)rŸ   Zparams_to_tieÚnÚparamZid_keyZ_param_namesÚ
param_name©Z_tied_paramsÚparam_init_fnr   r   Úparam_init_fn_tied_param"  s   €ù	z7ensure_weights_retied.<locals>.param_init_fn_tied_param)rA   ÚsplitrU   Úget_submoduler¢   rY   ÚnnÚModule)	rª   r-   rœ   Z_tied_namesry   r¨   Úmodr§   r«   r   r©   r   Úensure_weights_retied  s   


r±   Úfull_sdr;   c              	   C   s˜  ddl m} ddlm}m} | ¡ }i }dd„ }	dd„ }
| jr�| ¡ D ]\\}}||vr=td|› d	t	|ƒ› d
t	|ƒ› d�ƒ‚|| }|j
}| ¡  |j¡}t||ƒrU| ¡ }|j|d|jjd� ||||jƒ}|	|||ƒ\}}|
|||ƒ}|r{| d¡}|||< q#nB| ¡ D ]=\}}|j
}tj| ¡ |j|jd�}|j|d|jjd� ||||jƒ}|	|||ƒ\}}|
|||ƒ}|r¾| d¡}|||< q…|j|dd� |S )a�  
    Loads the full state dict (could be only on rank 0) into the sharded model. This is done by broadcasting the
    parameters from rank 0 to all other ranks. This function modifies the model in-place.

    Args:
        accelerator (`Accelerator`): The accelerator instance
        model (`torch.nn.Module`):
            The model to load the state dict into, expected to be on meta device or a VRAM spike can occur
        full_sd (`dict`): The full state dict to load, can only be on rank 0
        cpu_offload (`bool`, defaults to `False`):
            If True, move sharded parameters to CPU after distribution. Required when FSDP CPU offloading is enabled.
    r   N)ÚDTensorÚdistribute_tensorc           
      S   sŠ   z|   |¡}W n ty"   | dd¡\}}|  |¡}t||ƒ}Y nw ttdƒ}d }|o1|jtjk}	|jj	r;|	s;|j}|d uoB| 
¡ |fS )Nr�   r   Úfloat8_e4m3fn)Zget_parameter_or_bufferÚAttributeErrorÚrsplitr­   rA   ÚhasattrrY   Údtyperµ   Úis_floating_pointÚis_contiguous)
r-   r¨   Zempty_paramÚ	old_paramZbase_param_nameZlocal_param_nameÚ	submoduleZis_torch_e4m3fn_availableÚcasting_dtypeZis_param_float8_e4m3fnr   r   r   Ú_infer_parameter_dtypeS  s   
ü
z:fsdp2_load_full_state_dict.<locals>._infer_parameter_dtypec                 S   s$   |d ur
| j |d�} |r|  ¡ } | S )N)r¹   )ÚtoÚ
contiguous)ÚtensorÚto_contiguousr¹   r   r   r   Ú_cast_and_contiguouse  s
   z8fsdp2_load_full_state_dict.<locals>._cast_and_contiguouszParameter 'zZ' found in sharded model state dict but missing from full state dict. Full state dict has z keys, sharded has z keys.)ÚsrcÚgroupÚcpu)rœ   r¹   T)Úassign)Útorch.distributedr]   Útorch.distributed.tensorr³   r´   r,   rw   r¤   ÚKeyErrorr�   Údevice_meshÚdetachrÀ   Údevice_typeÚ
isinstanceZto_localÚ	broadcastrÆ   ÚWORLDÚ
placementsrY   ÚemptyÚsizer¹   r4   )ra   r-   r²   r;   Údistr³   r´   Zmeta_sharded_sdZ
sharded_sdr¿   rÄ   r¨   Zsharded_paramZ
full_paramrÌ   Zsharded_tensorrÃ   r¾   Úfull_tensorr   r   r   Úfsdp2_load_full_state_dict>  sb   ÿÿÿ
ý

çý

r×   r€   Úmappingc                    s\   ddl m} i }d||< z| jD ]}‡ fdd„|d D ƒ|d< qW dS  ty-   tdƒ‚w )	a¾  
    Switches the parameters of the optimizer to new ones (sharded parameters in usual case). This function modifies the
    optimizer in-place.

    Args:
        optimizer (`torch.optim.Optimizer`): Optimizer instance which contains the original model parameters
        mapping (`dict`): Mapping from the original parameter (specified by `data_ptr`) to the sharded parameter

    Raises:
        KeyError:
            If a parameter in the optimizer couldn't be switched to its sharded version. This should never happen and
            indicates a bug. If we kept the original params instead of raising, the training wouldn't be numerically
            correct and weights wouldn't get updated.
    r   )r³   Ú_local_tensorc                    s   g | ]}ˆ |j  ‘qS r   )Údata_ptr©Ú.0Úp©rØ   r   r   Ú
<listcomp>´  s    z5fsdp2_switch_optimizer_parameters.<locals>.<listcomp>Úparamsz„A parameter in the optimizer couldn't be switched to its sharded version. This breaks the training. Please raise an issue on GitHub.N)rÊ   r³   Úparam_groupsrË   )r€   rØ   r³   Úaccessor_mappingÚparam_groupr   rÞ   r   Ú!fsdp2_switch_optimizer_parametersž  s   
ÿÿýrä   c                 C   sr   ddl m}m} t| jj|ƒ}t|dd�dd… D ]\}}||ƒr6||dd�}| jjjr0||ƒ}| ||¡ q|S )	a8  
    Applies the activation checkpointing to the model.

    Args:
        accelerator (`Accelerator`): The accelerator instance
        model (`torch.nn.Module`): The model to apply the activation checkpointing to

    Returns:
        `torch.nn.Module`: The model with the activation checkpointing applied
    r   )Úcheckpoint_wrapperÚoffload_wrapperT)Úreturn_fqnsNrž   F)Úpreserve_rng_state)	Ú;torch.distributed.algorithms._checkpoint.checkpoint_wrapperrå   ræ   Úfsdp2_prepare_auto_wrap_policyrš   rC   r   Ú activation_checkpointing_offloadZset_submodule)ra   r-   rå   ræ   Úauto_wrap_policy_funcÚ
layer_nameÚlayerÚwrappedr   r   r   Úfsdp2_apply_ac½  s   
€rð   Úreturnc                 C   s^   t | ddƒ}|rt | |dƒn| }t|tjjƒsdS d}| ¡ D ]\}}dt|ƒjv r,|}q|S )a)  Find the final normalization layer before the output head.

    The final norm is conventionally a direct child of the base model (e.g. `model.norm`
    for Llama, `transformer.ln_f` for GPT-2), so we only scan the base model's direct
    children. Returns the last norm found there, or None.
    Úbase_model_prefixÚ NÚNorm)rA   rÏ   rY   r®   r¯   Únamed_childrenrs   Ú__name__)r-   Úbase_prefixÚ
base_modelÚ
final_normrK   rŸ   r   r   r   Ú_find_final_normÝ  s   €rú   c           !         s&  ddl m‰ m}m} t|ˆ ƒpt|ƒot|jˆ ƒ}|r|S | jj}| 	|¡ t
| ddƒ}|j|j|jp5|ƒ |durA|t| jjƒ nddœ}tddƒrY|jdurYt|j|| jƒ|d< d	}tƒ }	| ¡ D ]\}
}|jjd
kr~d}|js~| ¡ s~| ¡ s~|	 |¡ qb|	r£tddƒr£t| dtƒ ¡ƒ}||	B |d< | jr£t  dt!|	ƒ› d�¡ | j"dkrÞ|sÞg }| ¡ D ]\}
}|jrÊ|j#t$j%krÊ| &|
¡ |j' (t$j%¡|_'q°| jrÞ|rÞt) *dt!|ƒ› d|› d�¡ | +¡ }|j,�r|�st-|ddd�‰t. /‡fdd„| 0¡ D ƒ¡}| (t$ d¡¡}t1|dƒ�r| 2¡  t3||ƒ}|du�r9t4|ƒdd… D ]}||ƒ�r7t|ˆ ƒ�s7||fi |¤Ž �q"t
|ddd„ ƒƒ }t
|ddd„ ƒƒ }t
|ddƒ}t
|ddƒ}|du�o_||u }|�sv|du�rvt|ˆ ƒ�sv||fi |¤Ž t5|ƒ}|�rƒ|||f}n||f}‡ fdd„|D ƒ}|�r¡||fi i |¥d d	i¥¤Ž t|ˆ ƒ�s¸d!d„ | 6¡ D ƒ}||fi |¤Ž |j,�rÑ|�sÑdd"l m7} t8| ||t|j|ƒd#� |j,�r|�s| 6¡ D ]*\}}| (| j¡}d$|v �rù| 9d$d%¡\}}| :|¡} n|}|} | j;||d	d&� �qÜt1|dƒ�r| 2¡  |S )'a"  Prepares the model for FSDP2 in-place. Also returns the model to avoid misuse of the original model.

    Args:
        accelerator (`Accelerator`): The accelerator instance
        model (`torch.nn.Module`): The model to prepare

    Returns:
        `torch.nn.Module`: Prepared model
    r   )Ú
FSDPModuleÚMixedPrecisionPolicyÚfully_shardÚtorch_device_meshN)Úreshard_after_forwardZoffload_policyZ	mp_policyÚmeshr”   z2.7.0Zignored_paramsFÚ
Params4bitTzFound zÑ non-floating frozen Params4bit. Excluding from FSDP2 sharding to prevent quant_state corruption.To enable memory-efficient sharding of 4-bit weights, setbnb_4bit_quant_storage to a floating dtype (e.g. bf16).ÚnozŒFSDP upcast of low precision parameters to fp32 (since mixed_precision != 'no') may affect the precision of model checkpoints. This effects z parameters: z...)r    Úfqnsc                    s   i | ]\}}|ˆ v r||“qS r   r   ©rÜ   ÚkÚv)Únon_persistent_buffer_fqnsr   r   Ú
<dictcomp>I  ó    z'fsdp2_prepare_model.<locals>.<dictcomp>ÚmetaÚtie_weightsrž   Zget_input_embeddingsc                   S   ó   d S ©Nr   r   r   r   r   Ú<lambda>[  ó    z%fsdp2_prepare_model.<locals>.<lambda>Úget_output_embeddingsc                   S   r  r  r   r   r   r   r   r  \  r  Úweightc                    s"   g | ]}|d urt |ˆ ƒs|‘qS r  )rÏ   )rÜ   Úm)rû   r   r   rß   m  s   " z'fsdp2_prepare_model.<locals>.<listcomp>rÿ   c                 S   s   i | ]\}}|d kr||“qS )rÿ   r   r  r   r   r   r  t  r	  )ÚCPUOffloadPolicy)r;   r�   r   )Ú
persistent)<Útorch.distributed.fsdprû   rü   rý   rÏ   r   Ú	_orig_modrš   rC   Úset_auto_wrap_policyrA   rÿ   r;   Úmixed_precision_policyÚtupleÚparallelism_configÚfsdp_dim_namesr   Úignored_modulesÚget_parameters_from_modulesrœ   Úsetr¡   Ú	__class__rö   Úrequires_gradrº   Ú
is_complexÚaddÚgetrw   ÚwarningsÚwarnr�   Úmixed_precisionr¹   rY   Úfloat32r£   ÚdatarÀ   rW   rX   r,   Úcpu_ram_efficient_loadingr   ÚcopyÚdeepcopyÚnamed_buffersr¸   r  rê   r   rú   r¤   r  r×   r·   r­   Úregister_buffer)!ra   r-   rü   rý   Úis_type_fsdpÚfsdp2_pluginr   Zfsdp2_kwargsZmodel_has_params4bitZincompatible_params4bitry   r§   ÚignoredZupcasted_paramsZoriginal_sdZoriginal_non_persistent_buffersrì   rŸ   Zinput_embedZoutput_embedZinput_weightZoutput_weightZis_weights_tiedrù   Ztail_modulesÚtailZroot_kwargsr  ÚfqnZbuffer_tensorÚ
parent_fqnZlocal_buffer_nameÚparent_moduler   )rû   r  r   Úfsdp2_prepare_modelï  sÂ   

ÿ

û	
ÿ
€ÿ	
€
ÿÿÿ	ÿ

€ÿ
r5  c           
         sä   ddl m}m} ˆ j}t|tjƒr|j}||u r]t|ddƒ}|du r$g }t	|ƒ}ˆ j
dur0ˆ j
}tƒ ‰|D ]}t||ƒ}|du rHtd|› d�ƒ‚ˆ |¡ q5dtjjdtf‡fd	d
„}	|	S ||u rpdtjjdtf‡ fdd
„}	|	S dS )aà  Prepares the auto wrap policy based on its type, done to mimic the behaviour of FSDP1 auto wrap policy.

    Args:
        fsdp2_plugin (`FullyShardedDataParallelPlugin`):
            Instance of `FullyShardedDataParallelPlugin` containing the configuration options
        model (`torch.nn.Module`):
            The model to wrap

    Returns:
        `Callable[[torch.nn.Module], bool] | None`:
            The auto wrap policy function to be applied to the model or `None`
    r   )Úsize_based_auto_wrap_policyÚtransformer_auto_wrap_policyÚ_no_split_modulesNz+Could not find the transformer layer class z in the model.rŸ   rñ   c                    s"   ˆ sdS t | d| ƒ} t| tˆ ƒƒS )NFZ_checkpoint_wrapped_module)rA   rÏ   r  )rŸ   )Útransformer_cls_to_wrapr   r   Úpolicy¼  s   z.fsdp2_prepare_auto_wrap_policy.<locals>.policyc                    s    t dd„ |  ¡ D ƒƒ}|ˆ jkS )Nc                 s   s   � | ]}|  ¡ V  qd S r  )ÚnumelrÛ   r   r   r   Ú	<genexpr>Ç  s   € zAfsdp2_prepare_auto_wrap_policy.<locals>.policy.<locals>.<genexpr>)ÚsumÚ
parametersÚmin_num_params)rŸ   Zmodule_num_params)r/  r   r   r:  Æ  s   
)Útorch.distributed.fsdp.wrapr6  r7  Úauto_wrap_policyrÏ   Ú	functoolsÚpartialÚfuncrA   r�   Útransformer_cls_names_to_wrapr  r   rv   r"  rY   r®   r¯   Úbool)
r/  r-   r6  r7  ÚfnÚno_split_modulesrE  Úlayer_classÚtransformer_clsr:  r   )r/  r9  r   rê   ™  s.   

øþrê   c                  K   s   ddl m} |di | ¤ŽS )a  
    Returns a `GradScaler` for FSDP2, as the current implementation of `get_grad_scaler` doesn't accept other args. We
    need this as current `get_grad_scaler` accepts only `distributed_type` as arg, which doesn't differentiate between
    FSDP1 and FSDP2
    r   )Ú
GradScalerNr   )Ztorch.amp.grad_scalerrK  )ÚkwargsrK  r   r   r   Úget_fsdp2_grad_scalerÏ  s   rM  Únamed_paramsc                 C   s:   dd„ |   ¡ D ƒ} dd„ |   ¡ D ƒ} dd„ |   ¡ D ƒ} | S )a6  Removes parameter name modifiers in order to map them back to their original names.

    See huggingface/accelerate#3554 for more context.

    Args:
        named_params (`dict`): The named parameters dictionary to canonicalize.

    Returns:
        `dict`: The canonicalized named parameters dictionary
    c                 S   ó   i | ]\}}|  d d¡|“qS )z._checkpoint_wrapped_moduleró   r   r  r   r   r   r  å  r	  z,fsdp2_canonicalize_names.<locals>.<dictcomp>c                 S   s,   i | ]\}}|  d ¡r| d d¡n||“qS )z
_orig_mod.ró   )Ú
startswithr   r  r   r   r   r  æ  s     ÿc                 S   rO  )z
._orig_modró   r   r  r   r   r   r  é  r	  )r¤   )rN  r   r   r   Úfsdp2_canonicalize_namesÚ  s   ÿrQ  Úmodulesc                 C   s‚   | du rt ƒ S g }t| tƒr/t | ¡}g }| ¡ D ]\}}| |¡r,| |¡ | |¡ q|} | D ]}| 	t
| ¡ ƒ¡ q1t |ƒS )zöConverts modules to parameters where modules can be a string or list of torch.nn.Module

    Args:
        modules (`Union[Iterable[torch.nn.Module], str]`): List of modules

    Returns:
        `set[torch.nn.Parameter]`: List of parameters
    N)r  rÏ   ÚstrÚreÚcompileÚnamed_modulesÚ	fullmatchrÀ   r£   Úextendr�   r>  )rR  r-   rœ   r>  ÚregZmapped_modulesry   rŸ   r   r   r   r  í  s   




€r  )FN)r   FT)r   T)T)TF)F)Fr*  rB  r   rT  r—   r$  Úcollectionsr   Úcollections.abcr   Ú
contextlibr   Údataclassesr   Úpathlibr   Útypingr   r	   rY   Úloggingr   Ú	constantsr   r   r   r   r   Úmodelingr   r   Úotherr   r   r   Úversionsr   rö   rW   r    r"   r0   r5   rD   rk   r}   r„   rˆ   rS  rF  r�   r›   r®   r¯   rœ   r±   Údictr×   r…   Ú	Optimizerrä   rð   rú   r5  rê   rM  rQ  r  Ú	Parameterr  r   r   r   r   Ú<module>   sp   




P
aG
ÿTÿÿÿÿ
ÿ7.`  (+6ÿ
þ