o
    œôj©¬  ã                   @   sZ  d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d d	lmZ d d
lmZm Z  d dl!m"Z" d dl#m$Z$m%Z% d dl&m'Z' d dl(m)Z)m*Z*m+Z+ d dl,m-Z- d dl.m/Z/ d dl0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z7m8Z8m9Z9m:Z:m;Z; d dl<m=Z=m>Z>m?Z?m@Z@mAZA d dl<mBZC d dlDmEZEmFZFmGZGmHZH d dlImJZJ d dlKmLZLmMZMmNZNmOZO d dlPmQZQmZmRZR d dlSmTZT d dlUmVZVmWZW e
�r.d dlXmYZYmZZZm[Z[ eE�rd dl\m]Z] eeee  eee^e_ge^f e]f Z`neeee  eee^e_ge^f f Z`ee[ed f Zad ZbG d!d"„ d"e3e:ƒZcd#eeee eee  f  d$ed% d&efd'd(„Zdd)ed% d*ed&efd+d,„Zed-ed.ed&dfd/d0„ZfG d1d2„ d2e9ƒZgd3eee^d4f  d&d4fd5d6„Zhd7d8d*ed&d9fd:d;„Zid<e"d&e^fd=d>„Zjd-ed&ed? fd@dA„Zk	Bd`d-edCe_dDe^d&ed? fdEdF„ZldGed&e^fdHdI„ZmdGed&e^fdJdK„Znd-eod&e$e fdLdM„Zpd`dGed-edCe_dNe^d&df
dOdP„Zqd`dQeeref d-edCe_dNe^d&df
dRdS„ZsdTeee"f d&e^fdUdV„Ztd-ejujdWejvd&dfdXdY„ZwdZeeref dGed&dfd[d\„Zxd]eeref dGed&dfd^d_„ZydS )aé    N)Ú	ExitStack)Ú	timedelta)Úpartial)ÚPath)ÚTYPE_CHECKINGÚAnyÚCallableÚContextManagerÚDictÚ	GeneratorÚListÚLiteralÚOptionalÚSetÚTupleÚTypeÚUnion)ÚRequirementCache)Úrank_zero_only)ÚTensor)ÚModuleÚ	Parameter)Ú	Optimizer)Ú	TypeGuardÚoverride)ÚAccelerator)ÚCheckpointIOÚClusterEnvironmentÚ	Precision)Údefault_pg_timeout)ÚFSDPPrecision)Ú_SubprocessScriptLauncher)ÚParallelStrategy)Ú_StrategyRegistry)Ú
TBroadcastÚ_apply_filterÚ_BackwardSyncControlÚ_ShardedÚ!_validate_keys_for_strict_loading)ÚReduceOpÚ_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Ú_TORCH_GREATER_EQUAL_2_0Ú_TORCH_GREATER_EQUAL_2_1Ú_TORCH_GREATER_EQUAL_2_2Ú_TORCH_GREATER_EQUAL_2_3)Ú
_EmptyInit)Ú_METADATA_FILENAMEÚ
_lazy_loadÚ_materialize_tensorsÚ_move_state_into)Úrank_zero_deprecationr   Úrank_zero_warn)Ú
reset_seed)Ú_PATHÚ	_Stateful)Ú
CPUOffloadÚMixedPrecisionÚShardingStrategy©ÚModuleWrapPolicy)Ú
FULL_SHARDZSHARD_GRAD_OPZNO_SHARDZHYBRID_SHARD)ÚfsdpÚfsdp_cpu_offloadc                       s  e Zd ZdZdddddedddddddfdee deeej	  dee
 dee d	ee d
ee deeddf ded ded deeee eee  f  ded ddded deddf‡ fdd„Zeedefdd„ƒƒZejededdfdd„ƒƒZeedej	fdd „ƒƒZedefd!d"„ƒZejd#eddfd$d"„ƒZedefd%d&„ƒZeedeeef fd'd(„ƒƒZedee fd)d*„ƒZ eded fd+d,„ƒZ!eede"fd-d.„ƒƒZ#e#jedee" ddfd/d.„ƒƒZ#edkd0d1„ƒZ$edk‡ fd2d3„ƒZ%ed4ed5ee& de'eee& f fd6d7„ƒZ(ed4edefd8d9„ƒZ)ed:e&de&f‡ fd;d<„ƒZ*ed4eddfd=d>„ƒZ+edld?ee de,fd@dA„ƒZ-ede,fdBdC„ƒZ.e	DdmdEe/dFee dGeee0ef  de/fdHdI„ƒZ1edJededdfdKdL„ƒZ2edndNe3dOede3fdPdQ„ƒZ4e	R	Sdod4ed:e&dTee5ef dUee5ef dVede/fdWdX„ƒZ6e		dpdYe7dZeeeee&ef f d[ee d\eeee8eegef f  ddf
d]d^„ƒZ9e		SdqdYe7dZeeee&eeeee&ef f f  d_edeeef fd`da„ƒZ:e;edbe<ddfdcdd„ƒƒZ=dkdedf„Z>defdgdh„Z?dkdidj„Z@‡  ZAS )rÚFSDPStrategyat  Strategy for Fully Sharded Data Parallel provided by torch.distributed.

    .. warning::  This is an :ref:`experimental <versioning:Experimental API>` feature.

    Fully Sharded Training shards the entire model across all available GPUs, allowing you to scale model
    size, whilst using efficient communication to reduce overhead. In practice, this means we can remain
    at parity with PyTorch DDP, whilst scaling our model sizes dramatically. The technique is similar
    to ZeRO-Stage 3.

    For more information check out
    `this blogpost <https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api>`__.

    Defaults have been set and options have been exposed, but may require configuration
    based on your level of memory/speed efficiency. We suggest having a look at
    `this tutorial <https://pytorch.org/tutorials/intermediate/FSDP_tutorial.html>`__ for more information.

    Arguments:
        cpu_offload: See ``cpu_offload`` parameter in :class:`torch.distributed.fsdp.FullyShardedDataParallel`.
        mixed_precision: See ``mixed_precision`` parameter in :class:`torch.distributed.fsdp.FullyShardedDataParallel`.
        auto_wrap_policy: Same as ``auto_wrap_policy`` parameter in
            :class:`torch.distributed.fsdp.FullyShardedDataParallel`. For convenience, this also accepts a set of the
            layer classes to wrap.
        activation_checkpointing: Deprecated. Use ``activation_checkpointing_policy``.
        activation_checkpointing_policy: Same as ``auto_wrap_policy`` parameter in
            :class:`torch.distributed.fsdp.FullyShardedDataParallel` but used when selecting the modules for which you
            want to enable activation checkpointing. Enabling this can free up a significant amount of memory at the
            cost of speed since activations in these layers need to be recomputed during backpropagation. For
            convenience, this also accepts a set of the layer classes to wrap.
        sharding_strategy: Select whether to shard model parameters, gradients, optimizer states, or a combination of
            them. Available values are:

            - ``"FULL_SHARD"``: Shards model parameters, gradients, and optimizer states (default).
            - ``"SHARD_GRAD_OP"``: Shards gradients and optimizer states only. Model parameters get replicated.
            - ``"NO_SHARD"``: No sharding (identical to regular DDP).
            - ``"HYBRID_SHARD"``: Shards model parameters, gradients, and optimizer states within a single machine, but
              replicates across machines.

            Also accepts a :class:`torch.distributed.fsdp.ShardingStrategy` enum value.

        state_dict_type: The format in which the state of the model and optimizers gets saved into the checkpoint.

            - ``"full"``: The full weights and optimizer states get assembled on rank 0 and saved to a single file.
            - ``"sharded"``: Each rank saves its shard of weights and optimizer states to a file. The checkpoint is
              a folder with as many files as the world size.

        \**kwargs: See available parameters in :class:`torch.distributed.fsdp.FullyShardedDataParallel`.

    NrC   ÚshardedÚacceleratorÚparallel_devicesÚcluster_environmentÚ	precisionÚprocess_group_backendÚtimeoutÚcpu_offloadr>   Úmixed_precisionr?   Úauto_wrap_policyÚ_POLICYÚactivation_checkpointingÚactivation_checkpointing_policyÚsharding_strategyÚ_SHARDING_STRATEGYÚstate_dict_type)ÚfullrG   ÚkwargsÚreturnc                    s€   t ƒ j||||d� d| _|| _|| _tƒ | _t|	|ƒ| _t	r&| j 
dd¡ t|
|ƒ| _|| _t|| jƒ| _t|ƒ| _|| _d S )N)rH   rI   rJ   rK   é   Úuse_orig_paramsT)ÚsuperÚ__init__Ú
_num_nodesÚ_process_group_backendÚ_timeoutÚ_FSDPBackwardSyncControlÚ_backward_sync_controlÚ_auto_wrap_policy_kwargsÚ_fsdp_kwargsr0   Ú
setdefaultÚ _activation_checkpointing_kwargsÚ_state_dict_typeÚ_init_sharding_strategyrT   Ú_init_cpu_offloadrN   rO   )ÚselfrH   rI   rJ   rK   rL   rM   rN   rO   rP   rR   rS   rT   rV   rX   ©Ú	__class__© ús/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/strategies/fsdp.pyr]   Ž   s(   üÿ

zFSDPStrategy.__init__c                 C   ó   t dt| ƒj› d�ƒ‚)NúThe `z3` does not use the `CheckpointIO` plugin interface.©ÚNotImplementedErrorÚtypeÚ__name__©rj   rm   rm   rn   Úcheckpoint_io·   ó   zFSDPStrategy.checkpoint_ioÚioc                 C   ro   )Nrp   z3` does not support setting a `CheckpointIO` plugin.rq   )rj   rx   rm   rm   rn   rv   ¼   rw   c                 C   s   | j d usJ ‚| j | j S ©N)rI   Ú
local_rankru   rm   rm   rn   Úroot_deviceÁ   s   zFSDPStrategy.root_devicec                 C   ó   | j S ry   ©r^   ru   rm   rm   rn   Ú	num_nodesÇ   ó   zFSDPStrategy.num_nodesr~   c                 C   s
   || _ d S ry   r}   )rj   r~   rm   rm   rn   r~   Ë   s   
c                 C   s   | j d ur
t| j ƒS dS )Nr   )rI   Úlenru   rm   rm   rn   Únum_processesÏ   s   zFSDPStrategy.num_processesc                 C   s   | j | j | jdœS )N)Únum_replicasÚrank)r~   r�   Úglobal_rankru   rm   rm   rn   Údistributed_sampler_kwargsÓ   s   z'FSDPStrategy.distributed_sampler_kwargsc                 C   r|   ry   )r_   ru   rm   rm   rn   rL   Ø   r   z"FSDPStrategy.process_group_backendc                 C   s&   | j r| j S | j}t|tƒr|jS d S ry   )rO   rK   Ú
isinstancer    Úmixed_precision_config©rj   Úpluginrm   rm   rn   r‡   Ü   s   
z#FSDPStrategy.mixed_precision_configc                 C   s(   | j }|d urt|tƒsJ ‚|S tdƒS )Nz32-true)Ú
_precisionr†   r    rˆ   rm   rm   rn   rK   å   s
   zFSDPStrategy.precisionc                 C   s*   |d urt |tƒstd|› �ƒ‚|| _d S )NzGThe FSDP strategy can only work with the `FSDPPrecision` plugin, found )r†   r    Ú	TypeErrorrŠ   )rj   rK   rm   rm   rn   rK   î   s   
c                 C   s2   | j d usJ ‚| j jst| j | j| jƒ| _d S d S ry   )rJ   Úcreates_processes_externallyr!   r�   r~   Ú	_launcherru   rm   rm   rn   Ú_configure_launcherõ   s   ÿz FSDPStrategy._configure_launcherc                    s   t ƒ  ¡  |  ¡  d S ry   )r\   Úsetup_environmentÚ_setup_distributedru   rk   rm   rn   r�   û   s   
zFSDPStrategy.setup_environmentÚmoduleÚ
optimizersc                 C   sV   t stdt| ƒj› d�ƒ‚| j d¡}|du r"tdt| ƒj› d�ƒ‚|  |¡}||fS )zÞWraps the model into a :class:`~torch.distributed.fsdp.fully_sharded_data_parallel.FullyShardedDataParallel`
        module and sets `use_orig_params=True` to keep the reference to the original parameters in the optimizer.rp   z¯` does not support the joint setup of module and optimizer(s). Please do it in this order: Create the model, call `setup_module`, create the optimizer, call `setup_optimizer`.r[   Fz	You set `zõ(use_orig_params=False)` but this is not supported when setting the model and optimizer up jointly. Either set it to `True` or set the objects up in this order: Create the model, call `setup_module`, create the optimizer, call `setup_optimizer`.)r0   rr   rs   rt   rd   ÚgetÚ
ValueErrorÚsetup_module)rj   r‘   r’   r[   rm   rm   rn   Úsetup_module_and_optimizers   s   ÿÿ
z(FSDPStrategy.setup_module_and_optimizersc                    s”   ddl m‰  t‡ fdd„| ¡ D ƒƒr)t|ƒrtdƒ d| jv r(tdƒ | jd= nˆ d
|| j| j| j	| j
jdœ| j¤Ž}t|| j
ƒ t|| jƒ |S )z|Wraps the model into a :class:`~torch.distributed.fsdp.fully_sharded_data_parallel.FullyShardedDataParallel`
        module.r   ©ÚFullyShardedDataParallelc                 3   ó   � | ]}t |ˆ ƒV  qd S ry   ©r†   ©Ú.0Úmodr—   rm   rn   Ú	<genexpr>  ó   € z,FSDPStrategy.setup_module.<locals>.<genexpr>zYThe model is already wrapped in `FSDP` but there are still parameters on the meta device.rP   z_A FSDP `auto_wrap_policy` is set, but the model is already wrapped. The policy will be ignored.)r‘   rN   rO   rT   Ú	device_idNrm   )Útorch.distributed.fsdpr˜   ÚanyÚmodulesÚ_has_meta_device_parametersr:   rd   rN   r‡   rT   r{   ÚindexÚ_move_torchmetrics_to_deviceÚ_setup_activation_checkpointingrf   ©rj   r‘   rm   r—   rn   r•     s0   ÿ
ÿ€ûú	zFSDPStrategy.setup_moduleÚ	optimizerc                    s,   | j  d¡rtƒ  |¡S t|ƒstdƒ‚|S )aI  Set up an optimizer for a model wrapped with FSDP.

        This setup method doesn't modify the optimizer or wrap the optimizer. The only thing it currently does is verify
        that the optimizer was created after the model was wrapped with :meth:`setup_module` with a reference to the
        flattened parameters.

        r[   z�The optimizer does not seem to reference any FSDP parameters. HINT: Make sure to create the optimizer after setting up the model.)rd   r“   r\   Úsetup_optimizerÚ_optimizer_has_flat_paramsr”   )rj   r©   rk   rm   rn   rª   9  s   	ÿzFSDPStrategy.setup_optimizerc                 C   s   d S ry   rm   r¨   rm   rm   rn   Úmodule_to_deviceL  s   zFSDPStrategy.module_to_deviceÚ
empty_initc                 C   sb   | j  ¡ }|  ¡ }tt|ƒd�}tƒ }tr |r | t 	d¡¡ n| |¡ | |¡ | |¡ |S )N)ÚenabledÚmeta)
rK   Úmodule_init_contextÚmodule_sharded_contextr4   Úboolr   r1   Úenter_contextÚtorchÚdevice)rj   r­   Úprecision_init_ctxÚmodule_sharded_ctxZ	empty_ctxÚstackrm   rm   rn   r°   P  s   



z FSDPStrategy.module_init_contextc                 C   s>   ddl m} ddlm} |d|| j| j| j| jjdœ| j	¤ŽS )Nr   r—   )Úenable_wrap)Zwrapper_clsrN   rO   rT   r    rm   )
Ú2torch.distributed.fsdp.fully_sharded_data_parallelr˜   Útorch.distributed.fsdp.wrapr¹   rN   r‡   rT   r{   r¥   rd   )rj   r˜   r¹   rm   rm   rn   r±   a  s   ûúz#FSDPStrategy.module_sharded_contextÚmeanÚtensorr/   Ú	reduce_opc                 C   s   t |tƒrt|||d�S |S )N)r¾   )r†   r   r-   )rj   r½   r/   r¾   rm   rm   rn   Ú
all_reduceo  s   
zFSDPStrategy.all_reduceÚargsc                 O   s>   t ƒ sd S tj ¡ dkrtjj| jjgd� d S tj ¡  d S )NÚnccl)Ú
device_ids)r*   r´   ÚdistributedÚget_backendÚbarrierr{   r¥   )rj   rÀ   rX   rm   rm   rn   rÅ   w  s
   zFSDPStrategy.barrierr   ÚobjÚsrcc                 C   s,   t ƒ s|S |g}tjj||tjd� |d S )Nr.   r   )r*   r´   rÃ   Úbroadcast_object_listÚ_groupÚWORLD)rj   rÆ   rÇ   rm   rm   rn   Ú	broadcast€  s
   zFSDPStrategy.broadcastç       @TÚmax_normÚ	norm_typeÚerror_if_nonfinitec                 C   sN   ddl m} t||ƒstd| jj› d|jj› d�ƒ‚| j |¡ |j||d�S )zClip gradients by norm.r   r—   zFGradient clipping with FSDP is only possible if the module passed to `zE.clip_gradients_norm` is wrapped in `FullyShardedDataParallel`. Got: Ú.)rÍ   rÎ   )	rº   r˜   r†   r‹   rl   rt   rK   Úunscale_gradientsÚclip_grad_norm_)rj   r‘   r©   rÍ   rÎ   rÏ   r˜   rm   rm   rn   Úclip_gradients_norm‰  s   

ÿþÿz FSDPStrategy.clip_gradients_normÚpathÚstateÚstorage_optionsÚfilterc                 C   s¨  t stdƒ‚|durtdƒ‚|dur| jdkrtdƒ‚t|  |¡ƒ}| ¡ r6| jdkr6t|ƒs6td|› �ƒ‚dd	l	m
} d
d„ | ¡ D ƒ}t|ƒdkrOtdƒ‚t|ƒdkrYtdƒ‚|d }| jdkrÚ| ¡ rj| ¡  |jddd� t|ƒ}i }	i }
|�A | ¡ D ]4\}}t|tƒr�| ¡ }|	}nt|tƒrž| ||¡}|	}nt|tƒr§| ¡ n|}|
}t||p°i ||ƒ q€W d  ƒ n1 s¿w   Y  t|	|ƒ | jdkrØt |
|t ¡ dS dS | jdk�rLt|ƒrét  |¡ t!|| j"d�}i }|�? | ¡ D ]2\}}t|tƒ�r| ¡ }nt|tƒ�r| ||¡}nt|tƒ�r| ¡ n|}t||�p'i ||ƒ qùW d  ƒ n	1 �s7w   Y  | jdk�rJt ||¡ dS dS td| j› �ƒ‚)a  Save model, optimizer, and other state to a checkpoint on disk.

        If the state-dict-type is ``'full'``, the checkpoint will be written to a single file containing the weights,
        optimizer state and other metadata. If the state-dict-type is ``'sharded'``, the checkpoint gets saved as a
        directory containing one file per process, with model- and optimizer shards stored per file. Additionally, it
        creates a metadata file `meta.pt` with the rest of the user's state (only saved from rank 0).

        ú¶Saving and loading checkpoints with the `FSDPStrategy` is not supported in PyTorch < 2.0. Please upgrade `torch` or file an issue: `https://github.com/Lightning-AI/lightning/issues`.Nz�`FSDPStrategy.save_checkpoint(..., storage_options=...)` is not supported because `FSDPStrategy` does not use the `CheckpointIO`.rG   zVFSDP doesn't support loading sharded filtered checkpoints, so saving them is disabled.rW   z/The checkpoint path exists and is a directory: r   r—   c                 S   s   g | ]}t |ƒr|‘qS rm   ©Ú_has_fsdp_modules)rœ   r‘   rm   rm   rn   Ú
<listcomp>Æ  s    z0FSDPStrategy.save_checkpoint.<locals>.<listcomp>a  Could not find a FSDP model in the provided checkpoint state. Please provide the model as part of the state like so: `save_checkpoint(..., state={'model': model, ...})`. Make sure you set up the model (and optimizers if any) through the strategy before saving the checkpoint.rZ   záFound multiple FSDP models in the given state. Saving checkpoints with FSDP is currently limited to a single model per checkpoint. To save multiple models, call the save method for each model separately with a different path.T)ÚparentsÚexist_ok)Ú
world_sizezUnknown state_dict_type: )#r0   rr   r‹   rg   r   rË   Úis_dirÚ_is_sharded_checkpointÚIsADirectoryErrorr¡   r˜   Úvaluesr€   r”   Úis_fileÚunlinkÚmkdirÚ_get_sharded_state_dict_contextÚitemsr†   r   Ú
state_dictr   Úoptim_state_dictr=   r%   Ú_distributed_checkpoint_saver„   r´   Úsaver5   ÚshutilÚrmtreeÚ_get_full_state_dict_contextrÞ   )rj   rÔ   rÕ   rÖ   r×   ÚFSDPr£   r‘   Ústate_dict_ctxÚconverted_stateÚmetadataÚkeyrÆ   Ú	convertedÚtarget_dictZ
full_staterm   rm   rn   Úsave_checkpointŸ  sŠ   ÿÿÿÿÿ


õÿ

ÿ

ùÿ
ÿzFSDPStrategy.save_checkpointÚstrictc              	   C   s<  t stdƒ‚|std|›d�ƒ‚t|  |¡ƒ}t|tƒr't||| j|d� i S t|t	ƒr0tdƒ‚ddl
m} ddlm} dd	lm} d
d„ | ¡ D ƒ}t|ƒdkrUtdƒ‚dd„ | ¡ D ƒ}t|ƒdkrhtdƒ‚t| ¡ ƒd \}	}
t|ƒrýt|
ƒ}|�J |	|
 ¡ i}t||ƒ |
j||	 |d� |r¿ddlm} ||d�}| ¡ D ]\}}|||	 ||d�}|j|| |
|d�}| |¡ q¢W d  ƒ n1 sÉw   Y  t |t ¡}| ¡ | ¡  | ¡  }t|| ¡ |d� |D ]}||vróqì| |¡||< qì|S t |ƒ�r”t �r	t!|ƒntj|dd�}t"| |	¡|
| j|d� t|tƒ�r$i S t �r+t#|ƒ}| ¡ D ]F\}}t$|
| jdd��0 | |¡}tt|d  ¡ ƒd t%ƒ�rW| &||j'|
¡}|j||
|d�}| |¡ W d  ƒ n	1 �sow   Y  �q/| ¡ | ¡  | ¡  }t|| ¡ |d� t(|||d� |S tdt)|ƒ›d�ƒ‚)a  Load the contents from a checkpoint and restore the state of the given objects.

        The strategy currently only supports saving and loading sharded checkpoints which are stored in form of a
        directory of multiple files rather than a single file.

        rØ   z,Got FSDPStrategy.load_checkpoint(..., state=z—) but a state with at least  a model instance to reload is required. Pass it in like so: FSDPStrategy.load_checkpoint(..., state={'model': model, ...}))r‘   rÞ   r÷   z`Loading a single optimizer object from a checkpoint is not supported yet with the FSDP strategy.r   )Ú!load_sharded_optimizer_state_dictr—   )ÚOptimStateKeyTypec                 S   s   i | ]\}}t |ƒr||“qS rm   rÙ   )rœ   ró   r‘   rm   rm   rn   Ú
<dictcomp>0  s    z0FSDPStrategy.load_checkpoint.<locals>.<dictcomp>a  Could not find a FSDP model in the provided checkpoint state. Please provide the model as part of the state like so: `load_checkpoint(..., state={'model': model, ...})`. Make sure you set up the model (and optimizers if any) through the strategy before loading the checkpoint.c                 S   s    i | ]\}}t |tƒr||“qS rm   )r†   r   )rœ   ró   Úoptimrm   rm   rn   rú   7  s     rZ   zâFound multiple FSDP models in the given state. Loading checkpoints with FSDP is currently limited to a single model per checkpoint. To load multiple models, call the load method for each model separately with a different path.©r÷   ©ÚFileSystemReader©rÔ   )Úmodel_state_dictZoptimizer_keyÚstorage_reader)ré   Úmodelrû   NÚcpu)Úmap_locationF©rÞ   Ú
rank0_onlyrÕ   )ÚsourceÚdestinationÚkeysz	The path zœ does not point to a valid checkpoint. Make sure the path points to either a directory with FSDP checkpoint shards, or a single file with a full checkpoint.)*r0   rr   r”   r   rË   r†   r   Ú _load_raw_module_state_from_pathrÞ   r   Z&torch.distributed.checkpoint.optimizerrø   r¡   r˜   rù   rç   r€   Úlistrà   ræ   rè   Ú_distributed_checkpoint_loadÚload_state_dictÚtorch.distributed.checkpointrþ   Zoptim_state_dict_to_loadr´   Úloadr5   r	  r(   ÚpopÚ_is_full_checkpointr6   Ú_load_raw_module_stater7   rî   ÚintZrekey_optim_state_dictZ
PARAM_NAMEr8   Ústr)rj   rÔ   rÕ   r÷   rø   rï   rù   r£   r’   Ú
module_keyr‘   rð   Úmodule_staterþ   ÚreaderZ	optim_keyrû   Zoptim_stateZflattened_osdrò   Zrequested_metadata_keysró   Ú
checkpointZtemp_state_dictré   rm   rm   rn   Úload_checkpoint  s´   ÿ
ÿ

ÿÿÿ

ýý€é

ÿýò€ÿzFSDPStrategy.load_checkpointÚstrategy_registryc                 C   s4   t j ¡ sd S |jd| dd� |jd| ddd� d S )NrD   z+Fully Sharded Data Parallel (FSDP) training)ÚdescriptionrE   zQFully Sharded Data Parallel (FSDP) training with Full Sharding and CPU OffloadingT)r  rN   )r´   rÃ   Úis_availableÚregister)Úclsr  rm   rm   rn   Úregister_strategies•  s   
ý
üz FSDPStrategy.register_strategiesc                 C   s>   t ƒ  |  ¡  |  ¡ | _| jd usJ ‚t| j| j| jd� d S )N)rM   )r;   Ú_set_world_ranksÚ_get_process_group_backendr_   rJ   r,   r`   ru   rm   rm   rn   r�   §  s
   
zFSDPStrategy._setup_distributedc                 C   s   | j pt| jƒS ry   )r_   r+   r{   ru   rm   rm   rn   r!  ®  s   z'FSDPStrategy._get_process_group_backendc                 C   sJ   | j d ur| j  | j| j | j ¡ | j  | j| j ¡ | j t_	t
_	d S ry   )rJ   Úset_global_rankÚ	node_rankr�   rz   Úset_world_sizer~   r„   r   rƒ   Úutils_rank_zero_onlyru   rm   rm   rn   r   ±  s   
zFSDPStrategy._set_world_ranks)rY   Nry   )Nr¼   )r   )rÌ   T)NN)NT)Brt   Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r´   rµ   r   r   r  r   r   r²   r   r   r   r   r]   Úpropertyr   r   rv   Úsetterr{   r  r~   r�   r
   r…   rL   r‡   r    rK   rŽ   r�   r   r   r–   r•   rª   r¬   r	   r°   r±   r   r)   r¿   rÅ   r$   rË   ÚfloatrÓ   r<   r   rö   r  Úclassmethodr#   r  r�   r!  r   Ú__classcell__rm   rm   rk   rn   rF   \   s@   3òþýüûúùø	÷
öõôóòñð)ÿÿþ!ÿÿÿÿþúþý
ü
ûúùûþýüûúhüþ"ýü
û 
rF   rR   rS   rQ   rY   c                    s¸   | d u r
ˆ d u r
i S | d urˆ d urt dƒ‚| d ur>t| tƒr$t| ƒ‰n| f‰tr6td| › dtˆƒ› d�ƒ d‡fdd„iS tˆ tƒrRtrJtˆ i ƒS d‡ fdd„iS tsXt d	ƒ‚d
ˆ iS )NzeYou cannot set both `activation_checkpointing` and `activation_checkpointing_policy`. Use the latter.z'`FSDPStrategy(activation_checkpointing=zD)` is deprecated, use `FSDPStrategy(activation_checkpointing_policy=z)` instead.Úcheck_fnc                    s
   t | ˆ ƒS ry   rš   ©Ú	submodule)Úclassesrm   rn   Ú<lambda>Î  s   
 z2_activation_checkpointing_kwargs.<locals>.<lambda>c                    s   t | tˆ ƒƒS ry   )r†   Útupler/  )rS   rm   rn   r2  Ò  s    zW`activation_checkpointing_policy` requires torch >= 2.1.0. HINT: `pip install -U torch`rP   )r”   r†   r  r3  r1   r9   Úsetrc   )rR   rS   rm   )rS   r1  rn   rf   º  s0   ÿ

ÿÿ

rf   ÚpolicyrX   c                 C   sT   | d u r|S t | tƒr$trddlm} || ƒ} nddlm} t|| d�} | |d< |S )Nr   rA   )Útransformer_auto_wrap_policy)Ztransformer_layer_clsrP   )r†   r4  r1   r»   rB   r6  r   )r5  rX   rB   r6  rm   rm   rn   rc   Ø  s   

rc   r‘   Úactivation_checkpointing_kwargsc                    sx   |sd S ddl m‰  t‡ fdd„|  ¡ D ƒƒrtdƒ d S ddl m}m}m} ts0t	||j
d�}|| fd|i|¤Ž d S )	Nr   ©ÚCheckpointWrapperc                 3   r™   ry   rš   r›   r8  rm   rn   rž   ï  rŸ   z2_setup_activation_checkpointing.<locals>.<genexpr>ztFSDP checkpointing is configured, but the model already contains checkpointed layers. Checkpointing will be ignored.)ÚCheckpointImplÚapply_activation_checkpointingÚcheckpoint_wrapper)Zcheckpoint_implZcheckpoint_wrapper_fn)Z;torch.distributed.algorithms._checkpoint.checkpoint_wrapperr9  r¢   r£   r:   r:  r;  r<  r2   r   ZNO_REENTRANT)r‘   r7  r:  r;  r<  rm   r8  rn   r§   é  s   ÿr§   c                   @   s"   e Zd Zededefdd„ƒZdS )ra   r‘   rY   c                 C   s<   ddl m} t||ƒstd| jj› d|jj› d�ƒ‚| ¡ S )zuBlocks gradient synchronization inside the :class:`~torch.distributed.fsdp.FullyShardedDataParallel`
        wrapper.r   r—   zABlocking backward sync is only possible if the module passed to `zB.no_backward_sync` is wrapped in `FullyShardedDataParallel`. Got: rÐ   )rº   r˜   r†   r‹   rl   rt   Úno_sync)rj   r‘   r˜   rm   rm   rn   Úno_backward_sync  s   
ÿþÿz)_FSDPBackwardSyncControl.no_backward_syncN)rt   r&  r'  r   r   r	   r>  rm   rm   rm   rn   ra     s    ra   rN   r>   c                 C   s(   ddl m} t| |ƒr| S |t| ƒd�S )Nr   )r>   )Zoffload_params)r¡   r>   r†   r²   )rN   r>   rm   rm   rn   ri     s   ri   rT   rU   r@   c                 C   sŠ   ddl m} | d¡d ur| d¡d urtdƒ‚t| tƒr#||  ¡  n| }d|jv rC| d¡d u rC| d¡d u rC| d¡d u rCtdƒ‚|S )	Nr   )r@   Úprocess_groupÚdevice_meshznThe arguments `FSDPStrategy(process_group=..., device_mesh=...)` are mutually exclusive.Pass only one of them.ZHYBRIDrP   zŽThe hybrid sharding strategy requires you to pass at least one of the parameters: `auto_wrap_policy`, `process_group` tuple, or `device_mesh`.)	r¡   r@   r“   r”   r†   r  ÚupperÚnameÚRuntimeError)rT   rX   r@   Ústrategyrm   rm   rn   rh     s   ÿ
ÿrh   r©   c                 C   s   t dd„ | jD ƒƒS )Nc                 s   s*   � | ]}|d  D ]	}t |ddƒV  qqdS )ÚparamsZ_fsdp_flattenedFN)Úgetattr)rœ   r/   Úparamrm   rm   rn   rž   0  s   € ÿÿz-_optimizer_has_flat_params.<locals>.<genexpr>)r¢   Úparam_groups)r©   rm   rm   rn   r«   /  s   ÿr«   )NNNc                 C   sL   ddl m} ddlm}m}m} |dd�}|dd�}|j| |j||d�}|S )Nr   r—   )ÚShardedOptimStateDictConfigÚShardedStateDictConfigÚStateDictTypeT)Úoffload_to_cpu©r‘   rV   Ústate_dict_configÚoptim_state_dict_config)r¡   r˜   Útorch.distributed.fsdp.apirI  rJ  rK  rV   ZSHARDED_STATE_DICT)r‘   rï   rI  rJ  rK  rN  rO  Ústate_dict_type_contextrm   rm   rn   ræ   5  s   

üræ   TrÞ   r  c                 C   s~   ddl m}m} ddl m} |dkpt}|||d�}tr4ddlm} |||d�}	|j| |j	||	d�}
|
S |j| |j	|d�}
|
S )	Nr   )ÚFullStateDictConfigrK  r—   rZ   )rL  r  )ÚFullOptimStateDictConfigrM  )r‘   rV   rN  )
r¡   rR  rK  r˜   r1   r0   rP  rS  rV   ZFULL_STATE_DICT)r‘   rÞ   r  rR  rK  rï   rL  rN  rS  rO  rQ  rm   rm   rn   rî   D  s(   üûýrî   rÔ   c                 C   s   |   ¡ o	| t  ¡ S )z]A heuristic check to determine whether the path points to a directory with checkpoint shards.)rß   r5   rã   rÿ   rm   rm   rn   rà   a  s   rà   c                 C   s   |   ¡ S ry   )rã   rÿ   rm   rm   rn   r  f  s   r  c                    s0   ddl m‰  t| tƒot‡ fdd„|  ¡ D ƒƒS )Nr   r—   c                 3   r™   ry   rš   ©rœ   Úmr—   rm   rn   rž   m  rŸ   z$_has_fsdp_modules.<locals>.<genexpr>)r¡   r˜   r†   r   r¢   r£   )r‘   rm   r—   rn   rÚ   j  s   $rÚ   r÷   c                 C   s.   t | ƒstd| › �ƒ‚tt| ƒ|||d� dS )z;Loads the state dict from a file path into the FSDP module.zxFailed to load checkpoint directly into the model. The given path must be a single file containing the full state dict: )rè   r‘   rÞ   r÷   N)r  r”   r  r6   )rÔ   r‘   rÞ   r÷   rm   rm   rn   r
  p  s   ÿÿr
  rè   c                 C   sj   ddl m} t||ƒs|j| |d� dS t||dd�� |j| |d� W d  ƒ dS 1 s.w   Y  dS )zlLoads the state dict into the module by gathering all weights first and then and writing back to each shard.r   r—   rü   Fr  N)r¡   r˜   r†   r  rî   )rè   r‘   rÞ   r÷   rï   rm   rm   rn   r  {  s   
"ÿr  rÆ   c                 C   sR   t | tƒrtdd„ | jD ƒƒS t | tƒrtdd„ |  ¡ D ƒƒS tdt| ƒj› �ƒ‚)Nc                 s   s.   � | ]}|d  D ]}t |tƒr|jV  qqdS )rE  N)r†   r   Úis_meta)rœ   Úparam_groupÚtrm   rm   rn   rž   ˆ  s   € ÿ
ÿÿz._has_meta_device_parameters.<locals>.<genexpr>c                 s   s   � | ]}|j V  qd S ry   )rV  )rœ   rX  rm   rm   rn   rž   Œ  s   € z<Expected `torch.nn.Module` or `torch.optim.Optimizer`, got: )	r†   r   r¢   rH  r   Ú
parametersr‹   rs   rt   )rÆ   rm   rm   rn   r¤   †  s   
ÿ
r¤   rµ   c                    sB   t dƒsd S ddlm‰  ‡ fdd„|  ¡ D ƒD ]}| |¡ qd S )NÚtorchmetricsr   ©ÚMetricc                 3   s   � | ]
}t |ˆ ƒr|V  qd S ry   rš   rT  r[  rm   rn   rž   ˜  s   € z/_move_torchmetrics_to_device.<locals>.<genexpr>)r   rZ  r\  r£   Úto)r‘   rµ   Úmetricrm   r[  rn   r¦   �  s   ÿr¦   rñ   c                 C   sd   t rddlm} || |d� d S ddlm} trddlm} nddlm} ||dd�}|| |ƒ d S )Nr   )rë   ©Zcheckpoint_id)ÚFileSystemWriter)Úsave_state_dictT)rÔ   Zsingle_file_per_rank)r3   r  rë   r`  r2   ra  )rñ   rÔ   rë   r`  Úwriterrm   rm   rn   rê   œ  s   rê   r  c                 C   sb   t rddlm} || |d� d S ddlm} trddlm} nddlm} ||d�}|| |ƒ d S )Nr   )r  r_  rý   )r  rÿ   )r3   r  r  rþ   r2   r  )r  rÔ   r  rþ   r  rm   rm   rn   r  ¯  s   
r  )T)zrì   Ú
contextlibr   Údatetimer   Ú	functoolsr   Úpathlibr   Útypingr   r   r   r	   r
   r   r   r   r   r   r   r   r   r´   Ú lightning_utilities.core.importsr   Ú"lightning_utilities.core.rank_zeror   r%  r   Útorch.nnr   r   Útorch.optimr   Útyping_extensionsr   r   Úlightning.fabric.acceleratorsr   Úlightning.fabric.pluginsr   r   r   Ú5lightning.fabric.plugins.collectives.torch_collectiver   Ú'lightning.fabric.plugins.precision.fsdpr    Ú7lightning.fabric.strategies.launchers.subprocess_scriptr!   Ú$lightning.fabric.strategies.parallelr"   Ú$lightning.fabric.strategies.registryr#   Ú$lightning.fabric.strategies.strategyr$   r%   r&   r'   r(   Ú&lightning.fabric.utilities.distributedr)   r*   r+   r,   r-   r/   rÉ   Ú"lightning.fabric.utilities.importsr0   r1   r2   r3   Úlightning.fabric.utilities.initr4   Úlightning.fabric.utilities.loadr5   r6   r7   r8   Ú$lightning.fabric.utilities.rank_zeror9   r:   Úlightning.fabric.utilities.seedr;   Ú lightning.fabric.utilities.typesr<   r=   rº   r>   r?   r@   r»   rB   r²   r  rQ   rU   Ú_FSDP_ALIASESrF   rf   rc   r§   ra   ri   rh   r«   ræ   rî   rà   r  ÚobjectrÚ   r
  r  r  r¤   Únnrµ   r¦   rê   r  rm   rm   rm   rn   Ú<module>   s–   <&"    bÿþ
ýÿÿÿÿ
þ (
"