o
    žôjH  ã                
   @   sº  d dl Z d dlmZmZ d dlmZ d dlmZmZm	Z	m
Z
mZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/ edƒZ0edƒZ1e  2e3¡Z4G dd„ deƒZ5G dd„ deƒZ6G dd„ deƒZ7dee8 dee8 de9ddfdd „Z:d!e8d"e
e8ee8ege9f f d#e;d$e
e8ef ddf
d%d&„Z<dS )'é    N)ÚABCÚabstractmethod)Ú	ExitStack)
ÚAnyÚCallableÚContextManagerÚDictÚIterableÚListÚOptionalÚTupleÚTypeVarÚUnion)ÚTensor)ÚModule)Ú	Optimizer)Ú
DataLoader)ÚAccelerator)ÚCheckpointIO)ÚTorchCheckpointIO)Ú	Precision)Ú	_Launcher)Ú_StrategyRegistry)Úmove_data_to_device)Ú_TORCH_GREATER_EQUAL_2_0)Ú
_EmptyInit)Ú_PATHÚOptimizableÚReduceOpÚ	_StatefulÚ
TBroadcastÚTReducec                   @   sœ  e Zd ZdZ			dsdee dee dee ddfdd„Ze	e
dejfd	d
„ƒƒZe	e
defdd„ƒƒZe	dee fdd„ƒZe	dee fdd„ƒZejdeddfdd„ƒZe	defdd„ƒZejdeddfdd„ƒZe	defdd„ƒZejdee ddfdd„ƒZdtdd„Zdtdd„Zdedefdd„Zdefd d!„Zdud"ee defd#d$„Zd%ed&ee de eee f fd'd(„Z!d%edefd)d*„Z"d+edefd,d-„Z#e
d%eddfd.d/„ƒZ$dud0e%d1eej de%fd2d3„Z&d4e'd%ee d5e%d6e%ddf
d7d8„Z(d+e)d6e%de%fd9d:„Z*e
dvd4e'd<ee% d=ede'fd>d?„ƒZ+e
		@dwd4e,e'e%f d<ee% dAee,e-e.f  de,e'e%f fdBdC„ƒZ/e
dudDee. ddfdEdF„ƒZ0e
dxdHe1dIe2de1fdJdK„ƒZ3dydMedNedefdOdP„Z4		dzdQe5dRe6e.e,eee%f f dSee% dTee6e.e7e.e%gef f  ddf
dUdV„Z8d%ede6e.e,e%e'f f fdWdX„Z9	Ldyd%edYe6e.e,e%e'f f dZeddfd[d\„Z:d+ede6e.e'f fd]d^„Z;		Ld{dQe5dRee,eee6e.e,eee%f f f  dZede6e.e%f fd_d`„Z<dtdadb„Z=	c	Ld|d%ej>jd+edde,e?e2f dee,e?e2f dfedej'fdgdh„Z@d%ej>jd+edie,e?e2f ddfdjdk„ZAeBdleCddfdmdn„ƒZDde.fdodp„ZEdRe6e.e,eee%f f dTe6e.e7e.e%gef f de6e.e%f fdqdr„ZFdS )}ÚStrategyzcBase class for all strategies that change the behaviour of the training, validation and test- loop.NÚacceleratorÚcheckpoint_ioÚ	precisionÚreturnc                 C   s(   || _ || _d | _|| _d | _d | _d S ©N)Ú_acceleratorÚ_checkpoint_ioÚ
_precisionr%   Ú	_launcherÚ_backward_sync_control)Úselfr#   r$   r%   © r.   úw/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/strategies/strategy.pyÚ__init__-   s   
zStrategy.__init__c                 C   ó   dS )zReturns the root device.Nr.   ©r-   r.   r.   r/   Úroot_device;   ó    zStrategy.root_devicec                 C   r1   )zcWhether the current process is the rank zero process not only on the local node, but for all nodes.Nr.   r2   r.   r.   r/   Úis_global_zero@   r4   zStrategy.is_global_zeroc                 C   ó   | j S r'   )r+   r2   r.   r.   r/   ÚlauncherE   ó   zStrategy.launcherc                 C   r6   r'   ©r(   r2   r.   r.   r/   r#   I   r8   zStrategy.acceleratorc                 C   ó
   || _ d S r'   r9   )r-   r#   r.   r.   r/   r#   M   ó   
c                 C   s   | j d u r	tƒ | _ | j S r'   )r)   r   r2   r.   r.   r/   r$   Q   s   
zStrategy.checkpoint_ioÚioc                 C   r:   r'   )r)   )r-   r<   r.   r.   r/   r$   W   r;   c                 C   s   | j d ur| j S tƒ S r'   )r*   r   r2   r.   r.   r/   r%   [   s   zStrategy.precisionc                 C   r:   r'   )r*   )r-   r%   r.   r.   r/   r%   _   r;   c                 C   r1   )z&Attach the launcher based on Strategy.Nr.   r2   r.   r.   r/   Ú_configure_launcherc   r4   zStrategy._configure_launcherc                 C   s    | j dusJ ‚| j  | j¡ dS )zÄSetup any processes or distributed connections.

        This must be called by the framework at the beginning of every process, before any distributed communication
        takes place.

        N)r#   Úsetup_devicer3   r2   r.   r.   r/   Úsetup_environmentf   s   zStrategy.setup_environmentÚ
dataloaderc                 C   ó   |S )z“Wraps the dataloader if necessary.

        Args:
            dataloader: iterable. Ideally of type: :class:`torch.utils.data.DataLoader`

        r.   )r-   r@   r.   r.   r/   Úprocess_dataloaderp   s   zStrategy.process_dataloaderc                 C   s.   | j  ¡ }tƒ }tr| | j¡ | |¡ |S )z1Controls how tensors get created (device, dtype).)r%   Útensor_init_contextr   r   Úenter_contextr3   )r-   Zprecision_init_ctxÚstackr.   r.   r/   rC   y   s   

zStrategy.tensor_init_contextÚ
empty_initc                 C   sB   | j  ¡ }tƒ }tr| | j¡ | tt|ƒd�¡ | |¡ |S )a¨  A context manager wrapping the model instantiation.

        Here, the strategy can control how the parameters of the model get created (device, dtype) and or apply other
        patches to the model.

        Args:
            empty_init: Whether to initialize the model with empty weights (uninitialized memory).
                If ``None``, the strategy will decide. Some strategies may not support all options.

        )Úenabled)r%   Úmodule_init_contextr   r   rD   r3   r   Úbool)r-   rF   Zprecision_module_ctxrE   r.   r.   r/   rH   ‚   s   

zStrategy.module_init_contextÚmoduleÚ
optimizersc                    s$   ˆ   |¡}‡ fdd„|D ƒ}||fS )zÿSet up a model and multiple optimizers together.

        The returned objects are expected to be in the same order they were passed in. The default implementation will
        call :meth:`setup_module` and :meth:`setup_optimizer` on the inputs.

        c                    s   g | ]}ˆ   |¡‘qS r.   )Úsetup_optimizer)Ú.0Ú	optimizerr2   r.   r/   Ú
<listcomp>Ÿ   s    z8Strategy.setup_module_and_optimizers.<locals>.<listcomp>)Úsetup_module)r-   rJ   rK   r.   r2   r/   Úsetup_module_and_optimizers•   s   
	z$Strategy.setup_module_and_optimizersc                 C   rA   )zDPerforms setup for the model, e.g., by wrapping it by another class.r.   ©r-   rJ   r.   r.   r/   rP   ¢   ó   zStrategy.setup_modulerN   c                 C   rA   )zHPerforms setup for the optimizer, e.g., by wrapping it by another class.r.   ©r-   rN   r.   r.   r/   rL   ¦   rS   zStrategy.setup_optimizerc                 C   r1   )z&Moves the model to the correct device.Nr.   rR   r.   r.   r/   Úmodule_to_deviceª   r4   zStrategy.module_to_deviceÚbatchÚdevicec                 C   s   |p| j }t||ƒS )a&  Moves the batch to the correct device.

        The returned batch is of the same type as the input batch, just
        having all tensors on the correct device.

        Args:
            batch: The batch of samples to move to the correct device
            device: The target device

        )r3   r   )r-   rV   rW   r.   r.   r/   Úbatch_to_device®   s   

zStrategy.batch_to_deviceÚtensorÚargsÚkwargsc                 O   s<   | j  ||¡ | j j||g|¢R i |¤Ž | j  ||¡ dS )z0Forwards backward-calls to the precision plugin.N)r%   Úpre_backwardÚbackwardÚpost_backward)r-   rY   rJ   rZ   r[   r.   r.   r/   r]   ¼   s   zStrategy.backwardc                 K   s   | j j|fi |¤ŽS )zµPerforms the actual optimizer step.

        Args:
            optimizer: the optimizer performing the step
            **kwargs: Any extra arguments to ``optimizer.step``

        )r%   Úoptimizer_step)r-   rN   r[   r.   r.   r/   r_   Â   s   zStrategy.optimizer_stepFÚgroupÚ
sync_gradsc                 C   r1   )a  Perform an all_gather on all processes.

        Args:
            tensor: the tensor to all_gather
            group: the process group to gather results from
            sync_grads: flag that allows users to synchronize gradients for all_gather op

        Nr.   )r-   rY   r`   ra   r.   r.   r/   Ú
all_gatherÐ   r4   zStrategy.all_gatherÚmeanÚ	reduce_opc                 C   r1   )a,  Reduces the given tensor (e.g. across GPUs/processes).

        Args:
            tensor: the tensor to sync and reduce
            group: the process group to reduce
            reduce_op: the reduction operation. Defaults to 'mean'.
                Can also be a string 'sum' or ReduceOp.

        Nr.   )r-   rY   r`   rd   r.   r.   r/   Ú
all_reduceÛ   r4   zStrategy.all_reduceÚnamec                 C   r1   )z¯Synchronizes all processes which blocks processes until the whole group enters this function.

        Args:
            name: an optional name to pass into barrier.

        Nr.   )r-   rf   r.   r.   r/   Úbarrierì   r4   zStrategy.barrierr   ÚobjÚsrcc                 C   r1   )z…Broadcasts an object to all processes.

        Args:
            obj: the object to broadcast
            src: source rank

        Nr.   )r-   rh   ri   r.   r.   r/   Ú	broadcastõ   r4   zStrategy.broadcastTÚdecisionÚallc                 C   rA   )z/Reduce a boolean decision across all processes.r.   )r-   rk   rl   r.   r.   r/   Úreduce_boolean_decisionÿ   rS   z Strategy.reduce_boolean_decisionÚpathÚstateÚstorage_optionsÚfilterc                 C   s2   | j ||pi d�}| jr| jj|||d� dS dS )aÝ  Save model, optimizer, and other state as a checkpoint file.

        Args:
            path: A path to where the file(s) should be saved
            state: A dictionary with contents to be saved. If the dict contains modules or optimizers, their
                state-dict will be retrieved and converted automatically.
            storage_options: Additional options for the ``CheckpointIO`` plugin
            filter: An optional dictionary containing filter callables that return a boolean indicating whether the
                given item should be saved (``True``) or filtered out (``False``). Each filter key should match a
                state key, where its filter will be applied to the ``state_dict`` generated.

        )rq   )Ú
checkpointrn   rp   N)Ú"_convert_stateful_objects_in_stater5   r$   Úsave_checkpoint)r-   rn   ro   rp   rq   r.   r.   r/   rt     s   ÿzStrategy.save_checkpointc                 C   s   |  ¡ S )zReturns model state.)Ú
state_dictrR   r.   r.   r/   Úget_module_state_dict  s   zStrategy.get_module_state_dictru   Ústrictc                 C   s   |j ||d� dS )z%Loads the given state into the model.©rw   N)Úload_state_dict)r-   rJ   ru   rw   r.   r.   r/   Úload_module_state_dict  s   zStrategy.load_module_state_dictc                 C   s,   t |dƒr| ¡  | jr| ¡ S i S | ¡ S )z€Returns state of an optimizer.

        Allows for syncing/collating optimizer state from processes in custom plugins.

        Úconsolidate_state_dict)Úhasattrr{   r5   ru   rT   r.   r.   r/   Úget_optimizer_state$  s   
zStrategy.get_optimizer_statec                 C   sÚ   t j ¡  | j |¡}|s|S t|tƒr| j|||d� i S t|tƒr*| 	|¡ i S t
| ¡ | ¡ |d� | ¡  ¡ D ]/\}}||vrDq;t|tƒrct|tƒrZ| j|| |¡|d� q;| 	| |¡¡ q;| |¡||< q;|S )a�  Load the contents from a checkpoint and restore the state of the given objects.

        Args:
            path: A path to where the file is located
            state: Can be one of:

                - A dictionary of objects whose state will be restored in-place from the checkpoint path.
                - ``None`` or the empty dict: The loaded checkpoint will be returned in full.
                - A :class:`~torch.nn.Module` instance, if the checkpoint file contains a raw module state dict.
                - A :class:`~torch.optim.Optimizer` instance, if the checkpoint file contains a raw optimizer state.

            strict: Whether to enforce that the keys in `state` match the keys in the checkpoint.

        Returns:
            The remaining items that were not restored into the given state dictionary. If no state dictionary is
            given, the full checkpoint will be returned.

        )rJ   ru   rw   rx   )ÚtorchÚcudaÚempty_cacher$   Úload_checkpointÚ
isinstancer   rz   r   ry   Ú!_validate_keys_for_strict_loadingÚkeysÚcopyÚitemsr   Úpop)r-   rn   ro   rw   rr   rf   rh   r.   r.   r/   r�   3  s(   





zStrategy.load_checkpointc                 C   s0   | j  ¡  | jdusJ ‚| j ¡  | j ¡  dS )zŒThis method is called to teardown the training process.

        It is the right place to release memory and free other resources.

        N)r%   Úteardownr#   r$   r2   r.   r.   r/   rˆ   e  s   

zStrategy.teardownç       @Úmax_normÚ	norm_typeÚerror_if_nonfinitec                 C   s.   | j  |¡ | j  |¡}tjjj||||d�S )zClip gradients by norm.)rŠ   r‹   rŒ   )r%   Úunscale_gradientsÚmain_paramsr~   ÚnnÚutilsÚclip_grad_norm_)r-   rJ   rN   rŠ   r‹   rŒ   Ú
parametersr.   r.   r/   Úclip_gradients_normp  s
   	ÿzStrategy.clip_gradients_normÚclip_valc                 C   s*   | j  |¡ | j  |¡}tjjj||d�S )zClip gradients by value.)Ú
clip_value)r%   r�   rŽ   r~   r�   r�   Úclip_grad_value_)r-   rJ   rN   r”   r’   r.   r.   r/   Úclip_gradients_value  s   zStrategy.clip_gradients_valueÚstrategy_registryc                 C   s   d S r'   r.   )Úclsr˜   r.   r.   r/   Úregister_strategies…  rS   zStrategy.register_strategiesc                 C   s   dt | ƒj› d�S )NzThe `z£` does not support setting up the module and optimizer(s) independently. Please call `setup_module_and_optimizers(model, [optimizer, ...])` to jointly set them up.)ÚtypeÚ__name__r2   r.   r.   r/   Ú_err_msg_joint_setup_required‰  s   ÿz&Strategy._err_msg_joint_setup_requiredc                 C   sp   i }|  ¡ D ]/\}}t|tƒr| j|d�}nt|tƒr"| j|d�}nt|tƒr,| ¡ }n|}t||||ƒ q|S )N)rJ   )rN   )	r†   r‚   r   rv   r   r}   r   ru   Ú_apply_filter)r-   ro   rq   Zconverted_stateÚkeyrh   Ú	convertedr.   r.   r/   rs   �  s   



z+Strategy._convert_stateful_objects_in_state)NNN)r&   Nr'   )NF)Nrc   )r   )T)NN)NT)r‰   T)Grœ   Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r0   Úpropertyr   r~   rW   r3   rI   r5   r   r7   r#   Úsetterr$   r%   r=   r?   r   rB   r   rC   rH   r   r
   r   r   rQ   rP   rL   rU   r   rX   r   r]   r   r_   rb   r   r   Ústrre   rg   r    Úintrj   rm   r   r   r   rt   rv   rz   r}   r�   rˆ   r�   Úfloatr“   r—   Úclassmethodr   rš   r�   rs   r.   r.   r.   r/   r"   *   s   üþýü
û


		ÿÿ
þ"þý
ü"
ü
þýü
û	ûþýüû
ú"ÿÿÿÿ
þüþ"ýü

û
2úþý
ü
ûú
ù&ÿÿ
þr"   c                   @   s&   e Zd ZdZededefdd„ƒZdS )Ú_BackwardSyncControla†  Interface for any :class:`Strategy` that wants to offer a functionality to enable or disable gradient
    synchronization during/after back-propagation.

    The most common use-case is gradient accumulation. If a :class:`Strategy` implements this interface, the user can
    implement their gradient accumulation loop very efficiently by disabling redundant gradient synchronization.

    rJ   r&   c                 C   r1   )z©Blocks the synchronization of gradients during the backward pass.

        This is a context manager. It is only effective if it wraps a call to `.backward()`.

        Nr.   rR   r.   r.   r/   Úno_backward_syncª  r4   z%_BackwardSyncControl.no_backward_syncN)rœ   r¡   r¢   r£   r   r   r   r«   r.   r.   r.   r/   rª   ¡  s    rª   c                   @   s"   e Zd ZdZedefdd„ƒZdS )Ú_ShardedzkMixin-interface for any :class:`Strategy` that wants to expose functionality for sharding model parameters.r&   c                 C   r1   )a  A context manager that goes over the instantiation of an :class:`torch.nn.Module` and handles sharding of
        parameters on creation.

        By sharding layers directly on instantiation, one can reduce peak memory usage and initialization time.

        Nr.   r2   r.   r.   r/   Úmodule_sharded_context¶  r4   z_Sharded.module_sharded_contextN)rœ   r¡   r¢   r£   r   r   r­   r.   r.   r.   r/   r¬   ³  s    r¬   Úrequested_keysÚcheckpoint_keysrw   r&   c                    s6   ‡ fdd„| D ƒ}|r|rt d|d › d�ƒ‚d S d S )Nc                    s   g | ]}|ˆ vr|‘qS r.   r.   )rM   Úk©r¯   r.   r/   rO   Ã  s    z5_validate_keys_for_strict_loading.<locals>.<listcomp>z$The requested state contains a key 'r   z^' that does not exist in the loaded checkpoint. To disable strict loading, set `strict=False`.)ÚKeyError)r®   r¯   rw   Úinvalid_keysr.   r±   r/   rƒ   À  s   ÿÿrƒ   rŸ   rq   Úsource_dictÚtarget_dictc                 C   s^   | |v r)t |tƒr)||  }| ¡ D ]\}}|||ƒr&| | i ¡ |||  |< qd S ||| < d S r'   )r‚   Údictr†   Ú
setdefault)rŸ   rq   r´   rµ   Ú	filter_fnr°   Úvr.   r.   r/   rž   Ë  s   
€ürž   )=ÚloggingÚabcr   r   Ú
contextlibr   Útypingr   r   r   r   r	   r
   r   r   r   r   r~   r   Útorch.nnr   Útorch.optimr   Útorch.utils.datar   Úlightning.fabric.acceleratorsr   Ú)lightning.fabric.plugins.io.checkpoint_ior   Ú$lightning.fabric.plugins.io.torch_ior   Ú"lightning.fabric.plugins.precisionr   Ú.lightning.fabric.strategies.launchers.launcherr   Ú$lightning.fabric.strategies.registryr   Ú%lightning.fabric.utilities.apply_funcr   Ú"lightning.fabric.utilities.importsr   Zlightning.fabric.utilities.initr   Ú lightning.fabric.utilities.typesr   r   r   r   r    r!   Ú	getLoggerrœ   Úlogr"   rª   r¬   r¦   rI   rƒ   Úobjectrž   r.   r.   r.   r/   Ú<module>   s^   0
  yÿÿÿ
þÿÿÿ
ÿþ