o
    œôjn&  ã                   @   s`  d dl mZ d dlmZ d dlmZmZmZmZm	Z	m
Z
mZ d dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1m2Z2 d dl-m3Z4 d dl5mZ dZ6G dd„ de'ƒZ7G dd„ de,ƒZ8dS )é    )Únullcontext)Ú	timedelta)ÚAnyÚContextManagerÚDictÚListÚLiteralÚOptionalÚUnionN)Úrank_zero_only)ÚTensor)ÚModule)ÚDistributedDataParallel)Úoverride)ÚAccelerator)Údefault_pg_timeout)ÚClusterEnvironment)ÚCheckpointIO)Ú	Precision)Ú_MultiProcessingLauncher)Ú_SubprocessScriptLauncher)ÚParallelStrategy)Ú_StrategyRegistry)Ú
TBroadcastÚ_BackwardSyncControl)ÚReduceOpÚ_distributed_is_initializedÚ-_get_default_process_group_backend_for_deviceÚ_init_dist_connectionÚ_sync_ddp_if_available©Úgroup)Úddp_forkÚddp_notebookc                       sp  e Zd ZdZddddddedfdee deeej	  dee
 dee dee d	ee d
ee ded deddf‡ fdd„Zeedej	fdd„ƒƒZedefdd„ƒZejdeddfdd„ƒZedefdd„ƒZeedeeef fdd„ƒƒZedee fdd„ƒZedFdd„ƒZedF‡ fdd „ƒZed!edefd"d#„ƒZ ed!eddfd$d%„ƒZ!e	&dGd'e"d(ee d)ee#e$ef  de"fd*d+„ƒZ%ed,ededdfd-d.„ƒZ&edHd0e'd1ede'fd2d3„ƒZ(ed!edeee#ee"f f f‡ fd4d5„ƒZ)e	6dId!ed7eee#ee"f f d8e*ddf‡ fd9d:„ƒZ+e,ed;e-ddfd<d=„ƒƒZ.dFd>d?„Z/defd@dA„Z0dFdBdC„Z1deee  fdDdE„Z2‡  Z3S )JÚDDPStrategyzKStrategy for multi-process single-device training on one or multiple nodes.NÚpopenÚacceleratorÚparallel_devicesÚcluster_environmentÚcheckpoint_ioÚ	precisionÚprocess_group_backendÚtimeoutÚstart_method)r%   ÚspawnÚforkÚ
forkserverÚkwargsÚreturnc	           
         s@   t ƒ j|||||d� d| _|| _|| _|| _tƒ | _|	| _d S )N)r&   r'   r(   r)   r*   é   )	ÚsuperÚ__init__Ú
_num_nodesÚ_process_group_backendÚ_timeoutÚ_start_methodÚ_DDPBackwardSyncControlÚ_backward_sync_controlÚ_ddp_kwargs)
Úselfr&   r'   r(   r)   r*   r+   r,   r-   r1   ©Ú	__class__© úr/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/strategies/ddp.pyr5   7   s   û
zDDPStrategy.__init__c                 C   s   | j d usJ ‚| j | j S ©N)r'   Ú
local_rank©r=   r@   r@   rA   Úroot_deviceQ   s   zDDPStrategy.root_devicec                 C   ó   | j S rB   ©r6   rD   r@   r@   rA   Ú	num_nodesW   ó   zDDPStrategy.num_nodesrH   c                 C   s
   || _ d S rB   rG   )r=   rH   r@   r@   rA   rH   [   s   
c                 C   s   | j d ur
t| j ƒS dS )Nr   )r'   ÚlenrD   r@   r@   rA   Únum_processes`   s   zDDPStrategy.num_processesc                 C   s   | j | j | jdœS )N)Únum_replicasÚrank)rH   rK   Úglobal_rankrD   r@   r@   rA   Údistributed_sampler_kwargsd   s   z&DDPStrategy.distributed_sampler_kwargsc                 C   rF   rB   )r7   rD   r@   r@   rA   r+   i   rI   z!DDPStrategy.process_group_backendc                 C   sD   | j d usJ ‚| jdkrt| j | j| jƒ| _d S t| | jd�| _d S )Nr%   )r-   )r(   r9   r   rK   rH   Z	_launcherr   rD   r@   r@   rA   Ú_configure_launcherm   s   
zDDPStrategy._configure_launcherc                    s   t ƒ  ¡  |  ¡  d S rB   )r4   Úsetup_environmentÚ_setup_distributedrD   r>   r@   rA   rQ   u   s   
zDDPStrategy.setup_environmentÚmodulec                 C   sf   |   ¡ }|durtj tj ¡ ¡ntƒ }|� td||dœ| j¤ŽW  d  ƒ S 1 s,w   Y  dS )z^Wraps the model into a :class:`~torch.nn.parallel.distributed.DistributedDataParallel` module.N)rS   Ú
device_idsr@   )Ú_determine_ddp_device_idsÚtorchÚcudaÚstreamÚStreamr   r   r<   )r=   rS   rT   Úctxr@   r@   rA   Úsetup_modulez   s
    $ÿzDDPStrategy.setup_modulec                 C   s   |  | j¡ d S rB   )ÚtorE   ©r=   rS   r@   r@   rA   Úmodule_to_deviceƒ   s   zDDPStrategy.module_to_deviceÚmeanÚtensorr!   Ú	reduce_opc                 C   s   t |tƒrt|||d�S |S )a  Reduces a tensor from several distributed processes to one aggregated tensor.

        Args:
            tensor: the tensor to sync and reduce
            group: the process group to gather results from. Defaults to all processes (world)
            reduce_op: the reduction operation. Defaults to 'mean'/'avg'.
                Can also be a string 'sum' to calculate the sum during reduction.

        Return:
            reduced value, except when the input was not a tensor the output remains is unchanged

        )ra   )Ú
isinstancer   r   )r=   r`   r!   ra   r@   r@   rA   Ú
all_reduce‡   s   
zDDPStrategy.all_reduceÚargsc                 O   s<   t ƒ sd S tj ¡ dkrtjj|  ¡ d� d S tj ¡  d S )NÚnccl)rT   )r   rV   ÚdistributedÚget_backendÚbarrierrU   )r=   rd   r1   r@   r@   rA   rh   ›   s
   zDDPStrategy.barrierr   ÚobjÚsrcc                 C   s,   t ƒ s|S |g}tjj||tjd� |d S )Nr    r   )r   rV   rf   Úbroadcast_object_listÚ_groupÚWORLD)r=   ri   rj   r@   r@   rA   Ú	broadcast¤   s
   zDDPStrategy.broadcastc                    s   t |tƒr|j}tƒ  |¡S rB   )rb   r   rS   r4   Úget_module_state_dictr]   r>   r@   rA   ro   ­   s   
z!DDPStrategy.get_module_state_dictTÚ
state_dictÚstrictc                    s&   t |tƒr|j}tƒ j|||d� d S )N)rS   rp   rq   )rb   r   rS   r4   Úload_module_state_dict)r=   rS   rp   rq   r>   r@   rA   rr   ³   s   
z"DDPStrategy.load_module_state_dictÚstrategy_registryc                 C   s0   d}|D ]\}}|j || d|›d�|d� qd S )N))Úddpr%   )Z	ddp_spawnr.   )r"   r/   )r#   r/   z DDP strategy with `start_method=ú`)Údescriptionr-   )Úregister)Úclsrs   ÚentriesÚnamer-   r@   r@   rA   Úregister_strategies»   s   
üÿzDDPStrategy.register_strategiesc                 C   s8   |   ¡  |  ¡ | _| jd usJ ‚t| j| j| jd� d S )N)r,   )Ú_set_world_ranksÚ_get_process_group_backendr7   r(   r   r8   rD   r@   r@   rA   rR   Ì   s   
zDDPStrategy._setup_distributedc                 C   s   | j pt| jƒS rB   )r7   r   rE   rD   r@   r@   rA   r}   Ò   s   z&DDPStrategy._get_process_group_backendc                 C   sJ   | j d ur| j  | j| j | j ¡ | j  | j| j ¡ | j t_	t
_	d S rB   )r(   Úset_global_rankÚ	node_rankrK   rC   Úset_world_sizerH   rN   r   rM   Úutils_rank_zero_onlyrD   r@   r@   rA   r|   Õ   s   
zDDPStrategy._set_world_ranksc                 C   s   | j jdkrd S | j jgS )NÚcpu)rE   ÚtypeÚindexrD   r@   r@   rA   rU   Ý   s   z%DDPStrategy._determine_ddp_device_ids)r2   N)Nr_   )r   )T)4Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r	   r   r   rV   Údevicer   r   r   Ústrr   r   r   r5   Úpropertyr   rE   ÚintrH   ÚsetterrK   r   rO   r+   rP   rQ   r   r   r[   r^   r   r
   r   rc   rh   r   rn   ro   Úboolrr   Úclassmethodr   r{   rR   r}   r|   rU   Ú__classcell__r@   r@   r>   rA   r$   4   s°    ÷þýüûúùø	÷
öõÿÿÿÿþ(ÿÿÿÿþ

r$   c                   @   s"   e Zd Zededefdd„ƒZdS )r:   rS   r2   c                 C   s0   t |tƒstd| jj› d|jj› d�ƒ‚| ¡ S )z{Blocks gradient synchronization inside the :class:`~torch.nn.parallel.distributed.DistributedDataParallel`
        wrapper.zABlocking backward sync is only possible if the module passed to `zA.no_backward_sync` is wrapped in `DistributedDataParallel`. Got: Ú.)rb   r   Ú	TypeErrorr?   r…   Úno_syncr]   r@   r@   rA   Úno_backward_syncâ   s   
ÿþÿz(_DDPBackwardSyncControl.no_backward_syncN)r…   r†   r‡   r   r   r   r”   r@   r@   r@   rA   r:   á   s    r:   )9Ú
contextlibr   Údatetimer   Útypingr   r   r   r   r   r	   r
   rV   Útorch.distributedÚ"lightning_utilities.core.rank_zeror   r�   r   Útorch.nnr   Ztorch.nn.parallel.distributedr   Útyping_extensionsr   Ú)lightning.fabric.accelerators.acceleratorr   Z5lightning.fabric.plugins.collectives.torch_collectiver   Ú9lightning.fabric.plugins.environments.cluster_environmentr   Ú)lightning.fabric.plugins.io.checkpoint_ior   Z"lightning.fabric.plugins.precisionr   Z5lightning.fabric.strategies.launchers.multiprocessingr   Z7lightning.fabric.strategies.launchers.subprocess_scriptr   Ú$lightning.fabric.strategies.parallelr   Ú$lightning.fabric.strategies.registryr   Ú$lightning.fabric.strategies.strategyr   r   Ú&lightning.fabric.utilities.distributedr   r   r   r   r   r!   rl   Ú$lightning.fabric.utilities.rank_zeroÚ_DDP_FORK_ALIASESr$   r:   r@   r@   r@   rA   Ú<module>   s6   $ .