o
    œôj@  ã                   @   sÄ   d dl mZmZmZmZmZ d dlZd dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZmZ d dlmZ d dlmZ G dd„ deƒZdS )é    )ÚAnyÚDictÚListÚOptionalÚUnionN)ÚTensor)ÚDataParallelÚModule)Úoverride)ÚAccelerator)ÚCheckpointIO)Ú	Precision)ÚParallelStrategy)Ú_StrategyRegistry)Ú
TBroadcastÚTReduce)Úapply_to_collection)ÚReduceOpc                       sÈ  e Zd ZdZ				d4dee deeej  dee	 dee
 f‡ fdd„Zeed	ejfd
d„ƒƒZeed5dd„ƒƒZeded	efdd„ƒZeded	dfdd„ƒZed6dedeej d	efdd„ƒZe	d7dedee deeeef  d	efdd„ƒZededed	dfdd „ƒZed8d"ed#ed	efd$d%„ƒZed9d'ed(ed	efd)d*„ƒZ eded	e!eeee"f f f‡ fd+d,„ƒZ#e	&d9ded-e!eeee"f f d.ed	df‡ fd/d0„ƒZ$e%ed1e&d	dfd2d3„ƒƒZ'‡  Z(S ):ÚDataParallelStrategyzŒImplements data-parallel training in a single process, i.e., the model gets replicated to each device and each
    gets a split of the data.NÚacceleratorÚparallel_devicesÚcheckpoint_ioÚ	precisionc                    s   t ƒ j||d ||d� d S )N)r   r   Úcluster_environmentr   r   )ÚsuperÚ__init__)Úselfr   r   r   r   ©Ú	__class__© úq/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/strategies/dp.pyr   #   s   
ûzDataParallelStrategy.__init__Úreturnc                 C   s   | j d usJ ‚| j d S )Nr   )r   ©r   r   r   r    Úroot_device2   s   
z DataParallelStrategy.root_devicec                 C   ó   d S ©Nr   r"   r   r   r    Údistributed_sampler_kwargs8   ó   z/DataParallelStrategy.distributed_sampler_kwargsÚmodulec                 C   s   t || jd�S )zDWraps the given model into a :class:`~torch.nn.DataParallel` module.)r(   Ú
device_ids)r   r   ©r   r(   r   r   r    Úsetup_module=   s   z!DataParallelStrategy.setup_modulec                 C   s   |  | j¡ d S r%   )Útor#   r*   r   r   r    Úmodule_to_deviceB   s   z%DataParallelStrategy.module_to_deviceÚbatchÚdevicec                 C   ó   |S r%   r   )r   r.   r/   r   r   r    Úbatch_to_deviceF   r'   z$DataParallelStrategy.batch_to_deviceÚmeanÚ
collectionÚgroupÚ	reduce_opc                 C   s   dt dt fdd„}t|t |ƒS )NÚtr!   c                 S   s   | j }|  ¡  ¡  |¡S r%   )ÚdtypeÚfloatr2   r,   )r6   Úoriginal_dtyper   r   r    r2   O   s   z-DataParallelStrategy.all_reduce.<locals>.mean)r   r   )r   r3   r4   r5   r2   r   r   r    Ú
all_reduceK   s   zDataParallelStrategy.all_reduceÚargsÚkwargsc                 O   r$   r%   r   )r   r;   r<   r   r   r    ÚbarrierU   ó   zDataParallelStrategy.barrierr   ÚobjÚsrcc                 C   r0   r%   r   )r   r?   r@   r   r   r    Ú	broadcastY   r>   zDataParallelStrategy.broadcastTÚdecisionÚallc                 C   r0   r%   r   )r   rB   rC   r   r   r    Úreduce_boolean_decision]   r>   z,DataParallelStrategy.reduce_boolean_decisionc                    s   t |tƒr|j}tƒ  |¡S r%   )Ú
isinstancer   r(   r   Úget_module_state_dictr*   r   r   r    rF   a   s   
z*DataParallelStrategy.get_module_state_dictÚ
state_dictÚstrictc                    s&   t |tƒr|j}tƒ j|||d� d S )N)r(   rG   rH   )rE   r   r(   r   Úload_module_state_dict)r   r(   rG   rH   r   r   r    rI   g   s   
z+DataParallelStrategy.load_module_state_dictÚstrategy_registryc                 C   s   |j d| | jd� d S )NÚdp)Údescription)ÚregisterÚ__name__)ÚclsrJ   r   r   r    Úregister_strategieso   s   z(DataParallelStrategy.register_strategies)NNNN)r!   Nr%   )Nr2   )r   )T))rN   Ú
__module__Ú__qualname__Ú__doc__r   r   r   Útorchr/   r   r   r   Úpropertyr
   r#   r&   r	   r   r+   r-   r   r1   r   r   r   Ústrr:   r=   r   ÚintrA   ÚboolrD   r   r   rF   rI   Úclassmethodr   rP   Ú__classcell__r   r   r   r    r      sx    ûþýüû ÿÿÿÿþ	(ÿÿÿÿþr   )Útypingr   r   r   r   r   rT   r   Útorch.nnr   r	   Útyping_extensionsr
   Úlightning.fabric.acceleratorsr   Ú)lightning.fabric.plugins.io.checkpoint_ior   Z"lightning.fabric.plugins.precisionr   Ú$lightning.fabric.strategies.parallelr   Ú$lightning.fabric.strategies.registryr   Ú$lightning.fabric.strategies.strategyr   r   Ú%lightning.fabric.utilities.apply_funcr   Ú&lightning.fabric.utilities.distributedr   r   r   r   r   r    Ú<module>   s   