o
    ŸôjØ,  ã                   @   s  d dl Z d dlmZmZmZmZmZmZmZ d dl	Z	d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZ d dlmZmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z'm(Z( er€d dl)m*Z* G dd„ deƒZ+dS )é    N)ÚTYPE_CHECKINGÚAnyÚCallableÚDictÚListÚOptionalÚUnion)ÚTensor)ÚModule)Ú	Optimizer)Ú
DataLoader)Úoverride)ÚAccelerator)Ú_XLA_GREATER_EQUAL_2_1)ÚXLAPrecision)ÚXLAEnvironment)ÚXLACheckpointIO)ÚParallelStrategyÚ_StrategyRegistry)Ú_XLALauncher)Ú
TBroadcast)Úrank_zero_only)Ú_PATHÚReduceOp©ÚMpDeviceLoaderc                       sÎ  e Zd ZdZ					dIdee deeej  dee	 dee
 ded	df‡ fd
d„Zeed	ejfdd„ƒƒZed	efdd„ƒZeed	e	fdd„ƒƒZejedee	 d	dfdd„ƒƒZeed	e
fdd„ƒƒZejedee
 d	dfdd„ƒƒZeed	ef‡ fdd„ƒƒZeed	ef‡ fdd„ƒƒZeed	ef‡ fdd„ƒƒZeed	ef‡ fdd„ƒƒZedJdd „ƒZedJ‡ fd!d"„ƒZed#ed	efd$d%„ƒZed#ed	dfd&d'„ƒZed(ed	d)fd*d+„ƒZedKd-e d.ee! d/ed	e fd0d1„ƒZ"e	dLd2e#e e!f d.ee! d3ee#e$e%f  d	e fd4d5„ƒZ&edMd6ee% d7e!d8e!d	dfd9d:„ƒZ'edNd<e(d=ed	e(fd>d?„ƒZ)e		dLd@e*dAe+e%e#ee,e!f f dBee! dCee+e%e-e%e!gef f  d	df
‡ fdDdE„ƒZ.e/edFe0d	dfdGdH„ƒƒZ1‡  Z2S )OÚXLAStrategyzxStrategy for training multiple TPU devices using the :func:`torch_xla.distributed.xla_multiprocessing.spawn`
    method.NTÚacceleratorÚparallel_devicesÚcheckpoint_ioÚ	precisionÚsync_module_statesÚreturnc                    s.   t ƒ j||tƒ ||d� d | _d| _|| _d S )N)r   r   Úcluster_environmentr   r    F)ÚsuperÚ__init__r   Ú_backward_sync_controlÚ	_launchedÚ_sync_module_states)Úselfr   r   r   r    r!   ©Ú	__class__© úr/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/strategies/xla.pyr%   +   s   û
zXLAStrategy.__init__c                 C   s(   | j stdƒ‚dd lm  m} | ¡ S )NzFAccessing the XLA device before processes have spawned is not allowed.r   )r'   ÚRuntimeErrorÚtorch_xla.core.xla_modelÚcoreÚ	xla_modelÚ
xla_device)r)   Úxmr,   r,   r-   Úroot_device>   s   zXLAStrategy.root_devicec                 C   s   | j d ur
t| j ƒS dS ©Nr   )r   Úlen©r)   r,   r,   r-   Únum_processesG   s   zXLAStrategy.num_processesc                 C   s&   | j }|d urt|tƒsJ ‚|S tƒ S ©N)Ú_checkpoint_ioÚ
isinstancer   ©r)   Úpluginr,   r,   r-   r   K   s
   zXLAStrategy.checkpoint_ioÚioc                 C   ó*   |d urt |tƒstd|› �ƒ‚|| _d S )NzHThe XLA strategy can only work with the `XLACheckpointIO` plugin, found )r;   r   Ú	TypeErrorr:   )r)   r>   r,   r,   r-   r   T   ó   
c                 C   s(   | j }|d urt|tƒsJ ‚|S tdƒS )Nz32-true)Ú
_precisionr;   r   r<   r,   r,   r-   r    [   s
   zXLAStrategy.precisionc                 C   r?   )NzEThe XLA strategy can only work with the `XLAPrecision` plugin, found )r;   r   r@   rB   )r)   r    r,   r,   r-   r    d   rA   c                    ó   | j rtƒ jS dS r5   )r'   r$   Úglobal_rankr7   r*   r,   r-   rD   k   ó   zXLAStrategy.global_rankc                    rC   r5   )r'   r$   Ú
local_rankr7   r*   r,   r-   rF   p   rE   zXLAStrategy.local_rankc                    rC   r5   )r'   r$   Ú	node_rankr7   r*   r,   r-   rG   u   rE   zXLAStrategy.node_rankc                    rC   )Né   )r'   r$   Ú
world_sizer7   r*   r,   r-   rI   z   rE   zXLAStrategy.world_sizec                 C   s   t | ƒ| _d S r9   )r   Ú	_launcherr7   r,   r,   r-   Ú_configure_launcher   s   zXLAStrategy._configure_launcherc                    sN   | j d usJ ‚t| j ƒdkrtdt| ƒj› d�ƒ‚d| _| jt_t	ƒ  
¡  d S )NrH   zThe zƒ does not support running on a single device with the PjRT runtime. Try using all devices or the `SingleDeviceXLAStrategy` strategyT)r   r6   ÚNotImplementedErrorÚtypeÚ__name__r'   rD   r   Úrankr$   Úsetup_environmentr7   r*   r,   r-   rP   ƒ   s   ÿzXLAStrategy.setup_environmentÚmodulec                 C   s0   | j rtrddlm} nddlm} ||ƒ |S )Nr   )Úbroadcast_master_param)r(   r   r/   rR   Ztorch_xla.experimental.pjrt)r)   rQ   rR   r,   r,   r-   Úsetup_module’   s   zXLAStrategy.setup_modulec                 C   s   |  | j¡ d S r9   )Útor4   )r)   rQ   r,   r,   r-   Úmodule_to_devicež   s   zXLAStrategy.module_to_deviceÚ
dataloaderr   c                 C   sD   ddl m} t||ƒr|S ||| jƒ}|jj|_t|jdd ƒ|_|S )Nr   r   Úbatch_sampler)Ú%torch_xla.distributed.parallel_loaderr   r;   r4   Ú_loaderÚdatasetÚgetattrrW   )r)   rV   r   r,   r,   r-   Úprocess_dataloader¢   s   

zXLAStrategy.process_dataloaderFÚtensorÚgroupÚ
sync_gradsc                 C   s    | j s|S t|tƒstdt| ƒj› d|› �ƒ‚| ¡ dkr"| d¡}|j}| 	| j
¡}ddlm  m} ddlm  m} |rD| |¡n| |¡}| 	|¡}|S )aC  Function to gather a tensor from several distributed processes.

        Args:
            tensor: tensor to all-gather.
            group: unused.
            sync_grads: flag that allows users to synchronize gradients for the all-gather operation.
        Return:
            A tensor of shape (world_size, ...)

        ú`z4.all_gather` is only implemented for tensors. Given r   N)r'   r;   r	   rL   rM   rN   ÚdimÚ	unsqueezeÚdevicerT   r4   Ztorch_xla.core.functionsr0   Ú	functionsr/   r1   Ú
all_gather)r)   r]   r^   r_   Úoriginal_deviceÚxfr3   r,   r,   r-   re   °   s   
ÿ

zXLAStrategy.all_gatherÚoutputÚ	reduce_opc                 C   sž   t |tƒstj|| jd�}t |tƒo|tjk}t |tƒo!| ¡ dv}|s&|r-t	d|› �ƒ‚dd l
m  m} | d|t¡}t |tƒrM| ¡ dv rM|| j }|S )N)rc   )ÚsumÚmeanÚavgz]Currently, the XLAStrategy only supports `sum`, `mean`, `avg` for the reduce operation, got: r   Úreduce)rl   rk   )r;   r	   Útorchr]   r4   r   ÚSUMÚstrÚlowerÚ
ValueErrorr/   r0   r1   Zmesh_reducerj   rI   )r)   rh   r^   ri   Zinvalid_reduce_opZinvalid_reduce_op_strr3   r,   r,   r-   Ú
all_reduceÎ   s   
ÿÿ
zXLAStrategy.all_reduceÚnameÚargsÚkwargsc                 O   s6   | j sd S dd lm  m} |d u rd}| |¡ d S )Nr   Ú )r'   r/   r0   r1   Ú
rendezvous)r)   rt   ru   rv   r3   r,   r,   r-   Úbarrierå   s   zXLAStrategy.barrierr   ÚobjÚsrcc                 C   sÖ   | j s|S dd lm  m} t|tƒ}|r*| ¡ dkr | d¡}|j}| 	| j
¡}nt ¡ }t ||¡ tjt| ¡ ƒ| j
tjd�}|g}|j||d� |d }|sdt | ¡  ¡  ¡ ¡}t |¡}|S | 	|¡}|S )Nr   )rc   Údtype)Zroot_ordinal)r'   r/   r0   r1   r;   r	   ra   rb   rc   rT   r4   r>   ÚBytesIOrn   Úsaver]   Ú	bytearrayÚ	getbufferÚfloatZcollective_broadcastÚcpuÚbyteÚnumpyÚload)r)   rz   r{   r3   Ú	is_tensorrf   Úbufferr,   r,   r-   Ú	broadcastð   s.   

ÿ

þzXLAStrategy.broadcastÚpathÚstateÚstorage_optionsÚfilterc                    s2   ddl m  m} | ¡  tƒ j||||d� dS )a„  Save model, optimizer, and other state as a checkpoint file.

        Args:
            path: A path to where the file(s) should be saved
            state: A dictionary with contents to be saved. If the dict contains modules or optimizers, their
                state-dict will be retrieved and converted automatically.
            storage_options: Additional options for the ``CheckpointIO`` plugin
            filter: An optional dictionary of the same format as ``state`` mapping keys to callables that return a
                boolean indicating whether the given parameter should be saved (``True``) or filtered out (``False``).

        r   N)r‹   rŒ   )r/   r0   r1   Z	mark_stepr$   Úsave_checkpoint)r)   r‰   rŠ   r‹   rŒ   r3   r*   r,   r-   r�     s   zXLAStrategy.save_checkpointÚstrategy_registryc                 C   s   |j d| | jd� d S )NÚxla)Údescription)ÚregisterrN   )ÚclsrŽ   r,   r,   r-   Úregister_strategies-  s   zXLAStrategy.register_strategies)NNNNT)r"   N)NF)NNr9   )r   )3rN   Ú
__module__Ú__qualname__Ú__doc__r   r   r   rn   rc   r   r   Úboolr%   Úpropertyr   r4   Úintr8   r   Úsetterr    rD   rF   rG   rI   rK   rP   r
   rS   rU   r   r\   r	   r   re   r   r   rp   rs   ry   r   rˆ   r   r   r   r   r�   Úclassmethodr   r“   Ú__classcell__r,   r,   r*   r-   r   '   sº    úþýüûúù"ÿ
ÿÿÿþ"
"ûþýüûúr   ),r>   Útypingr   r   r   r   r   r   r   rn   r	   Útorch.nnr
   Útorch.optimr   Útorch.utils.datar   Útyping_extensionsr   Úlightning.fabric.acceleratorsr   Ú!lightning.fabric.accelerators.xlar   Úlightning.fabric.pluginsr   Ú%lightning.fabric.plugins.environmentsr   Úlightning.fabric.plugins.io.xlar   Úlightning.fabric.strategiesr   r   Ú)lightning.fabric.strategies.launchers.xlar   Ú$lightning.fabric.strategies.strategyr   Ú$lightning.fabric.utilities.rank_zeror   Ú lightning.fabric.utilities.typesr   r   rX   r   r   r,   r,   r,   r-   Ú<module>   s*   $