o
    ©ôjÉl  ã                   @   s´  d dl Z d dlmZ d dlmZmZmZmZmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZmZmZmZmZmZmZmZ d dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d dl'm(Z( d dl)m*Z* d dl+m,Z,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9 d dl:m;Z; d dl<m=Z=m>Z> d dl?m@Z@mAZA d dlBmCZC d dlDmEZE eee!ef ZFG dd„ dƒZGdee, dee. fdd„ZHdeIfdd„ZJdS )é    N)ÚCounter)ÚAnyÚDictÚListÚOptionalÚUnionÚcast)Úget_args)ÚACCELERATOR_REGISTRY)ÚAccelerator)ÚCUDAAccelerator)ÚMPSAccelerator)ÚXLAAccelerator)ÚBitsandbytesPrecisionÚCheckpointIOÚDeepSpeedPrecisionÚHalfPrecisionÚMixedPrecisionÚ	PrecisionÚTransformerEnginePrecisionÚXLAPrecision)ÚClusterEnvironmentÚLightningEnvironmentÚLSFEnvironmentÚMPIEnvironmentÚSLURMEnvironmentÚTorchElasticEnvironment)ÚDoublePrecision)ÚFSDPPrecision)Ú_PRECISION_INPUTÚ_PRECISION_INPUT_INTÚ_PRECISION_INPUT_STRÚ_PRECISION_INPUT_STR_ALIASÚ%_PRECISION_INPUT_STR_ALIAS_CONVERSION)ÚSTRATEGY_REGISTRYÚDeepSpeedStrategyÚParallelStrategyÚSingleDeviceStrategyÚSingleDeviceXLAStrategyÚStrategyÚXLAFSDPStrategyÚXLAStrategy)Ú_DDP_FORK_ALIASES)Ú_FSDP_ALIASESÚFSDPStrategy)Úrank_zero_infoÚrank_zero_warn)Ú_determine_root_gpu_device)Ú_IS_INTERACTIVEc                   @   s~  e Zd ZdZ						d+deeef deeef deee	 ee	f de	d	e
e d
e
eeee f  ddfdd„Zdeeef deeef d	e
e d
e
eeee f  ddf
dd„Zdeee	 ee	f de	ddfdd„Zdefdd„Zedefdd„ƒZd,dd„Zd,dd„Zdefdd„Zdeeef fdd„Zd,dd„Zd,d d!„Zdefd"d#„Zd,d$d%„Zed&ed'ed(edefd)d*„ƒZdS )-Ú
_Connectorad  The Connector parses several Fabric arguments and instantiates the Strategy including its owned components.

        A. accelerator flag could be:
            1. accelerator class
            2. accelerator str
            3. accelerator auto

        B. strategy flag could be:
            1. strategy class
            2. strategy str registered with STRATEGY_REGISTRY
            3. strategy str in _strategy_type enum which listed in each strategy as
               backend (registed these too, and _strategy_type could be deprecated)

        C. plugins flag could be:
            1. precision class (should be removed, and precision flag should allow user pass classes)
            2. checkpoint_io class
            3. cluster_environment class

    priorities which to take when:
        A. Class > str
        B. Strategy > Accelerator/precision/plugins

    Úautoé   NÚacceleratorÚstrategyÚdevicesÚ	num_nodesÚ	precisionÚpluginsÚreturnc                 C   s(  | j d|dd�}| j d|dd�}| j d|dd�}t| j d|dd�ƒ}| j d|d d�}t ¡ | _t ¡ | _d| _d| _	d	| _
d | _d | _g | _d | _| j||||d
� | j||d� | j	dkrd|  ¡ | _	n
| j	dkrn|  ¡ | _	|  ¡  |  ¡ | _| jdkr�|  ¡ | _|  ¡  |  ¡  |  ¡ | _|  ¡  d S )Nr6   r4   )Údefaultr7   r8   r9   r5   r:   ú32-true)r7   r6   r:   r;   )r8   r9   Úgpu)Ú_argument_from_envÚintr$   Zavailable_strategiesÚ_registered_strategiesr
   Úavailable_acceleratorsÚ_registered_acceleratorsÚ_strategy_flagÚ_accelerator_flagÚ_precision_inputÚ_precision_instanceÚ_cluster_environment_flagÚ_parallel_devicesÚcheckpoint_ioÚ!_check_config_and_set_final_flagsÚ(_check_device_config_and_set_final_flagsÚ_choose_auto_acceleratorÚ_choose_gpu_accelerator_backendÚ*_set_parallel_devices_and_init_acceleratorÚ$_choose_and_init_cluster_environmentÚcluster_environmentÚ_choose_strategyÚ_check_strategy_and_fallbackÚ_init_strategyÚ_check_and_init_precisionr:   Ú_lazy_init_strategy)Úselfr6   r7   r8   r9   r:   r;   © rY   úm/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/connector.pyÚ__init__a   sB   


ü






z_Connector.__init__c                 C   sœ  |durt |tƒs|gn|}t |tƒr| ¡ }|| _|dkr0|| jvr0t |tƒs0td|›d�ƒ‚|| jvrM|dvrMt |t	ƒsMtd|›dd 
| j¡› d	�ƒ‚t |tƒoUd
|v }t |tƒo^d|v }t |tƒogd|v }t |tƒpr|pr|pr|}t ¡ o|dv pt |tƒ}	|	rŒ|rŒtd|› d�ƒ‚|| _t|ƒ}
|�r	tƒ }|D ]?}t |tƒr¯|| _|tj  d7  < q›t |tƒrÁ|| _|tj  d7  < q›t |tƒrÓ|| _|tj  d7  < q›td|› d�ƒ‚dd„ | ¡ D ƒ}|rñtdd 
|¡› d�ƒ‚| tj¡�r	|
du�r	td|
› d| j› d�ƒ‚|
du �rdn|
| _t | jtƒ�rÈ| jj�r.| jdk�r)tdƒ‚| jj| _| jj�r@| j�r;tdƒ‚| jj| _| jj�rR| j�rMtdƒ‚| jj| _t| jddƒ�ri| j�rbtdƒ‚t| jdƒ| _t | jd ƒ�rÊ| jj!�rÌ| jj!d! j"d"k�r›| j�r˜| jd#v�r˜td$| jj#j› d%| j› d&�ƒ‚d"| _| jj!d! j"d'k�rÁ| j�r¾| jd(v�r¾td)| jj#j› d%| j› d&�ƒ‚d'| _| jj!| _$dS dS dS dS )*a¯  This method checks:

        1. strategy: whether the strategy name is valid, and sets the internal flags if it is.
        2. accelerator: if the value of the accelerator argument is a type of accelerator (instance or string),
            set self._accelerator_flag accordingly.
        3. precision: The final value of the precision flag may be determined either by the precision argument or
            by a plugin instance.
        4. plugins: The list of plugins may contain a Precision plugin, CheckpointIO, ClusterEnvironment and others.
            Additionally, other flags such as `precision` can populate the list with the
            corresponding plugin instances.

        Nr4   z1You selected an invalid strategy name: `strategy=zÞ`. It must be either a string or an instance of `lightning.fabric.strategies.Strategy`. Example choices: auto, ddp, ddp_spawn, deepspeed, dp, ... Find a complete list of options in our documentation at https://lightning.ai)r4   r?   z7You selected an invalid accelerator name: `accelerator=z`. Available names are: auto, z, Ú.ÚddpÚdpÚ	deepspeed)Úmpsr4   r?   NzYou set `strategy=z�` but strategies from the DDP family are not supported on the MPS accelerator. Either explicitly set `accelerator='cpu'` or change the strategy.r5   zFound invalid type for plugin z>. Expected one of: Precision, CheckpointIO, ClusterEnviroment.c                 S   s   g | ]
\}}|d kr|‘qS )r5   rY   )Ú.0ÚkÚvrY   rY   rZ   Ú
<listcomp>ó   s    z@_Connector._check_config_and_set_final_flags.<locals>.<listcomp>zReceived multiple values for z> flags in `plugins`. Expected one value for each type at most.zReceived both `precision=z` and `plugins=z`. Choose one.r>   zLaccelerator set through both strategy class and accelerator flag, choose onezAprecision set through both strategy class and plugins, choose onezEcheckpoint_io set through both strategy class and plugins, choose onerR   zKcluster_environment set through both strategy class and plugins, choose oneÚparallel_devicesr   Úcpu)r4   rf   z!CPU parallel_devices set through z class, but accelerator set to z, please choose one device typeÚcuda)r4   rg   r?   z!GPU parallel_devices set through )%Ú
isinstanceÚlistÚstrÚlowerrE   rB   r)   Ú
ValueErrorrD   r   Újoinr&   r   Úis_availablerF   Ú"_convert_precision_to_unified_argsr   r   rH   Ú__name__r   rK   r   rI   Ú	TypeErrorÚitemsÚgetrG   Ú_acceleratorÚ
_precisionZ_checkpoint_ioÚgetattrÚhasattrre   ÚtypeÚ	__class__rJ   )rX   r7   r6   r:   r;   Z
is_ddp_strZ	is_dp_strZis_deepspeed_strZis_parallel_strategyZis_mps_acceleratorZprecision_inputZplugins_flags_typesÚpluginZduplicated_plugin_keyrY   rY   rZ   rL   ¢   sº   

ÿ
ÿ
ÿÿÿ
ÿ



ÿÿÿ





ÿÿÿÿÞz,_Connector._check_config_and_set_final_flagsc                 C   sv   t |tƒr	|dk rtd|› d�ƒ‚|| _|| _| jg ddfv r9t | jtƒr*| jjjn| j}td| j›d|› d�ƒ‚d S )	Nr5   z0`num_nodes` must be a positive integer, but got r\   r   Ú0z`Fabric(devices=z$)` value is not a valid input using z accelerator.)	rh   rA   rl   Ú_num_nodes_flagÚ_devices_flagrF   r   ry   Ú__qualname__)rX   r8   r9   Úaccelerator_namerY   rY   rZ   rM   )  s   
ÿý
ÿÿúz3_Connector._check_device_config_and_set_final_flagsc                 C   s(   t  ¡ rdS t ¡ rdS t ¡ rdS dS )zTChoose the accelerator type (str) based on availability when ``accelerator='auto'``.Útpur`   rg   rf   )r   rn   r   r   ©rX   rY   rY   rZ   rN   ;  s   z#_Connector._choose_auto_acceleratorc                   C   s    t  ¡ rdS t ¡ rdS tdƒ‚)Nr`   rg   zNo supported gpu backend found!)r   rn   r   ÚRuntimeErrorrY   rY   rY   rZ   rO   E  s
   z*_Connector._choose_gpu_accelerator_backendc                 C   sœ   t | jtƒr| j| _n| jd usJ ‚t | j¡| _| jj}| ¡ s5dd„ | jD ƒ}t	d|j
› d|› d�ƒ‚|  ¡  | | j¡| _| jsL| | j¡| _d S d S )Nc                 S   s    g | ]}t | d   ¡ r|‘qS )r6   )r
   rn   )ra   Zacc_strrY   rY   rZ   rd   V  s    ýÿzI_Connector._set_parallel_devices_and_init_accelerator.<locals>.<listcomp>ú`zª` can not run on your system since the accelerator is not available. The following accelerator(s) is available and can be passed into `accelerator` argument of `Fabric`: r\   )rh   rF   r   r6   r
   rs   ry   rn   rD   r‚   r~   Ú _set_devices_flag_if_auto_passedÚparse_devicesr}   rJ   Úget_parallel_devices)rX   Zaccelerator_clsZavailable_acceleratorrY   rY   rZ   rP   M  s&   
þ
ýÿÿz5_Connector._set_parallel_devices_and_init_acceleratorc                 C   sh   | j dkrd S tr,t| jtƒr,| j ¡ dkr,d| _ td| j ¡ › d| j ¡ › d�ƒ d S | j ¡ | _ d S )Nr4   r5   zFabric will use only 1 of zm GPUs because it is running inside an interactive / notebook environment. You may try to set `Fabric(devices=zŒ)` but please note that multi-GPU inside interactive / notebook environments is considered experimental and unstable. Your mileage may vary.)r}   r2   rh   r6   r   Úauto_device_countr/   r�   rY   rY   rZ   r„   h  s   
ÿ
þþÿz+_Connector._set_devices_flag_if_auto_passedc                 C   s<   t | jtƒr	| jS ttttfD ]}| ¡ r|ƒ   S qtƒ S )N)	rh   rI   r   r   r   r   r   Údetectr   )rX   Zenv_typerY   rY   rZ   rQ   z  s   û
ÿz/_Connector._choose_and_init_cluster_environmentc                 C   s´   | j dkst| j tƒr| jrt| jƒdkrdS t| jd d�S | jdkr&dS t| jƒdkrMt| j ttfƒs@t| j t	ƒrF| j dv rFt
| jƒ}nd}t|d�S t| jƒdkrXtrXd	S dS )
Nr€   r5   Úxlar   )Údevicer]   )rg   r?   r`   rf   Zddp_fork)rF   rh   r   rJ   Úlenr(   r|   r   r   rj   r1   r'   r2   ©rX   rŠ   rY   rY   rZ   rS   ˆ  s"   

ÿ

z_Connector._choose_strategyc                 C   s®   t | jtƒrdn| j}|dkr| jdkrd}|dkr(| jdkr(t|›d�ƒ d}|tv r;d	tj ¡ vr;t	d
|› d�ƒ‚|t
v sEt | jtƒrN| jdvrNt	dƒ‚|rU|| _dS dS )zªChecks edge cases when the strategy selection was a string input, and we need to fall back to a different
        choice depending on other parameters or the environment.Ú Úfsdpr€   Zxla_fsdpr^   rf   z: is not supported on CPUs, hence setting `strategy='ddp'`.r]   ÚforkzYou selected `Fabric(strategy='zn')` but process forking is not supported on this platform. We recommed `Fabric(strategy='ddp_spawn')` instead.)rg   r?   z’You selected the FSDP strategy but FSDP is only available on GPU. Set `Fabric(accelerator='gpu', ...)` to continue or select a different strategy.N)rh   rE   r)   rF   r0   r,   ÚtorchÚmultiprocessingÚget_all_start_methodsrl   r-   r.   )rX   Zstrategy_flagrY   rY   rZ   rT   �  s$   
ÿ
ÿ
ÿz'_Connector._check_strategy_and_fallbackc                 C   s>   t | jttfƒs
J ‚t | jtƒrt | j¡| _dS | j| _dS )zNInstantiate the Strategy given depending on the setting of ``_strategy_flag``.N)rh   rE   rj   r)   r$   rs   r7   r�   rY   rY   rZ   rU   ¹  s   z_Connector._init_strategyc                 C   sF  t | jtƒrt | jtƒrt | jtƒstdƒ‚| jS t | jtt	t
fƒr't| jƒS t | jtƒr2t| jƒS t | jtƒr>t| jd�S | jdv rHt| jƒS | jdkrPtƒ S | jdkrXtƒ S | jdkrcttjd�S | jdkrnttjd�S | jd	kr| jd
krtdƒ d| _| jdv rŸt| jd	krŒdndƒ | jd
kr–d
nd}t| j|d�S tdƒ‚)Nz,Bitsandbytes is only supported on CUDA GPUs.)r:   )z16-truez	bf16-truer>   z64-trueztransformer-engine)Zweights_dtypeztransformer-engine-float16ú16-mixedrf   z�You passed `Fabric(accelerator='cpu', precision='16-mixed')` but AMP with fp16 is not supported on CPU. Using `precision='bf16-mixed'` instead.ú
bf16-mixed)r“   r”   z,Using 16-bit Automatic Mixed Precision (AMP)z.Using bfloat16 Automatic Mixed Precision (AMP)rg   )r:   rŠ   zNo precision set)rh   rH   r   r   r6   r   r‚   r7   r(   r+   r*   r   rG   r%   r   r.   r   r   r   r   r�   Úbfloat16Úfloat16rF   r0   r/   r   rŒ   rY   rY   rZ   rV   Â  sH   ÿ







ÿ

ÿýz$_Connector._check_and_init_precisionc                 C   s  | j | j_ | jr| j| j_| jr| j| j_t| jdƒr+| jjdu r&| j| j_| jj| _t| jdƒr@| jjr;| jj| _n| j| j_t| jdƒrK| j| j_	t| jdƒrV| j 
¡  | j ¡  tro| jjro| jjjsotd| j›d�ƒ‚t| j tƒr‰t| jtttfƒs‹td| jjj› d	�ƒ‚dS dS )
zFLazily set missing attributes on the previously instantiated strategy.rR   Nre   r9   Ú_set_world_ranksz`Fabric(strategy=a  )` is not compatible with an interactive environment. Run your code as a script, or choose one of the compatible strategies: `Fabric(strategy='dp'|'ddp_notebook')`. In case you are spawning processes yourself, make sure to include the Fabric creation inside the worker function.zsThe `XLAAccelerator` can only be used with a `SingleDeviceXLAStrategy`, `XLAStrategy`, or `XLAFSDPStrategy`. Found r\   )r6   r7   r:   rK   rw   rR   re   rJ   r|   Z
_num_nodesr—   Z_configure_launcherr2   ÚlauncherZis_interactive_compatibler‚   rE   rh   r   r(   r+   r*   rl   ry   rp   r�   rY   rY   rZ   rW   ì  s>   








ÿ
ÿÿÿýz_Connector._lazy_init_strategyÚnameÚcurrentr=   c              
   C   sp   t j d|  ¡  ¡}|d u r|S |d ur6|t|ƒkr6t|ƒt|ƒkr6tƒ r6td| › d|›d| › d|› d�	ƒ‚|S )NZLT_zYour code has `Fabric(ú=z+, ...)` but it conflicts with the value `--zZ` set through the CLI.  Remove it either from the CLI or from the Lightning Fabric object.)ÚosÚenvironrs   Úupperrj   Ú_is_using_clirl   )r™   rš   r=   Z	env_valuerY   rY   rZ   r@     s   *ÿÿÿz_Connector._argument_from_env)r4   r4   r4   r5   NN)r<   N)rp   Ú
__module__r~   Ú__doc__r   rj   r   r)   r   rA   r   r   Ú_PLUGIN_INPUTr[   rL   rM   rN   ÚstaticmethodrO   rP   r„   r   rQ   rS   rT   rU   r   rV   rW   r   r@   rY   rY   rY   rZ   r3   H   sb    ù
þ
ýüûúù
øA
þ
ýüû
ú $




	
*) r3   r:   r<   c                 C   s˜   | d u rd S t tƒt tƒ t tƒ }| |vr"tdt| ƒ› d|› �ƒ‚t| ƒ} | t tƒv rGt| ƒd d… dvrCtd| › dt|  › d�ƒ t|  } t	t| ƒS )Nz
Precision z' is invalid. Allowed precision values: é   )Ú32Ú64z`precision=za` is supported for historical reasons but its usage is discouraged. Please set your precision to z	 instead!)
r	   r!   r    r"   rl   Úreprrj   r0   r#   r   )r:   Zsupported_precisionrY   rY   rZ   ro   %  s    ÿÿÿ
ro   c                   C   s   t ttj dd¡ƒƒS )NZLT_CLI_USEDr{   )ÚboolrA   rœ   r�   rs   rY   rY   rY   rZ   rŸ   ;  s   rŸ   )Krœ   Úcollectionsr   Útypingr   r   r   r   r   r   r�   Útyping_extensionsr	   Zlightning.fabric.acceleratorsr
   Ú)lightning.fabric.accelerators.acceleratorr   Ú"lightning.fabric.accelerators.cudar   Ú!lightning.fabric.accelerators.mpsr   Ú!lightning.fabric.accelerators.xlar   Úlightning.fabric.pluginsr   r   r   r   r   r   r   r   Z%lightning.fabric.plugins.environmentsr   r   r   r   r   r   Z)lightning.fabric.plugins.precision.doubler   Z'lightning.fabric.plugins.precision.fsdpr   Z,lightning.fabric.plugins.precision.precisionr   r    r!   r"   r#   Úlightning.fabric.strategiesr$   r%   r&   r'   r(   r)   r*   r+   Zlightning.fabric.strategies.ddpr,   Ú lightning.fabric.strategies.fsdpr-   r.   Úlightning.fabric.utilitiesr/   r0   Ú(lightning.fabric.utilities.device_parserr1   Ú"lightning.fabric.utilities.importsr2   r¢   r3   ro   r¨   rŸ   rY   rY   rY   rZ   Ú<module>   s8    ( 
(
   `