o
    pôjBF  ã                   @   s~   d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	m
Z
mZmZmZ d dlmZ er4d dlmZ eG dd„ dƒƒZdS )	é    N)Ú	dataclass)ÚTYPE_CHECKINGÚLiteralÚOptionalÚUnion)ÚDeepSpeedSequenceParallelConfigÚDistributedTypeÚTorchContextParallelConfigÚTorchTensorParallelConfig)Úis_torch_version)ÚAcceleratorc                   @   sæ  e Zd ZU dZdZee ed< dZee ed< dZ	ee ed< dZ
ee ed< dZed ed< dZee ed	< dZed
 ed< dZedef ed< dZedef ed< dZedef ed< dZdd„ Zdd„ Zedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd„ ƒZedd „ ƒZ ed!d"„ ƒZ!ed#d$„ ƒZ"ed%d&„ ƒZ#ed'd(„ ƒZ$ed)d*„ ƒZ%ed+d,„ ƒZ&ed-d.„ ƒZ'd/e(fd0d1„Z)dBd/ee( fd2d3„Z*d4e+e+ed5f e+e(d5f f fd6d7„Z,d8d9„ Z-d:e(d;efd<d=„Z.dCd@dA„Z/dS )DÚParallelismConfigaÅ  
    A dataclass to configure parallelisms applied to the model. Inspired by torchtitan's `ParallelDims`
    https://github.com/pytorch/torchtitan/blob/main/torchtitan/distributed/parallel_dims.py

    Args:
        dp_replicate_size (`int`, defaults to `1`):
            The size of the data parallel group. If `dp_replicate_size` is set to 1, the data parallel replication
            group will not be used.
        dp_shard_size (`int`, defaults to `1`):
            The size of the model shard group. If `dp_replicate_size > 1` and `tp_size > 1`, `dp_shard_size` must also
            be greater than 1, as composing DDP + TP is currently not supported.
        tp_size (`int`, defaults to `1`):
            The size of the tensor parallel group. If `tp_size` is set to `1`, the tensor parallel group will not be
            used.
        tp_handler (`~utils.TorchTensorParallelConfig`, defaults to `None`):
            The handler for the tensor parallel group.
        cp_size (`int`, defaults to `1`):
            The size of the context parallel group. Currently not supported, but reserved for future use and enabled
            for downstream libraries.
        cp_backend (`str`, defaults to `torch`):
            Which CP backend to use: `torch` (FSDP2)
        sp_size (`int`, defaults to `1`):
            The size of the sequence parallel group.
        sp_backend (`str`, defaults to `deepspeed`):
            Which SP backend to use:`deepspeed` (ALST/Ulysses)

    You may obtain different distributed data parallel paradigms by configuring `dp_replicate_size` and `dp_shard_size`
    together:
        - `dp_replicate_size == 1` and `dp_shard_size > 1`, we obtain Fully Sharded Data Parallel (FSDP).
        - `dp_replicate_size > 1` and `dp_shard_size > 1`, we obtain Hybrid Sharded Data Parallel (HSDP).
        - `dp_replicate_size > 1` and `dp_shard_size == 1` is an invalid configuration, to use pure DP, use
          `DistributedDataParallelKwargs` instead.

    NÚdp_replicate_sizeÚdp_shard_sizeÚtp_sizeÚcp_sizeÚtorchÚ
cp_backendÚsp_sizeÚ	deepspeedÚ
sp_backendÚ
tp_handlerÚ
cp_handlerÚ
sp_handlerc                 C   sV   d| j › d| j› d| j› d| j› d| j› d| j› d| j› d| j› d	| j› d
| j	› d�S )Nz'ParallelismConfig(
 	dp_replicate_size=z,
	dp_shard_size=z,
	tp_size=z,
	cp_size=z,
	cp_backend=z,
	sp_size=z,
	sp_backend=z,
	total_size=z
	tp_handler=z,
	cp_handler=z)
)
r   r   r   r   r   r   r   Ú
total_sizer   r   ©Úself© r   ún/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/parallelism_config.pyÚ__repr__U   s,   ÿþýüûúùø	÷
öÿzParallelismConfig.__repr__c                    s2   dd l ‰dg‰ ˆ ‡ ‡fdd„| j ¡ D ƒ¡ d S )Nr   Údevice_meshc                    s4   i | ]\}}|ˆ vr|t |d ƒrˆ |j¡n|“qS )Ú__dict__)ÚhasattrÚdeepcopyr!   )Ú.0ÚkÚv©Z_non_serializable_fieldsÚcopyr   r   Ú
<dictcomp>j   s
    þz-ParallelismConfig.to_json.<locals>.<dictcomp>)r(   r#   r!   Úitemsr   r   r'   r   Úto_jsond   s   þÿzParallelismConfig.to_jsonc                 C   ó(   g }| j r
|dg7 }| jr|dg7 }|S )zENames of enabled dimensions across which data parallelism is applied.Údp_replicateÚdp_shard)Údp_replicate_enabledÚdp_shard_enabled©r   Údimsr   r   r   Údp_dim_namesq   ó   

zParallelismConfig.dp_dim_namesc                 C   ó8   g }| j r
|dg7 }| jr|dg7 }| jr|dg7 }|S )z]Names of enabled dimensions which will receive the same batch (non-data parallel dimensions).ÚtpÚcpÚsp)Ú
tp_enabledÚ
cp_enabledÚ
sp_enabledr1   r   r   r   Únon_dp_dim_names{   ó   


z"ParallelismConfig.non_dp_dim_namesc                 C   r,   )zlNames of enabled dimensions which will be flattened into a joint mesh across which is model sharded in FSDP.r.   r7   )r0   r:   r1   r   r   r   Údp_shard_cp_dim_names‡   r4   z'ParallelismConfig.dp_shard_cp_dim_namesc                 C   r5   )z@Names of enabled dimensions across which loss should be averagedr-   r.   r7   )r/   r0   r:   r1   r   r   r   Údp_cp_dim_names‘   r=   z!ParallelismConfig.dp_cp_dim_namesc                 C   s"   g }| j r
|dg7 }|dg7 }|S )z^Names of enabled dimensions across which FSDP is applied, including data parallel replication.r-   Údp_shard_cp)r/   r1   r   r   r   Úfsdp_dim_names�   s
   

z ParallelismConfig.fsdp_dim_namesc                 C   s   | j | j | j | j | j S )zSThe total size of the parallelism configuration, which is the product of all sizes.)r   r   r   r   r   r   r   r   r   r   ¦   s   zParallelismConfig.total_sizec                 C   s   | j | j | j S )zhThe size of the non-data parallel dimensions, which is the product of tensor and context parallel sizes.)r   r   r   r   r   r   r   Únon_data_parallel_size«   s   z(ParallelismConfig.non_data_parallel_sizec                 C   s   | j | j S )z_The size of the data parallel dimensions, which is the product of data parallel replication and)r   r   r   r   r   r   Údata_parallel_size°   ó   z$ParallelismConfig.data_parallel_sizec                 C   ó
   | j dkS )zKTrue if data parallel replication is enabled, i.e. `dp_replicate_size > 1`.é   )r   r   r   r   r   r/   µ   ó   
z&ParallelismConfig.dp_replicate_enabledc                 C   rE   )zDTrue if data parallel sharding is enabled, i.e. `dp_shard_size > 1`.rF   )r   r   r   r   r   r0   º   rG   z"ParallelismConfig.dp_shard_enabledc                 C   rE   )z:True if tensor parallelism is enabled, i.e. `tp_size > 1`.rF   )r   r   r   r   r   r9   ¿   rG   zParallelismConfig.tp_enabledc                 C   rE   )z;True if context parallelism is enabled, i.e. `cp_size > 1`.rF   )r   r   r   r   r   r:   Ä   rG   zParallelismConfig.cp_enabledc                 C   rE   )z;True if context parallelism is enabled, i.e. `sp_size > 1`.rF   )r   r   r   r   r   r;   É   rG   zParallelismConfig.sp_enabledc                 C   s   | j | j S )z$Names of all active mesh dimensions.)r3   r<   r   r   r   r   Úactive_mesh_dimsÎ   rD   z"ParallelismConfig.active_mesh_dimsÚdevice_typec                 C   s¬   | j dkr| jdkrdS tddƒrddlm} ntdƒ‚|  ¡ }t|ƒdkr(dS |\}}||||d	�}| jr>|| j  	d
¡ | j
rI|| j
  	d¡ | jrT|| j  	d¡ |S )a!  Builds a device mesh for the given device type based on the parallelism configuration.
        This method will also create required joint meshes (e.g. `dp_shard_cp`, `dp_cp`, `dp`).

        Args:
            device_type (`str`): The type of device for which to build the mesh, e
        r   rF   Nz>=z2.2.0r   )Úinit_device_meshz4Building a device_mesh requires to have torch>=2.2.0)Úmesh_dim_namesÚdpr@   Zdp_cp)r   r   r   Útorch.distributed.device_meshrJ   ÚRuntimeErrorÚ	_get_meshÚlenr3   Ú_flattenr>   r?   )r   rI   rJ   ÚmeshrK   Ú
mesh_shaper    r   r   r   Úbuild_device_meshÓ   s*   	
ýz#ParallelismConfig.build_device_meshc                 C   s`   | j d u r|d ur|  |¡| _ | j S tdƒ‚|d ur-| j j|kr-td| j j› d|› d�ƒ‚| j S )Nz@You need to pass a device_type e.g cuda to build the device meshz4The device_mesh is already created with device type z@. However, you are trying to get a device mesh with device_type z<. Please check if you correctly initialized your device_mesh)r    rT   Ú
ValueErrorrI   )r   rI   r   r   r   Úget_device_meshö   s   
	ùÿz!ParallelismConfig.get_device_meshÚreturn.c                    s@   ‡fdd„ˆj D ƒ}g d¢‰ t| ¡ ‡ fdd„d�}tt|Ž ƒS )zQGenerate mesh shape and dimension names for torch.distributed.init_device_mesh().c                    s   i | ]}|ˆ j | “qS r   )Ú_sizes)r$   Úparallelismr   r   r   r)     s    z/ParallelismConfig._get_mesh.<locals>.<dictcomp>)r-   r.   r7   r8   r6   c                    s   ˆ   | d ¡S )Nr   )Úindex)Úx)Ú
mesh_orderr   r   Ú<lambda>  s    z-ParallelismConfig._get_mesh.<locals>.<lambda>)Úkey)rH   Úsortedr*   ÚtupleÚzip)r   Z	mesh_dimsZsorted_itemsr   )r\   r   r   rO     s   
þzParallelismConfig._get_meshc                 C   sÀ  | j d u rttj dd¡ƒ| _ | jd u rttj dd¡ƒ| _| jd u r-ttj dd¡ƒ| _| jd u r<ttj dd¡ƒ| _| jd u rItj dd¡| _| j	d u rXttj dd¡ƒ| _	| j
d u retj d	d
¡| _
| jdkrs| jd u rstƒ | _| jdkr¤| jd u r‚tƒ | _n"ttd�}t| j|| j ƒs¤td| j› d|| j › dt| jƒ› �ƒ‚| j	dkr²| jd u r²tƒ | _| j dk r¿td| j › �ƒ‚| jdk rÌtd| j› �ƒ‚| jdk rÙtd| j› �ƒ‚| jdk rætd| j› �ƒ‚dg}| j|vrùtd|› d| j› �ƒ‚| j	dk �rtd| j	› �ƒ‚d
g}| j
|v�rtd|› d| j
› �ƒ‚| jdk�r4| j	dk�r4td| j› d| j	› d�ƒ‚| jdk�s@| jdk�rP| j dk�rP| jdk�rPtdƒ‚| j | j| j| j| j	dœ| _d S )NZ$PARALLELISM_CONFIG_DP_REPLICATE_SIZEÚ1Z PARALLELISM_CONFIG_DP_SHARD_SIZEZPARALLELISM_CONFIG_TP_SIZEZPARALLELISM_CONFIG_CP_SIZEZPARALLELISM_CONFIG_CP_BACKENDr   ZPARALLELISM_CONFIG_SP_SIZEZPARALLELISM_CONFIG_SP_BACKENDr   rF   )r   zParallelismConfig's cp_backend=z
 requires z, but cp_handler was set to z.dp_replicate_size must be at least 1, but got z*dp_shard_size must be at least 1, but got z$tp_size must be at least 1, but got z$cp_size must be at least 1, but got zcp_backend must be one of z
, but got z$sp_size must be at least 1, but got zsp_backend must be one of z[Context Parallelism (CP) and Sequence Parallelism (SP) are mutually exclusive. Got cp_size=z and sp_size=z+. Please set either cp_size=1 or sp_size=1.aC  Tensor/Context parallelism (tp/cp_size > 1) cannot be used with pure data parallelism (dp_replicate_size > 1 and dp_shard_size == 1). Please set dp_shard_size > 1 and dp_replicate_size == 1 to compose FSDP + TP/CP for 2D parallel, or set dp_replicate_size == 1 and dp_shard_size > 1 to compose HSDP + TP/CP for 3D parallel.)r-   r.   r6   r7   r8   )r   ÚintÚosÚenvironÚgetr   r   r   r   r   r   r   r
   r   r	   ÚdictÚ
isinstancerU   Útyper   r   rX   )r   Zcp_backends_config_mapZvalid_cp_backendsZvalid_sp_backendsr   r   r   Ú__post_init__  s€   











ÿ"ÿ






ÿÿÿ0ÿûzParallelismConfig.__post_init__rY   Úsizec                 C   sB   || j  ¡ v sJ d| j  ¡ › �ƒ‚|| j |< t| |› d�|ƒ d S )NzParallelism must be one of Ú_size)rX   ÚkeysÚsetattr)r   rY   rk   r   r   r   Ú	_set_size^  s   "
zParallelismConfig._set_sizeÚacceleratorr   c                 C   s2  t ƒ }|js| jdkrd S | jdkr|  d|j¡ | jdkr$| jdkr$n| j|jkr7td| j› d|j› d�ƒ‚|jrD| j	sD| j
sDtdƒ‚| jdkr^|js^|js^|jtjks^td|j› d	�ƒ‚| j ¡ D ]\}}|dkr‚t| |› d
�d ƒd ur‚| d|› d|› d�¡ qc|r•|jr—t dd |¡ t¡ d S d S d S )NrF   r-   r   zParallelismConfig total_size (z ) does not match num_processes (zJ). Please adjust dp_replicate_size/ dp_shard_size/tp_size/cp_size/sp_size.a×  FSDP is enabled but the parallelism config has no dimension for FSDP to shard across (both `dp_shard_size` and `cp_size` are 1). This usually means a model that is already parallelized another way -- e.g. loaded with `DistributedConfig(tp_size=N)` or `enable_expert_parallel=True`, which makes the whole world size tensor/expert parallel -- was launched under an FSDP config. Either launch it without the FSDP config, or leave ranks for FSDP to use by lowering `tp_size`.z�ParallelismConfig is only compatible DistributedType.FSDP (version 2) or DistributedType.Multi{Device} or DistributedType.DEEPSPEED, but got Ú.Ú_handlerzParallelismConfig.z_handler is set, but z0_size is set to 1. This handler will be ignored.z.ParallelismConfig has the following warnings:
Ú
)ÚsetÚmulti_devicer   ro   Únum_processesr   r   rU   Úis_fsdp2r0   r:   Údistributed_typer   Ú	DEEPSPEEDrX   r*   ÚgetattrÚaddÚis_main_processÚwarningsÚwarnÚjoinÚUserWarning)r   rp   Ú	_warningsrY   rk   r   r   r   Ú_validate_acceleratorc  sL   

ÿÿÿ
ÿþÿÿ€
þÿz'ParallelismConfig._validate_accelerator)N)rp   r   )0Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rc   Ú__annotations__r   r   r   r   r   r   r   r   r   r
   r   r	   r   r   r    r   r+   Úpropertyr3   r<   r>   r?   rA   r   rB   rC   r/   r0   r9   r:   r;   rH   ÚstrrT   rV   r`   rO   rj   ro   r‚   r   r   r   r   r   !   sb   
 #
	

	










#&Lr   )rd   r}   Údataclassesr   Útypingr   r   r   r   Úaccelerate.utils.dataclassesr   r   r	   r
   Zaccelerate.utils.versionsr   Ú
accelerater   r   r   r   r   r   Ú<module>   s   