o
    qôjY  ã                   @   sþ  d dl Z d dlZd dlZd dlZd dlmZ d dl mZ d dlmZm	Z	 d dl
mZ d dlmZ d dlZd dlZd dlmZ d dlmZ d	d
lmZ d	dlmZ d	dlmZ ddlmZ ddlmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z% ddl&m'Z' ddl(m)Z) ddl*m+Z+ ee,ƒZ-e$ƒ r’d dl.m/  m0Z1 dej2j3de4fdd„Z5dej2j3de4fdd„Z6dej2j3de4fdd„Z7dej2j3de4fdd„Z8dej2j3dej2j3fdd„Z9dej2j3fd d!„Z:dej2j3dej2j3fd"d#„Z;d$ej2j3de4fd%d&„Z<d$ej2j3dee= fd'd(„Z>	*dPd+e4d,e4d-e4fd.d/„Z?d0d1„ Z@d2eAfd3d4„ZBdQd5e4d6e4fd7d8„ZCe"d9ƒ�r#ejDnej/ZEeEjFjGejHeejIgZJe"d:ƒ�r<eJ KejLjM¡ dRd;d<„ZNd=d>„ ZOd?d@„ ZPdRdAee= de4fdBdC„ZQde=fdDdE„ZRdFdG„ ZSdHdI„ ZTdJeUfdKdL„ZVdSd$ej2j3dMe4deWej2j3 fdNdO„ZXdS )Té    N)Úencode)ÚOrderedDict)ÚpartialÚreduce)Ú
MethodType)ÚOptional)ÚVersion)Ú	save_fileé   )Úwrite_basic_config)Ú
get_logger)ÚPartialStateé   )ÚFSDP_PYTORCH_VERSION)ÚDistributedType)Úis_deepspeed_availableÚis_numpy_availableÚis_torch_distributed_availableÚis_torch_xla_availableÚis_weights_only_available)Úid_tensor_storage)Úconvert_model)Úis_torch_versionÚmoduleÚreturnc                 C   s   t tdƒsdS t| tjjjƒS )zD
    Check whether the module was compiled with torch.compile()
    Ú_dynamoF)ÚhasattrÚtorchÚ
isinstancer   Ú
eval_frameÚOptimizedModule©r   © r"   úg/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/utils/other.pyÚis_compiled_module6   s   
r$   c                 C   s<   t tdƒsdS | jr|  ¡ D ]}t|tjjjƒr dS qdS )z\
    Check whether the module has submodules that were compiled with `torch.compile()`.
    r   FT)r   r   Ú_modulesÚmodulesr   r   r   r    ©r   Ú	submoduler"   r"   r#   Úhas_compiled_regions@   s   
ÿr)   c                    s0   t ˆ tjjƒotˆ ƒdkot‡ fdd„ˆ D ƒƒS )zÓ
    Check whether the module is a repeated block, i.e. `torch.nn.ModuleList` with all children of the same class. This
    is useful to determine whether we should apply regional compilation to the module.
    r   c                 3   s    � | ]}t |ˆ d  jƒV  qdS )r   N)r   Ú	__class__)Ú.0Úmr!   r"   r#   Ú	<genexpr>X   s   € z%is_repeated_blocks.<locals>.<genexpr>)r   r   ÚnnÚ
ModuleListÚlenÚallr!   r"   r!   r#   Úis_repeated_blocksO   s
   
ÿýr2   c                 C   s&   | j r|  ¡ D ]	}t|ƒr dS qdS )zý
    Check whether the module has repeated blocks, i.e. `torch.nn.ModuleList` with all children of the same class, at
    any level of the module hierarchy. This is useful to determine whether we should apply regional compilation to the
    module.
    TF)r%   r&   r2   r'   r"   r"   r#   Úhas_repeated_blocks\   s   ÿr3   c                    sF   dt jjdt jjf‡ fdd„‰ ˆ | fi |¤Ž}d|jvr!| |jd< |S )a_  
    Performs regional compilation where we target repeated blocks of the same class and compile them sequentially to
    hit the compiler's cache. For example, in `GPT2LMHeadModel`, the repeated block/class is `GPT2Block`, and can be
    accessed as `model.transformer.h[0]`. The rest of the model (e.g. model.lm_head) is compiled separately.

    This allows us to speed up the compilation overhead / cold start of models like LLMs and Transformers in general.
    See https://pytorch.org/tutorials/recipes/regional_compilation.html for more details.

    Args:
        module (`torch.nn.Module`):
            The model to compile.
        **compile_kwargs:
            Additional keyword arguments to pass to `torch.compile()`.

    Returns:
        `torch.nn.Module`: A new instance of the model with some compiled regions.

    Example:
    ```python
    >>> from accelerate.utils import compile_regions
    >>> from transformers import AutoModelForCausalLM

    >>> model = AutoModelForCausalLM.from_pretrained("gpt2")
    >>> compiled_model = compile_regions(model, mode="reduce-overhead")
    >>> compiled_model.transformer.h[0]
    OptimizedModule(
        (_orig_mod): GPT2Block(
                (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
                (attn): GPT2Attention(
                (c_attn): Conv1D(nf=2304, nx=768)
                (c_proj): Conv1D(nf=768, nx=768)
                (attn_dropout): Dropout(p=0.1, inplace=False)
                (resid_dropout): Dropout(p=0.1, inplace=False)
            )
            (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
            (mlp): GPT2MLP(
                (c_fc): Conv1D(nf=3072, nx=768)
                (c_proj): Conv1D(nf=768, nx=3072)
                (act): NewGELUActivation()
                (dropout): Dropout(p=0.1, inplace=False)
            )
        )
    )
    ```
    r   r   c                    sê   t | ƒrtj ¡ }| D ]}| tj|fi |¤Ž¡ q|S t| ƒrj| j | j¡}|j	 
| j	¡ t|j	 ¡ ƒD ]\}}t|dƒrOt|dd ƒ| u rOt|j|ƒ|j	|< q5i |_|  ¡ D ]\}}| |ˆ |fi |¤Ž¡ qW|S tj| fi |¤Ž}|S )NÚ__func__Ú__self__)r2   r   r.   r/   ÚappendÚcompiler3   r*   Ú__new__Ú__dict__ÚupdateÚlistÚitemsr   Úgetattrr   r4   r%   Únamed_childrenÚ
add_module)r   Úcompile_kwargsÚ
new_moduler(   ÚnameÚvalue©Ú_compile_regionsr"   r#   rE   ™   s$   
ô€þz)compile_regions.<locals>._compile_regionsÚ	_orig_mod)r   r.   ÚModuler9   )r   r@   rA   r"   rD   r#   Úcompile_regionsj   s
   /

rH   c                 K   sd   t | ƒr| D ]
}|jdi |¤Ž qdS t| ƒr(|  ¡ D ]
}t|fi |¤Ž qdS | jdi |¤Ž dS )a  
    Performs regional compilation the same way as `compile_regions`, but specifically for `DeepSpeedEngine.module`.
    Since the model is wrapped in a `DeepSpeedEngine` and has many added hooks, offloaded parameters, etc that
    `torch.compile(...)` interferes with, version of trgional compilation uses the inplace `module.compile()` method
    instead.

    Args:
        module (`torch.nn.Module`):
            The model to compile.
        **compile_kwargs:
            Additional keyword arguments to pass to `module.compile()`.
    Nr"   )r2   r7   r3   ÚchildrenÚcompile_regions_deepspeed©r   r@   r(   Úchildr"   r"   r#   rJ   µ   s   ÿÿrJ   c                 K   sd   t | ƒr| D ]
}|jdi |¤Ž q| S t| ƒr(|  ¡ D ]
}t|fi |¤Ž q| S | jdi |¤Ž | S )a³  
    Like `compile_regions`, but uses the in-place `module.compile()` instead of `torch.compile(module)`.

    Needed for the FSDP2 prepare path: `torch.compile(module)` returns an `OptimizedModule` whose `__call__`
    bypasses `nn.Module._call_impl`, so forward/pre hooks added later by `fully_shard` never fire and per-layer
    all-gather/reshard is lost. The in-place `module.compile()` keeps `_call_impl` (and its runtime hook check)
    on the call path, so FSDP hooks installed afterwards still run.

    Args:
        module (`torch.nn.Module`):
            The model to compile.
        **compile_kwargs:
            Additional keyword arguments to pass to `module.compile()`.
    Nr"   )r2   r7   r3   rI   Úcompile_regions_fsdp2rK   r"   r"   r#   rM   Í   s   ûÿrM   Úmodelc                    s>   t ddƒrddlm‰  nddlm‰  t‡ fdd„|  ¡ D ƒƒS )zÊ
    Check if the model has DTensor parameters.

    Args:
        model (`torch.nn.Module`):
            The model to check.

    Returns:
        `bool`: Whether the model has DTensor parameters.
    ú>=z2.5.0r   ©ÚDTensorc                 3   s   � | ]}t |ˆ ƒV  qd S ©N)r   )r+   ÚprP   r"   r#   r-   ø   s   € z$model_has_dtensor.<locals>.<genexpr>)r   Útorch.distributed.tensorrQ   Ztorch.distributed._tensorÚanyÚ
parameters)rN   r"   rP   r#   Úmodel_has_dtensorç   s   
rW   c                 C   sL   t | ddƒ}|dur|S t t | ddƒddƒ}t|tƒr | d¡S t |ddƒS )a  
    Get the tensor parallel degree a `transformers` model was sharded with, or `None` if it was not sharded.

    Args:
        model (`torch.nn.Module`):
            The model to inspect.

    Returns:
        `Optional[int]`: The model's tensor parallel size.
    Útp_sizeNÚconfigÚdistributed_config)r=   r   ÚdictÚget)rN   rX   rZ   r"   r"   r#   Úget_model_tp_sizeû   s   

r]   TFÚkeep_fp32_wrapperÚkeep_torch_compileÚ	recursivec                    sb  t jjjt jjf}t| ƒ}t| ƒ}d}|r| }| j} n|r)d| jv r)| }| jd } t	ƒ r7ddl
m} ||f7 }tdtƒrJtƒ rJddlm}	 ||	f7 }t| |ƒrW| j} t| |ƒsO|rc‡ fdd„‰ ˆ | ƒ} |s—| j}
| j d	d¡}|dur‹t|
d
ƒr…|
j}
|
|kr€nt|
d
ƒsxt|
| ƒ| _t| ddƒr—t| dd� |r¯|dur¯|r¦| |_|} | S |r¯| |jd< |} | S )a—  
    Extract a model from its distributed containers.

    Args:
        model (`torch.nn.Module`):
            The model to extract.
        keep_fp32_wrapper (`bool`, *optional*):
            Whether to remove mixed precision hooks from the model.
        keep_torch_compile (`bool`, *optional*):
            Whether to unwrap compiled model.
        recursive (`bool`, *optional*, defaults to `False`):
            Whether to recursively extract all cases of `module.module` from `model` as well as unwrap child sublayers
            recursively, not just the top-level distributed containers.

    Returns:
        `torch.nn.Module`: The extracted model.
    NrF   r   )ÚDeepSpeedEnginerO   )ÚFullyShardedDataParallelc                    s@   t | dƒrˆ | jƒ}n| }| ¡ D ]\}}t||ˆ |ƒƒ q|S )Nr   )r   r   r>   Úsetattr)r   Zunwrapped_modulerB   rL   ©Ú_recursive_unwrapr"   r#   re   D  s   
z6extract_model_from_parallel.<locals>._recursive_unwrapÚ_original_forwardÚ__wrapped__Z _converted_to_transformer_engineF)Zto_transformer_engine)r   r.   ÚparallelÚDistributedDataParallelÚDataParallelr$   r)   rF   r9   r   Ú	deepspeedra   r   r   r   Ú2torch.distributed.fsdp.fully_sharded_data_parallelrb   r   r   ÚforwardÚpopr   rg   r   r=   r   )rN   r^   r_   r`   ÚoptionsZis_compiledZhas_compiledZcompiled_modelra   ÚFSDPrm   Úoriginal_forwardr"   rd   r#   Úextract_model_from_parallel  sX   





ÿ

ýü
rr   c                   C   s   t ƒ  ¡  dS )a  
    Introduces a blocking point in the script, making sure all processes have reached this point before continuing.

    <Tip warning={true}>

    Make sure all processes will reach this instruction otherwise one of your processes will hang forever.

    </Tip>
    N)r   Úwait_for_everyoner"   r"   r"   r#   rs   j  s   
rs   Ú
state_dictc                    sÎ   t  t¡}ˆ  ¡ D ]\}}t|tƒs|t|ƒ  |¡ q	dd„ | ¡ D ƒ}tƒ }| 	¡ D ] }‡ fdd„|D ƒ}| 
|dd… ¡ |dd… D ]}ˆ |= qFq,t|ƒdkr\t d|› d	�¡ d
d„ ˆ  ¡ D ƒ‰ ˆ S )z°
    Cleans the state dictionary from a model and removes tensor aliasing if present.

    Args:
        state_dict (`dict`):
            The state dictionary from a model
    c                 S   s"   i | ]\}}t |ƒd kr||“qS )r   )r0   )r+   ÚptrÚnamesr"   r"   r#   Ú
<dictcomp>†  s   " z4clean_state_dict_for_safetensors.<locals>.<dictcomp>c                    s   g | ]}|ˆ v r|‘qS r"   r"   )r+   rB   ©rt   r"   r#   Ú
<listcomp>Ž  s    z4clean_state_dict_for_safetensors.<locals>.<listcomp>r   Nr   zRemoved shared tensor zk while saving. This should be OK, but check by verifying that you don't receive any warning while reloadingc                 S   s*   i | ]\}}|t |tjƒr| ¡ n|“qS r"   )r   r   ÚTensorÚ
contiguous)r+   ÚkÚvr"   r"   r#   rw   –  s   * )ÚcollectionsÚdefaultdictr;   r<   r   Ústrr   r6   ÚsetÚvaluesr:   r0   ÚloggerÚwarning)rt   ZptrsrB   ÚtensorZshared_ptrsZ
warn_namesrv   Zfound_namesr"   rx   r#   Ú clean_state_dict_for_safetensorsw  s&   

€ÿ
ÿr†   Úsave_on_each_nodeÚsafe_serializationc                 C   s‚   t ƒ jtjkrt | ¡} |r ttddid�}t| t	ƒrt
| ƒ} ntj}t ƒ jr0|s0|| |ƒ dS t ƒ jr=|r?|| |ƒ dS dS dS )aõ  
    Save the data to disk. Use in place of `torch.save()`.

    Args:
        obj:
            The data to save
        f:
            The file (or file-like object) to use to save the data
        save_on_each_node (`bool`, *optional*, defaults to `False`):
            Whether to only save on the global main process
        safe_serialization (`bool`, *optional*, defaults to `False`):
            Whether to save `obj` using `safetensors` or the traditional PyTorch way (that uses `pickle`).
    ÚformatÚpt)ÚmetadataN)r   Údistributed_typer   ÚXLAÚxmZ_maybe_convert_to_cpur   Úsafe_save_filer   r   r†   r   ÚsaveÚis_main_processÚis_local_main_process)ÚobjÚfr‡   rˆ   Z	save_funcr"   r"   r#   r�   š  s   

€ÿr�   z2.0.0z1.25.0c              	   K   sž   z;t ƒ rtj ¡ }d|vrd|d< tj t¡ n| dd¡ tj| fd|i|¤Ž}W t ƒ r:tj ¡  |r:tj |¡ |S t ƒ rMtj ¡  |rNtj |¡ w w w )a  
    Compatible drop-in replacement of `torch.load()` which allows for `weights_only` to be used if `torch` version is
    2.4.0 or higher. Otherwise will ignore the kwarg.

    Will also add (and then remove) an exception for numpy arrays

    Args:
        f:
            The file (or file-like object) to use to load the data
        map_location:
            a function, `torch.device`, string or a dict specifying how to remap storage locations
        **kwargs:
            Additional keyword arguments to pass to `torch.load()`.
    Úweights_onlyTNÚmap_location)	r   r   ÚserializationÚget_safe_globalsÚadd_safe_globalsÚTORCH_SAFE_GLOBALSrn   ÚloadÚclear_safe_globals)r”   r–   ÚkwargsZold_safe_globalsZ
loaded_objr"   r"   r#   r›   Ì  s&   

ü
ýr›   c                 C   sH   t | dƒst | dƒst| d| ƒ} t | dƒr| jS t | dƒr | jS t| ƒS )z(
    Gets a pretty name from `obj`.
    Ú__qualname__Ú__name__r*   )r   r=   rž   rŸ   r€   )r“   r"   r"   r#   Úget_pretty_nameì  s   

r    c                 C   s@   |   ¡ D ]\}}t|tƒr| |i ¡}t||ƒ q|||< q|S )zÃ
    Recursively merges two dictionaries.

    Args:
        source (`dict`): The dictionary to merge into `destination`.
        destination (`dict`): The dictionary to merge `source` into.
    )r<   r   r[   Ú
setdefaultÚmerge_dicts)ÚsourceÚdestinationÚkeyrC   Únoder"   r"   r#   r¢   ù  s   

r¢   Úportc                 C   sR   | du rd} t   t jt j¡�}| d| f¡dkW  d  ƒ S 1 s"w   Y  dS )z²
    Checks if a port is in use on `localhost`. Useful for checking if multiple `accelerate launch` commands have been
    run and need to see if the port is already in use.
    Ni<s  Ú	localhostr   )ÚsocketÚAF_INETÚSOCK_STREAMÚ
connect_ex)r§   Úsr"   r"   r#   Úis_port_in_use  s
   $ÿr®   c                  C   sJ   t   t jt j¡�} |  d¡ |  ¡ d W  d  ƒ S 1 sw   Y  dS )zÂ
    Gets a free port on `localhost`. Useful for automatic port selection when port 0 is specified in distributed
    training scenarios.

    Returns:
        int: An available port number
    )Ú r   r   N)r©   rª   r«   ÚbindÚgetsockname)r­   r"   r"   r#   Úget_free_port  s   

$þr²   c                 C   sB   dD ]}| dk rt | dƒ› d|› �  S | d } qt | dƒ› d�S )z7Converts `size` from bytes to the largest possible unit)ÚbytesÚKBÚMBÚGBÚTBg      �@r
   ú z PB)Úround)ÚsizeÚxr"   r"   r#   Úconvert_bytes#  s
   
r¼   c                  C   sj   t  ¡ } | j}|dkrdS t d| j¡^}}}d}t|ƒt|ƒk r3d|› d|› d�}tj|dd	� dS dS )
zFWarns if the kernel version is below the recommended minimum on Linux.ÚLinuxNz(\d+\.\d+\.\d+)z5.5.0zDetected kernel version z,, which is below the recommended minimum of zo; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.T)Úmain_process_only)	ÚplatformÚunameÚsystemÚreÚsplitÚreleaser   rƒ   r„   )ÚinforÁ   Ú_ÚversionÚmin_versionÚmsgr"   r"   r#   Úcheck_os_kernel-  s   ÿûrÊ   Úattrc                 C   s   dd„ }t || g| d¡ ƒS )zÑ
    Recursive `getattr`.

    Args:
        obj:
            A class instance holding the attribute.
        attr (`str`):
            The attribute that is to be retrieved, e.g. 'attribute1.attribute2'.
    c                 S   s
   t | |ƒS rR   )r=   )r“   rË   r"   r"   r#   Ú_getattrJ  s   
z#recursive_getattr.<locals>._getattrÚ.)r   rÃ   )r“   rË   rÌ   r"   r"   r#   Úrecursive_getattr?  s   rÎ   Úreturn_fqnsc           
      C   s´   |s| nd| f}|g}g }|rS|  ¡ }|r|\}}| ¡ D ]$\}}t|tjjƒrA|r<|r2|d | n|}	| |	|f¡ q| |¡ q|rL| ||f¡ n| |¡ |s|ddd… S )aA  Traverse the model in bottom-up order and return the children modules in that order.

    Args:
        model (`torch.nn.Module`): the model to get the children of

    Returns:
        `list[torch.nn.Module]`: a list of children modules of `model` in bottom-up order. The last element is the
        `model` itself.
    r¯   rÍ   Néÿÿÿÿ)rn   r>   r   r   r.   rG   r6   )
rN   rÏ   ÚtopÚstackZordered_modulesZcurrent_moduleZcurrent_module_namerB   rË   Ú
child_namer"   r"   r#   Úget_module_children_bottom_upP  s&   

€
òrÔ   )TTF)FFrR   )F)Yr~   r¿   rÂ   r©   Úcodecsr   r   Ú	functoolsr   r   Útypesr   Útypingr   ÚnumpyÚnpr   Úpackaging.versionr   Úsafetensors.torchr	   r�   Zcommands.config.defaultr   Úloggingr   Ústater   Ú	constantsr   Údataclassesr   Úimportsr   r   r   r   r   Úmodelingr   Útransformer_enginer   Úversionsr   rŸ   rƒ   Útorch_xla.core.xla_modelÚcoreÚ	xla_modelrŽ   r.   rG   Úboolr$   r)   r2   r3   rH   rJ   rM   rW   Úintr]   rr   rs   r[   r†   r�   Ú_coreZnp_coreÚ
multiarrayÚ_reconstructÚndarrayÚdtyperš   r6   ÚdtypesÚUInt32DTyper›   r    r¢   r®   r²   r¼   rÊ   r€   rÎ   r;   rÔ   r"   r"   r"   r#   Ú<module>   s|   
Kÿÿÿ
ÿX#$ú
	
 
(