o
    pôj��  ã                   @   sØ  d Z ddlZddlZddlmZ ddlmZmZ ddlm	Z	m
Z
 ddlmZ ddlZddlmZmZ d	d
lmZ d	dlmZmZ d	dlmZmZmZ d	dlmZ eƒ r\ddlm  mZ eƒ reddl m!Z! dd„ Z"dd„ Z#dd„ Z$dd„ Z%dd„ Z&e"ddœdd„Z'drdd„Z(dd „ Z)d!d"„ Z*d#d$„ Z+d%d&„ Z,d'd(„ Z-d)d*„ Z.d+d,„ Z/d-d.„ Z0G d/d0„ d0e1ƒZ2d1d2„ Z3d3d4„ Z4e3d5d6„ ƒZ5d7efd8d9„Z6d7efd:d;„Z7d7efd<d=„Z8dsd>d?„Z9dtdAdB„Z:ej;d	ej<dej=dCej>dDej?dEej@dFejAdGejBdHejCdIejDdJi
ZEdKdL„ eE F¡ D ƒZGdMdN„ ZHdudOejIfdPdQ„ZJe3dsdReKfdSdT„ƒZLdsdReKfdUdV„ZMdsdReKfdWdX„ZNdvdYdZ„ZOdsd[d\„ZPG d]d^„ d^eQƒZRe4dwd_d`„ƒZSdsdadb„ZTe3dxdedf„ƒZUdgdh„ ZVG didj„ djƒZWdkdl„ ZXdmdn„ ZYedydpdq„ƒZZdS )zzB
A set of basic tensor ops compatible with tpu, gpu, and multigpu
é    N)ÚMapping)ÚcontextmanagerÚnullcontext)Úupdate_wrapperÚwraps)ÚAnyé   )ÚAcceleratorStateÚPartialStateé   )Ú!TORCH_DISTRIBUTED_OPERATION_TYPES)ÚDistributedTypeÚTensorInformation)Úis_npu_availableÚis_torch_distributed_availableÚis_torch_xla_available)Úis_torch_version)ÚReduceOpc                 C   s   t | tjƒS ©N)Ú
isinstanceÚtorchÚTensor©Útensor© r   úl/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/utils/operations.pyÚis_torch_tensor-   s   r   c              	   C   s2   t | tjjtjjtjjtjjtjjtjjtjj	ƒS r   )
r   r   ÚxpuÚFloatTensorÚ
ByteTensorÚ	IntTensorÚ
LongTensorÚ
HalfTensorÚDoubleTensorÚBFloat16Tensorr   r   r   r   Úis_torch_xpu_tensor1   s   ør%   c                 C   s
   t | tƒS r   )r   r   ©Útensor_infor   r   r   Úis_tensor_information>   ó   
r(   c                 C   s   t | tƒot| dƒot| dƒS )z’
    Checks if `data` is a `namedtuple` or not. Can have false positives, but only if a user is trying to mimic a
    `namedtuple` perfectly.
    Ú_asdictÚ_fields)r   ÚtupleÚhasattr©Údatar   r   r   Úis_namedtupleB   s   r0   c                 C   s$   t | ƒrt| ƒt|ƒŽ S t| ƒ|ƒS )zO
    Cast a generator to the same type as obj (list, tuple, or namedtuple)
    )r0   ÚtypeÚlist)ÚobjÚ	generatorr   r   r   Ú
honor_typeJ   s   r5   F©Ú	test_typeÚerror_on_other_typec                   s¨   t |ttfƒrt|‡ ‡‡‡‡fdd„|D ƒƒS t |tƒr/t|ƒ‡ ‡‡‡‡fdd„| ¡ D ƒƒS ˆ|ƒr>ˆ|gˆ ¢R i ˆ¤ŽS ˆrRtdt|ƒ› dˆj› dˆj› d�ƒ‚|S )	a   
    Recursively apply a function on a data structure that is a nested list/tuple/dictionary of a given base type.

    Args:
        func (`callable`):
            The function to recursively apply.
        data (nested list/tuple/dictionary of objects accepted by `test_type`):
            The data on which to apply `func`
        *args:
            Positional arguments that will be passed to `func` when applied on the unpacked data.
        test_type (`Callable[[Any], bool]`, *optional*, defaults to `is_torch_tensor`):
            The predicate deciding whether `func` should be applied to a given leaf object.
        error_on_other_type (`bool`, *optional*, defaults to `False`):
            Whether to return an error or not if after unpacking `data`, we get on an object for which `test_type`
            returns `False`. If `False`, the function will leave such objects unchanged.
        **kwargs (additional keyword arguments, *optional*):
            Keyword arguments that will be passed to `func` when applied on the unpacked data.

    Returns:
        The same data structure as `data` with `func` applied to every object for which `test_type` returns `True`.
    c                 3   s.   � | ]}t ˆ|gˆ ¢R ˆˆd œˆ¤ŽV  qdS )r6   N©Úrecursively_apply)Ú.0Úo©Úargsr8   ÚfuncÚkwargsr7   r   r   Ú	<genexpr>n   s   € ýÿÿÿÿ
ÿz$recursively_apply.<locals>.<genexpr>c                    s0   i | ]\}}|t ˆ|gˆ ¢R ˆˆd œˆ¤Ž“qS )r6   r9   ©r;   ÚkÚvr=   r   r   Ú
<dictcomp>w   s    ýÿÿÿÿÿz%recursively_apply.<locals>.<dictcomp>zUnsupported types (z) passed to `z?`. Only nested list/tuple/dicts of objects that are valid for `z` should be passed.)	r   r,   r2   r5   r   r1   ÚitemsÚ	TypeErrorÚ__name__)r?   r/   r7   r8   r>   r@   r   r=   r   r:   U   s,   üþ
	üÿÿÿr:   c              
      s4  t | ƒs	t| dƒr^ˆ dkrd‰ z| jˆ ˆd�W S  ty%   |  ˆ ¡ Y S  tyF } ztƒ r:tˆ tƒr9dˆ › �‰ n|‚W Y d}~nd}~ww z| jˆ ˆd�W S  ty]   |  ˆ ¡ Y S w t| tt	fƒrst
| ‡ ‡‡fdd„| D ƒƒS t| tƒr˜tˆtƒr�ˆg‰nˆdu r‡g ‰t| ƒ‡ ‡‡fd	d
„|  ¡ D ƒƒS | S )a‹  
    Recursively sends the elements in a nested list/tuple/dictionary of tensors to a given device.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to send to a given device.
        device (`torch.device`):
            The device to send the data to.
        non_blocking (`bool`, *optional*, defaults to `False`):
            If `True`, the transfer to the device is performed asynchronously, which can overlap
            data movement with computation. Only effective when the device supports it (e.g. CUDA).
        skip_keys (`str` or `List[str]`, *optional*):
            A key or list of keys in a dictionary `tensor` whose values should not be sent to
            the given `device`. Entries with these keys are left on their original device.

    Returns:
        The same data structure as `tensor` with all tensors sent to the proper device.
    ÚtoÚnpuznpu:0)Únon_blockingznpu:Nc                 3   s    � | ]}t |ˆ ˆˆd �V  qdS )©rK   Ú	skip_keysN©Úsend_to_device)r;   Út©ÚdevicerK   rM   r   r   rA   ±   s   € z!send_to_device.<locals>.<genexpr>c              	      s.   i | ]\}}||ˆv r|nt |ˆ ˆˆd �“qS )rL   rN   )r;   rC   rP   rQ   r   r   rE   ¹   s    ÿÿz"send_to_device.<locals>.<dictcomp>)r   r-   rI   rG   ÚAssertionErrorr   r   Úintr,   r2   r5   r   Ústrr1   rF   )r   rR   rK   rM   Úerrorr   rQ   r   rO   ˆ   sF   

€€ùÿÿ

þÿrO   c                 C   ó   dd„ }t || ƒS )aK  
    Recursively gathers the information needed to rebuild a nested list/tuple/dictionary of tensors.

    Args:
        data (nested list/tuple/dictionary of `torch.Tensor`):
            The data to send to analyze.

    Returns:
        The same data structure as `data` with [`~utils.TensorInformation`] instead of tensors.
    c                 S   s   t | j| jd�S )N)ÚshapeÚdtype)r   rX   rY   r   r   r   r   Ú_get_data_structureÎ   s   z/get_data_structure.<locals>._get_data_structurer9   )r/   rZ   r   r   r   Úget_data_structureÂ   ó   
r[   c                 C   rW   )a:  
    Recursively gathers the shape of a nested list/tuple/dictionary of tensors as a list.

    Args:
        data (nested list/tuple/dictionary of `torch.Tensor`):
            The data to send to analyze.

    Returns:
        The same data structure as `data` with lists of tensor shapes instead of tensors.
    c                 S   s
   t | jƒS r   )r2   rX   r   r   r   r   Ú
_get_shapeà   r)   zget_shape.<locals>._get_shaper9   )r/   r]   r   r   r   Ú	get_shapeÔ   r\   r^   c                 C   s   dd„ }t || td�S )zÜ
    Recursively initializes tensors from a nested list/tuple/dictionary of [`~utils.TensorInformation`].

    Returns:
        The same data structure as `data` with tensors instead of [`~utils.TensorInformation`].
    c                 S   s   t j| jd| jiŽS ©NrY   )r   ÚemptyrX   rY   r&   r   r   r   Ú_initialize_tensorî   ó   z.initialize_tensors.<locals>._initialize_tensor©r7   )r:   r(   )Zdata_structurera   r   r   r   Úinitialize_tensorsæ   s   rd   c                 C   sœ   t | tttfƒrt| ƒdkrtdt| ƒ› d�ƒ‚t | ttfƒr%t| d ƒS t | tƒr9|  ¡ D ]	}t| | ƒ  S nt | t	j
ƒsItdt| ƒ› d�ƒ‚| jd S )a  
    Recursively finds the batch size in a nested list/tuple/dictionary of lists of tensors.

    Args:
        data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to find the batch size.

    Returns:
        `int`: The batch size.
    r   z&Cannot find the batch size from empty Ú.z0Can only find the batch size of tensors but got )r   r,   r2   r   ÚlenÚ
ValueErrorr1   Úfind_batch_sizeÚkeysr   r   rG   rX   )r/   rC   r   r   r   rh   ô   s   

ÿ
rh   c              	   C   s$   zt | ƒW S  ttfy   Y dS w )a  
    Same as [`utils.operations.find_batch_size`] except will ignore if `ValueError` and `TypeErrors` are raised

    Args:
        data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to find the batch size.

    Returns:
        `int`: The batch size.
    N)rh   rg   rG   r.   r   r   r   Úignorant_find_batch_size  s   

þrj   c                 C   rW   )aS  
    Recursively finds tensors in a nested list/tuple/dictionary and converts them to a list of numbers.

    Args:
        data (nested list/tuple/dictionary of `torch.Tensor`): The data from which to convert to regular numbers.

    Returns:
        The same data structure as `data` with lists of numbers instead of `torch.Tensor`.
    c                 S   s,   |   ¡  ¡ } | jtjkr|  tj¡} |  ¡ S r   )ÚdetachÚcpurY   r   Úbfloat16rI   Úfloat32Útolistr   r   r   r   Ú_convert_to_list'  s   z!listify.<locals>._convert_to_listr9   )r/   rp   r   r   r   Úlistify  s   
	rq   c                 C   s"   dd„ }t || dd�}t ¡  |S )Nc                 S   s0   | j dkr|  ¡ d  } |  ¡ s|  ¡ } t | ¡S )Nr   )ÚndimÚcloneÚis_contiguousÚ
contiguousÚxmÚ
all_gatherr   r   r   r   Ú_tpu_gather_one4  s
   

z$_tpu_gather.<locals>._tpu_gather_oneT©r8   )r:   rv   Ú	mark_step)r   rx   Úresr   r   r   Ú_tpu_gather3  s   	r|   c                    sJ   t ƒ ‰tjj‰ ˆjjdkrtddƒrtj ¡  ‡ ‡fdd„}t	|| dd�S )Nr   z<=z2.8c                    s´   ˆ j dkrˆ  ¡ d  ‰ ˆ  ¡ sˆ  ¡ ‰ ˆjd ur@ˆjdkr@tjˆjˆ  ¡  ˆ j	ˆj
d�}ˆ|ˆ ƒ |jdgˆ  ¡ dd … ¢R Ž S ‡ fdd„tˆjƒD ƒ}tj |ˆ ¡ tj|dd�S )	Nr   Úgloo©rY   rR   éÿÿÿÿr   c                    s   g | ]}t  ˆ ¡‘qS r   )r   Ú
empty_like©r;   Ú_r   r   r   Ú
<listcomp>b  ó    z8_gpu_gather.<locals>._gpu_gather_one.<locals>.<listcomp>©Údim)rr   rs   rt   ru   Úbackendr   r`   Únum_processesÚnumelrY   rR   ÚviewÚsizeÚrangeÚdistributedrw   Úcat)r   Úoutput_tensors©Z	gather_opÚstater   r   Ú_gpu_gather_oneJ  s   
ý
z$_gpu_gather.<locals>._gpu_gather_oneTry   )
r
   r   r�   Úall_gather_into_tensorrR   r1   r   r   Úsynchronizer:   )r   r’   r   r�   r   Ú_gpu_gatherB  s   
r•   c                   @   s   e Zd ZdZdS )ÚDistributedOperationExceptionz‰
    An exception class for distributed operations. Raised if the operation cannot be performed due to the shape of the
    tensors.
    N)rH   Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r–   i  s    r–   c                    ó   t ˆ ƒ‡ fdd„ƒ}|S )zv
    Verifies that `tensor` is the same shape across all processes. Only ran if `PartialState().debug` is `True`.
    c                     s  t ƒ jtjkst ƒ jsˆ | i |¤ŽS ˆ j› dˆ j› �}d|v r$|d }n| d }t ƒ jjt	|ƒjkrNt
d|› d|jj› dt ƒ jj› dt ƒ jj› d|› d�ƒ‚t|ƒ}t|gƒ}|d d ur€| |d ¡t|ƒk}|s€d	 d
d„ t|ƒD ƒ¡}t
d|› d|› �ƒ‚ˆ | i |¤ŽS )Nre   r   r   z%One or more of the tensors passed to z were not on the z+ while the `Accelerator` is configured for z. Please move it to the z before calling z
  - c                 S   s    g | ]\}}d |› d|› �‘qS )zProcess z: r   )r;   ÚirX   r   r   r   rƒ   Š  s     z5verify_operation.<locals>.wrapper.<locals>.<listcomp>znCannot apply desired operation due to shape mismatches. All shapes across devices must be valid.

Operation: `z`
Input shapes:
  - )r
   Údistributed_typer   ÚNOÚdebugr—   rH   rR   r1   Úfind_devicer–   r^   Úgather_objectÚcountrf   ÚjoinÚ	enumerate)r>   r@   Ú	operationr   ÚshapesÚoutputZare_sameZprocess_shape_str©Úfunctionr   r   Úwrapperw  s8   
ÿÿÿ
þþÿz!verify_operation.<locals>.wrapper©r   ©r¨   r©   r   r§   r   Úverify_operationr  s   r¬   c                    rš   )z‘
    Checks that `verify_operation` failed and if so reports a more helpful error chaining the existing
    `DistributedOperationException`.
    c               
      sN   zˆ | i |¤ŽW S  t y& } zˆ j› dˆ j› �}t d|› d�ƒ|‚d }~ww )Nre   zError found while calling `z1`. Please see the earlier error for more details.)r–   r—   rH   )r>   r@   Úer¤   r§   r   r   r©   ›  s   
ÿþ€þz"chained_operation.<locals>.wrapperrª   r«   r   r§   r   Úchained_operation•  s   	r®   c                 C   s.   t ƒ jtjkrt| ƒS t ƒ jtv rt| ƒS | S )a4  
    Recursively gather tensor in a nested list/tuple/dictionary of tensors from all devices.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to gather.

    Returns:
        The same data structure as `tensor` with all tensors sent to the proper device.
    )r
   rœ   r   ÚXLAr|   r   r•   r   r   r   r   Úgather¨  s
   r°   Úobjectc                    sv  ddl }tƒ }|j‰|j}| ¡ }t | |¡ | ¡ }t|ƒ}t	j
|gt	jˆd�}‡fdd„t|ƒD ƒ}t	j ||¡ ttdd„ |D ƒƒƒ}	d‰ˆ|	k rTˆd	K ‰ˆ|	k sLt	j |¡}
t	 |
¡ ˆ¡}| ˆ¡ t	jˆ| t	jˆd�‰ ‡ ‡fd
d„t|ƒD ƒ}t	j ||¡ g }t|ƒD ]-\}}t||  ¡ ƒ}|d|…  ¡  ¡  ¡ }t |¡}t|tƒr³|  |¡ q‹| !|¡ q‹|S )aÒ  Gather picklable objects from all ranks with padded allgather sizes.

    On Neuron devices every unique tensor shape triggers a new NEFF compilation. The standard
    ``all_gather_object`` sizes its byte tensor to the exact max pickle size, which varies per call
    and causes unbounded compilation cache growth.  This variant rounds the allgather size up to
    the next power of 2 so the number of distinct compiled shapes stays bounded to O(log(max_size)).
    r   Nr~   c                    s   g | ]}t jd t jˆ d�‘qS )r   r~   )r   ÚzerosÚlongr�   ©rR   r   r   rƒ   Ò  s    z)_neuron_gather_object.<locals>.<listcomp>c                 s   s   � | ]}|  ¡ V  qd S r   )Úitem©r;   Úsr   r   r   rA   Ô  ó   € z(_neuron_gather_object.<locals>.<genexpr>é   r   c                    s$   g | ]}ˆ ˆ| ˆ|d   … ‘qS )r   r   ©r;   r›   )Ú	coalescedÚpadded_sizer   r   rƒ   â  s   $ )"Úior
   rR   rˆ   ÚBytesIOÚpickleÚdumpÚgetvaluerf   r   r   r³   rŒ   r�   rw   rT   ÚmaxÚByteStorageÚ_from_bufferr   rI   Úresize_r`   Úuint8r£   rµ   rl   ÚnumpyÚtobytesÚloadsr   r2   ÚextendÚappend)r±   r½   r‘   Ú
group_sizeÚbufÚ	raw_bytesÚ
local_sizeZlocal_size_tensorÚ	size_listÚmax_sizeÚbyte_storageÚinput_tensorr�   Úresultr›   r   Úobj_sizeZ	obj_bytesr3   r   )r»   rR   r¼   r   Ú_neuron_gather_object¼  s>   ÿ


rÖ   c                 C   s2   dd„ t tƒ jƒD ƒ}tj || ¡ dd„ |D ƒS )Nc                 S   s   g | ]}d ‘qS r   r   r�   r   r   r   rƒ   ó  s    z&_gpu_gather_object.<locals>.<listcomp>c                 S   s   g | ]	}|D ]}|‘qqS r   r   )r;   ÚyÚxr   r   r   rƒ   ö  s    )rŒ   r
   rˆ   r   r�   Úall_gather_object)r±   Zoutput_objectsr   r   r   Ú_gpu_gather_objectò  s   rÚ   c                 C   sD   t ƒ jtjkrtdƒ‚t ƒ jtv r t ƒ jjdkrt| ƒS t	| ƒS | S )a5  
    Recursively gather object in a nested list/tuple/dictionary of objects from all devices.

    Args:
        object (nested list/tuple/dictionary of picklable object):
            The data to gather.

    Returns:
        The same data structure as `object` with all the objects sent to every device.
    z&gather objects in TPU is not supportedÚneuron)
r
   rœ   r   r¯   ÚNotImplementedErrorr   rR   r1   rÖ   rÚ   )r±   r   r   r   r    ù  s   r    c                 C   s   ddd„}t || d|d�S )Nr   c                 S   s   t jj| |d� | S )N©Úsrc)r   r�   Ú	broadcast)r   rÞ   r   r   r   Ú_gpu_broadcast_one  s   z*_gpu_broadcast.<locals>._gpu_broadcast_oneT)r8   rÞ   ©r   r9   )r/   rÞ   rà   r   r   r   Ú_gpu_broadcast  s   
râ   úbroadcast tensorc                    sh   t | ttfƒrt| ‡ fdd„t| ƒD ƒƒS t | tƒr)t| ƒ‡ fdd„|  ¡ D ƒƒS t 	ˆ | ‡fdd„¡S )Nc                 3   s*   � | ]\}}t |ˆ › d |› �d�V  qdS )r‚   ©ÚnameN©Ú_tpu_broadcast)r;   r›   rP   rä   r   r   rA     ó   €( z!_tpu_broadcast.<locals>.<genexpr>c                    s(   i | ]\}}|t |ˆ › d |› �d�“qS )r‚   rä   ræ   rB   rä   r   r   rE     ó   ( z"_tpu_broadcast.<locals>.<dictcomp>c                    ó   | ˆ  S r   r   ©rØ   rÝ   r   r   Ú<lambda>  ó    z _tpu_broadcast.<locals>.<lambda>)
r   r2   r,   r5   r£   r   r1   rF   rv   Úmesh_reduce)r   rÞ   rå   r   )rå   rÞ   r   rç     s
   
rç   é   é   é   é   é   é   é	   é
   c                 C   s   i | ]\}}||“qS r   r   rB   r   r   r   rE   +  r„   rE   c                 C   sž   d}t ƒ }tj|tj|jd�}| dur/| j}t| j }tjt	|ƒ|g td�|dt
|ƒd …< t|dd�}|| ¡  }t|dd… d	 ƒ}|dd… }||fS )
ze
    Grabs the shape of `tensor` only available on one process and returns a tensor of its shape
    r¹   r~   N©rY   r   Úsum©Ú	reductionr   r   )r
   r   r`   rT   rR   rX   ÚTENSOR_TYPE_TO_INTrY   r   r2   rf   ÚreduceÚnonzero)r   Zmax_tensor_dimensionr‘   Zbase_tensorrX   Útensor_dtyperY   r   r   r   Úgather_tensor_shape.  s   
(rÿ   Úreturnc                 C   s@   t ƒ }t| ƒ\}}| du rtj|t| d� |j¡} t| dd�S )a�  
    Copies a tensor that only exists on a single device and broadcasts it to other devices. Differs from `broadcast` as
    each worker doesn't need to know its shape when used (and tensor can be `None`)

    Args:
        tensor (`torch.tensor`):
            The tensor that should be sent to all devices. Must only have it be defined on a single device, the rest
            should be `None`.
    Nr÷   rø   rù   )r
   rÿ   r   r²   ÚTENSOR_INT_TO_DTYPErI   rR   rü   )r   r‘   rX   rY   r   r   r   Úcopy_tensor_to_devicesG  s
   
r  Úfrom_processc                 C   s8   t ƒ jtjkrt| |dd�S t ƒ jtv rt| |d�S | S )a©  
    Recursively broadcast tensor in a nested list/tuple/dictionary of tensors to all devices.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to gather.
        from_process (`int`, *optional*, defaults to 0):
            The process from which to send the data

    Returns:
        The same data structure as `tensor` with all tensors broadcasted to the proper device.
    zaccelerate.utils.broadcast)rÞ   rå   rÝ   )r
   rœ   r   r¯   rç   r   râ   )r   r  r   r   r   rß   X  s
   rß   c                 C   s(  ddl }tƒ }|j}|j|kr"| ¡ }t | |¡ | ¡ }t|ƒ}nd}d}t	j
|gt	j|d�}t	j |¡ t| ¡ ƒ}	d}
|
|	k rJ|
dK }
|
|	k sB|j|krct	j |¡}t	 |¡ |¡}| |
¡ n	t	j|
t	j|d�}t	jj||d� t |d|	…  ¡  ¡  ¡ ¡}tt| ƒƒD ]}|| | |< q‰| S )a#  Broadcast a list of picklable objects with padded tensor sizes.

    On Neuron devices ``ProcessGroupNeuron.broadcast()`` is implemented as an allreduce,
    so every unique tensor shape triggers a new NEFF compilation.  The standard
    ``broadcast_object_list`` sizes its byte tensor to the exact serialized length, which
    varies per call and causes unbounded compilation cache growth.  This variant rounds the
    broadcast tensor size up to the next power of 2, bounding the number of distinct
    compiled shapes to ~O(log(max_size)).
    r   Nó    r~   r¹   r   rÝ   )r½   r
   rR   Úprocess_indexr¾   r¿   rÀ   rÁ   rf   r   r   r³   r�   Ú
all_reducerT   rµ   rÃ   rÄ   r   rI   rÅ   r²   rÆ   rß   rÉ   rl   rÇ   rÈ   rŒ   )Úobject_listr  r½   r‘   rR   rÍ   rÎ   rÏ   Zsize_tensorÚ	real_sizer¼   rÒ   Zdata_tensorrÔ   r›   r   r   r   Ú_neuron_broadcast_object_listn  s6   


ÿ
r	  c                    s|   t ƒ jtjkrt| ƒD ]\}}t d|‡ fdd„¡| |< q| S t ƒ jtv r<t ƒ jj	dkr4t
| ˆ d� | S tjj| ˆ d� | S )a–  
    Broadcast a list of picklable objects from one process to the others.

    Args:
        object_list (list of picklable objects):
            The list of objects to broadcast. This list will be modified inplace.
        from_process (`int`, *optional*, defaults to 0):
            The process from which to send the data.

    Returns:
        The same list containing the objects from process 0.
    z&accelerate.utils.broadcast_object_listc                    rê   r   r   rë   ©r  r   r   rì   ²  rí   z'broadcast_object_list.<locals>.<lambda>rÛ   r
  rÝ   )r
   rœ   r   r¯   r£   rv   rî   r   rR   r1   r	  r   r�   Úbroadcast_object_list)r  r  r›   r3   r   r
  r   r  £  s   ûÿr  c                 C   s   dd„ }t || |ƒS )aN  
    Recursively takes a slice in a nested list/tuple/dictionary of tensors.

    Args:
        data (nested list/tuple/dictionary of `torch.Tensor`):
            The data to slice.
        tensor_slice (`slice`):
            The slice to take.

    Returns:
        The same data structure as `data` with all the tensors slices.
    c                 S   s   | | S r   r   )r   Útensor_slicer   r   r   Ú_slice_tensorÉ  ó   z$slice_tensors.<locals>._slice_tensorr9   )r/   r  r  rˆ   r  r   r   r   Úslice_tensors»  s   r  c                    sÈ   t ˆ d ttfƒrtˆ d ‡ ‡fdd„ttˆ d ƒƒD ƒƒS t ˆ d tƒr9tˆ d ƒ‡ ‡fdd„ˆ d  ¡ D ƒƒS t ˆ d t	j
ƒrHt	jˆ ˆd�S t ˆ ttfƒrYtˆ ƒdkrYˆ d S tdtˆ d ƒ› �ƒ‚)	aù  
    Recursively concatenate the tensors in a nested list/tuple/dictionary of lists of tensors with the same shape.
    If there is only a single batch of data, it is returned as-is.

    Args:
        data (nested list/tuple/dictionary of lists of tensors `torch.Tensor`):
            The data to concatenate.
        dim (`int`, *optional*, defaults to 0):
            The dimension on which to concatenate.

    Returns:
        The same data structure as `data` with all the tensors concatenated.
    r   c                 3   s*   � | ]‰ t ‡ fd d„ˆD ƒˆd�V  qdS )c                    ó   g | ]}|ˆ  ‘qS r   r   ©r;   Úd©r›   r   r   rƒ   Þ  ó    z)concatenate.<locals>.<genexpr>.<listcomp>r…   N©Úconcatenate©r;   ©r/   r†   r  r   rA   Þ  rè   zconcatenate.<locals>.<genexpr>c                    s(   i | ]‰ ˆ t ‡ fd d„ˆD ƒˆd�“qS )c                    r  r   r   r  ©rC   r   r   rƒ   à  r  z*concatenate.<locals>.<dictcomp>.<listcomp>r…   r  r  r  r  r   rE   à  ré   zconcatenate.<locals>.<dictcomp>r…   r   z%Can only concatenate tensors but got )r   r,   r2   r5   rŒ   rf   r   r1   ri   r   r   rŽ   rG   r  r   r  r   r  Ï  s   *(r  c                   @   s   e Zd ZdS )ÚCannotPadNestedTensorWarningN)rH   r—   r˜   r   r   r   r   r  é  s    r  c                 C   s   ddd„}t || d|||d�S )	a3  
    Recursively pad the tensors in a nested list/tuple/dictionary of tensors from all devices to the same size so they
    can safely be gathered.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to gather.
        dim (`int`, *optional*, defaults to 0):
            The dimension on which to pad.
        pad_index (`int`, *optional*, defaults to 0):
            The value with which to pad.
        pad_first (`bool`, *optional*, defaults to `False`):
            Whether to pad at the beginning or the end.
    r   Fc           	         s  t | ddƒrt dt¡ | S ˆ t| jƒksˆ t| jƒ k r| S ˆ dk r*ˆ t| jƒ7 ‰ tj| j| jd�d  }t	|ƒ 
¡ }t‡ fdd„|D ƒƒ‰ˆ| jˆ  krO| S | j‰tˆƒ}ˆ|ˆ < |  t|ƒ¡| }|rwt‡ ‡‡fdd„tt|ƒƒD ƒƒ}nt‡ ‡fd	d„tt|ƒƒD ƒƒ}| ||< |S )
NÚ	is_nestedFzHCannot pad nested tensors without more information. Leaving unprocessed.r   r´   c                 3   s   � | ]}|ˆ  V  qd S r   r   r¶   r…   r   r   rA     r¸   zFpad_across_processes.<locals>._pad_across_processes.<locals>.<genexpr>c                 3   s2   � | ]}|ˆ krt ˆˆˆ   ˆƒnt d ƒV  qd S r   ©Úslicerº   ©r†   rÑ   Úold_sizer   r   rA     s   € "
ÿc                 3   ó.   � | ]}|ˆ krt d ˆˆ  ƒnt dƒV  qdS ©r   Nr  rº   ©r†   r  r   r   rA     ó   €, )ÚgetattrÚwarningsÚwarnr  rf   rX   r   r   rR   r°   rl   rÂ   r2   Ú	new_zerosr,   rŒ   )	r   r†   Ú	pad_indexÚ	pad_firstr‹   ÚsizesÚnew_sizeÚ
new_tensorÚindicesr   r  r   Ú_pad_across_processesþ  s4   þ

ÿ z3pad_across_processes.<locals>._pad_across_processesT)r8   r†   r(  r)  N©r   r   Fr9   )r   r†   r(  r)  r.  r   r   r   Úpad_across_processesí  s   
"ÿr0  c                 C   s   ddd„}t || d|||d�S )zþ
    Takes a `tensor` of arbitrary size and pads it so that it can work given `num_processes` needed dimensions.

    New tensors are just the last input repeated.

    E.g.:
      Tensor: ([3,4,4]) Num processes: 4 Expected result shape: ([4,4,4])

    r   c           
         s®   || }|||  }|| dkr|| }n|||  }|||@   kr'dk r-n n|| }| j ‰tˆƒ}|| |d< |  t|ƒ¡}t‡ ‡fdd„tt|ƒƒD ƒƒ}	| ||	< |S )Nr   r   c                 3   r   r!  r  rº   r"  r   r   rA   ?  r#  z@pad_input_tensors.<locals>._pad_input_tensors.<locals>.<genexpr>)rX   r2   r'  r,   rŒ   rf   )
r   Ú
batch_sizerˆ   r†   Ú	remainderZlast_inputsZto_padr+  r,  r-  r   r"  r   Ú_pad_input_tensors0  s   
 z-pad_input_tensors.<locals>._pad_input_tensorsT)r8   r1  rˆ   r†   Nrá   r9   )r   r1  rˆ   r†   r3  r   r   r   Úpad_input_tensors%  s   
úr4  Úmeanç      ð?c                 C   s   ddd„}t || d||d�S )	a^  
    Recursively reduce the tensors in a nested list/tuple/dictionary of lists of tensors across all processes by the
    mean of a given operation.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to reduce.
        reduction (`str`, *optional*, defaults to `"mean"`):
            A reduction method. Can be of "mean", "sum", "max", or "none"
        scale (`float`, *optional*):
            A default scaling value to be applied after the reduce, only valid on XLA.

    Returns:
        The same data structure as `data` with all the tensors reduced.
    r5  r6  c                 S   s¤   t ƒ }|  ¡ }|jtjkr|S |jtjkr0t ¡  |dkr tjntj	}t 
||g|¡ t ¡  n|jjtv rG|dkr=tjntj}tj 
||¡ |dkrP||j }|S )NrÂ   r5  )r
   rs   rœ   r   r�   r¯   rv   rz   Z
REDUCE_MAXZ
REDUCE_SUMr  Úvaluer   r   ÚMAXÚSUMr   r�   rˆ   )r   rú   Úscaler‘   Zcloned_tensorZxla_opZtorch_opr   r   r   Ú_reduce_across_processes_  s   

z(reduce.<locals>._reduce_across_processesT)r8   rú   r:  N©r5  r6  r9   )r   rú   r:  r;  r   r   r   rü   M  s   

ÿrü   c                 C   s   dd„ }dd„ }t || |d�S )av  
    Recursively converts the elements nested list/tuple/dictionary of tensors in FP16/BF16 precision to FP32.

    Args:
        tensor (nested list/tuple/dictionary of `torch.Tensor`):
            The data to convert from FP16/BF16 to FP32.

    Returns:
        The same data structure as `tensor` with all tensors that were in FP16/BF16 precision converted to FP32.
    c                 S   s   |   ¡ S r   )Úfloatr   r   r   r   Ú_convert_to_fp32…  r  z)convert_to_fp32.<locals>._convert_to_fp32c                 S   s$   t | ƒs	t| dƒo| jtjtjfv S r_   )r   r-   rY   r   Úfloat16rm   r   r   r   r   Ú_is_fp16_bf16_tensorˆ  s   þz-convert_to_fp32.<locals>._is_fp16_bf16_tensorrc   r9   )r   r>  r@  r   r   r   Úconvert_to_fp32y  s   rA  c                   @   s(   e Zd ZdZdd„ Zdd„ Zdd„ ZdS )	ÚConvertOutputsToFp32ae  
    Decorator to apply to a function outputting tensors (like a model forward pass) that ensures the outputs in FP16
    precision will be convert back to FP32.

    Args:
        model_forward (`Callable`):
            The function which outputs we want to treat.

    Returns:
        The same function as `model_forward` but with converted outputs.
    c                 C   s   || _ t| |ƒ d S r   )Úmodel_forwardr   )ÚselfrC  r   r   r   Ú__init__ž  s   zConvertOutputsToFp32.__init__c                 O   s   t | j|i |¤ŽƒS r   )rA  rC  )rD  r>   r@   r   r   r   Ú__call__¢  rb   zConvertOutputsToFp32.__call__c                 C   s
   t  d¡‚)Nz‘Cannot pickle a prepared model with automatic mixed precision, please unwrap the model with `Accelerator.unwrap_model(model)` before pickling it.)r¿   ÚPicklingError)rD  r   r   r   Ú__getstate__¥  s   ÿz!ConvertOutputsToFp32.__getstate__N)rH   r—   r˜   r™   rE  rF  rH  r   r   r   r   rB  ‘  s
    rB  c                    s   t ˆ ƒ‰ ‡ fdd„}ˆ |_|S )Nc                     s   ˆ | i |¤ŽS r   r   )r>   r@   ©rC  r   r   Úforward®  s   z(convert_outputs_to_fp32.<locals>.forward)rB  Ú__wrapped__)rC  rJ  r   rI  r   Úconvert_outputs_to_fp32«  s   rL  c                 C   s~   t | tƒr|  ¡ D ]}t|ƒ}|dur|  S q	dS t | ttfƒr4| D ]}t|ƒ}|dur1|  S q#dS t | tjƒr=| jS dS )zå
    Finds the device on which a nested dict/list/tuple of tensors lies (assuming they are all on the same device).

    Args:
        (nested list/tuple/dictionary of `torch.Tensor`): The data we want to know the device of.
    N)	r   r   ÚvaluesrŸ   r,   r2   r   r   rR   )r/   r3   rR   r   r   r   rŸ   ·  s"   
ÿþÿþÿrŸ   Tc                 c   s|   � t ƒ jtjkst ƒ jdurt ƒ j ¡ stƒ }nddl}|jj	| |||d�}|� dV  W d  ƒ dS 1 s7w   Y  dS )zˆ
    Wrapper around `deepspeed.runtime.zero.GatheredParameters`, but if Zero-3 is not enabled, will be a no-op context
    manager.
    Nr   )Úmodifier_rankÚ
fwd_moduleÚenabled)
r	   rœ   r   Ú	DEEPSPEEDÚdeepspeed_pluginÚis_zero3_init_enabledr   Ú	deepspeedÚzeroÚGatheredParameters)ÚparamsrN  rO  rP  Zgather_param_contextrT  r   r   r   rV  Ì  s   €
ÿÿ"ÿrV  )FNrá   )r   rã   r   )NNr/  r<  )NNT)[r™   r¿   r%  Úcollections.abcr   Ú
contextlibr   r   Ú	functoolsr   r   Útypingr   r   r‘   r	   r
   Ú	constantsr   Údataclassesr   r   Úimportsr   r   r   Úversionsr   Útorch_xla.core.xla_modelÚcoreÚ	xla_modelrv   Útorch.distributedr   r   r%   r(   r0   r5   r:   rO   r[   r^   rd   rh   rj   rq   r|   r•   Ú	Exceptionr–   r¬   r®   r°   rÖ   rÚ   r    râ   rç   r=  ÚdoubleÚhalfrm   rÆ   Úint8Úint16Úint32Úint64Úboolrû   rF   r  rÿ   r   r  rT   rß   r	  r  r  r  ÚUserWarningr  r0  r4  rü   rA  rB  rL  rŸ   rV  r   r   r   r   Ú<module>   s”   
3:'	#
6

	ö5


7(+