o
    /ôjþT  ã                   @   sÂ   d dl Z d dlZd dlZd dlmZ ddlmZ eejdƒs6edƒejj	d< edƒejj	d< edƒejj	d< d d	l
mZmZmZ d
d„ Zdd„ ZG dd„ dejjƒZG dd„ dƒZ	ddd„ZdS )é    N)Ú_pytreeé   )Ú_dummy_typeZ_CudaStreamBaseÚ
_CUDAGraphÚ_graph_pool_handleÚ_cuda_isCurrentStreamCapturing)r   r   r   c                   C   ó   t ƒ S )zÌReturn True if CUDA graph capture is underway on the current CUDA stream, False otherwise.

    If a CUDA context does not exist on the current device, returns False without initializing the context.
    )r   © r	   r	   úb/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/torch/cuda/graphs.pyÚis_current_stream_capturing   s   r   c                   C   r   )zÚReturn an opaque token representing the id of a graph memory pool.

    See :ref:`Graph memory management<graph-memory-management>`.

    .. warning::
        This API is in beta and may change in future releases.
    )r   r	   r	   r	   r
   Úgraph_pool_handle!   s   r   c                       sv   e Zd ZdZ‡ fdd„Zd‡ fdd„	Z‡ fdd	„Z‡ fd
d„Z‡ fdd„Z‡ fdd„Z	‡ fdd„Z
‡ fdd„Z‡  ZS )Ú	CUDAGraphzrWrapper around a CUDA graph.

    .. warning::
        This API is in beta and may change in future releases.
    c                    s   t ƒ  | ¡S ©N)ÚsuperÚ__new__)Úcls©Ú	__class__r	   r
   r   4   s   zCUDAGraph.__new__NÚglobalc                    s   t ƒ j||d� dS )að  Begin capturing CUDA work on the current stream.

        Typically, you shouldn't call ``capture_begin`` yourself.
        Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`,
        which call ``capture_begin`` internally.

        Arguments:
            pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or
                :meth:`other_Graph_instance.pool()<torch.cuda.CUDAGraph.pool>`) that hints this graph may share memory
                with the indicated pool.  See :ref:`Graph memory management<graph-memory-management>`.
            capture_error_mode (str, optional): specifies the cudaStreamCaptureMode for the graph capture stream.
                Can be "global", "thread_local" or "relaxed". During cuda graph capture, some actions, such as cudaMalloc,
                may be unsafe. "global" will error on actions in other threads, "thread_local" will only error for
                actions in the current thread, and "relaxed" will not error on these actions. Do NOT change this setting
                unless you're familiar with `cudaStreamCaptureMode <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g9d0535d93a214cbf126835257b16ba85>`_
        )ÚpoolÚcapture_error_modeN)r   Úcapture_begin)Úselfr   r   r   r	   r
   r   7   s   zCUDAGraph.capture_beginc                    ó   t ƒ  ¡  dS )aG  End CUDA graph capture on the current stream.

        After ``capture_end``, ``replay`` may be called on this instance.

        Typically, you shouldn't call ``capture_end`` yourself.
        Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`,
        which call ``capture_end`` internally.
        N)r   Úcapture_end©r   r   r	   r
   r   J   s   	zCUDAGraph.capture_endc                    r   )z,Replay the CUDA work captured by this graph.N)r   Úreplayr   r   r	   r
   r   U   ó   zCUDAGraph.replayc                    r   )z1Delete the graph currently held by this instance.N)r   Úresetr   r   r	   r
   r   Y   r   zCUDAGraph.resetc                    ó
   t ƒ  ¡ S )zäReturn an opaque token representing the id of this graph's memory pool.

        This id can optionally be passed to another graph's ``capture_begin``,
        which hints the other graph may share the same memory pool.
        )r   r   r   r   r	   r
   r   ]   s   
zCUDAGraph.poolc                    r   )z/Enable debugging mode for CUDAGraph.debug_dump.)r   Úenable_debug_moder   r   r	   r
   r    e   s   
zCUDAGraph.enable_debug_modec                    s   t ƒ  |¡S )zÖ
        Arguments:
            debug_path (required): Path to dump the graph to.

        Calls a debugging function to dump the graph if the debugging is
        enabled via CUDAGraph.enable_debug_mode()
        )r   Ú
debug_dump)r   Z
debug_pathr   r	   r
   r!   i   s   zCUDAGraph.debug_dump)Nr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r   r    r!   Ú__classcell__r	   r	   r   r
   r   -   s    r   c                   @   sJ   e Zd ZU dZdZejd ed< 			ddefdd„Z	d	d
„ Z
dd„ ZdS )ÚgraphaÅ  Context-manager that captures CUDA work into a :class:`torch.cuda.CUDAGraph` object for later replay.

    See :ref:`CUDA Graphs <cuda-graph-semantics>` for a general introduction,
    detailed use, and constraints.

    Arguments:
        cuda_graph (torch.cuda.CUDAGraph): Graph object used for capture.
        pool (optional): Opaque token (returned by a call to :func:`~torch.cuda.graph_pool_handle()` or
            :meth:`other_Graph_instance.pool()<torch.cuda.CUDAGraph.pool>`) hinting this graph's capture
            may share memory from the specified pool. See :ref:`Graph memory management<graph-memory-management>`.
        stream (torch.cuda.Stream, optional): If supplied, will be set as the current stream in the context.
            If not supplied, ``graph`` sets its own internal side stream as the current stream in the context.
        capture_error_mode (str, optional): specifies the cudaStreamCaptureMode for the graph capture stream.
            Can be "global", "thread_local" or "relaxed". During cuda graph capture, some actions, such as cudaMalloc,
            may be unsafe. "global" will error on actions in other threads, "thread_local" will only error for
            actions in the current thread, and "relaxed" will not error on actions. Do NOT change this setting
            unless you're familiar with `cudaStreamCaptureMode <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g9d0535d93a214cbf126835257b16ba85>`_

    .. note::
        For effective memory sharing, if you pass a ``pool`` used by a previous capture and the previous capture
        used an explicit ``stream`` argument, you should pass the same ``stream`` argument to this capture.

    .. warning::
        This API is in beta and may change in future releases.

    .. _cudaStreamCaptureMode:
        https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g9d0535d93a214cbf126835257b16ba85
    Nztorch.cuda.StreamÚdefault_capture_streamr   r   c                 C   sr   | j jd u rtj ¡ | j _|d u rdn|f| _|d ur|n| j j| _| jd us)J ‚tj | j¡| _|| _	|| _
d S )Nr	   )r   r(   ÚtorchÚcudaÚStreamr   Zcapture_streamÚstreamÚ
stream_ctxÚ
cuda_graphr   )r   r.   r   r,   r   r	   r	   r
   Ú__init__”   s   
ÿ
zgraph.__init__c                 C   s@   t j ¡  t ¡  t j ¡  | j ¡  | jj	| j
d| jiŽ d S )Nr   )r)   r*   ÚsynchronizeÚgcÚcollectÚempty_cacher-   Ú	__enter__r.   r   r   r   r   r	   r	   r
   r4   ª   s   


ÿ
ÿzgraph.__enter__c                 C   s   | j  ¡  | j |||¡ d S r   )r.   r   r-   Ú__exit__)r   Úexc_typeÚ	exc_valueÚ	tracebackr	   r	   r
   r5   ¸   s   
zgraph.__exit__)NNr   )r"   r#   r$   r%   r(   ÚtypingÚOptionalÚ__annotations__Ústrr/   r4   r5   r	   r	   r	   r
   r'   t   s   
 û
ûr'   é   Fc           &         sp  t  ¡ rt  ¡ rtdƒ‚d}t| tƒsd}| f} |f}g ‰ t| |ƒD ]L\}}t|t jjƒrUt	|j
ƒdkrBt	|jƒdkrBt	|jƒdksFJ dƒ‚tdd„ | ¡ D ƒƒsUJ dƒ‚tj|Ž }ˆ  t|ƒ¡ td	d„ |D ƒƒsnJ d
ƒ‚q"dd„ ˆ D ƒ}	dd„ | D ƒ‰‡ ‡fdd„tt	| ƒƒD ƒ}
dd„ tt	| ƒƒD ƒ}dd„ tt	| ƒƒD ƒ}|du r¨tƒ n|}t j ¡  t j t j ¡ ¡�D t| ||
ƒD ]5\}}}t|ƒD ])}t ||Ž ¡}t jjtdd„ |D ƒƒtdd„ |D ƒƒtdd„ |D ƒƒd|d�}qÈ~~q¿W d  ƒ n1 sÿw   Y  t j ¡  g }g }t| ||ƒD ]6\}}}t jj||d�� ||Ž }W d  ƒ n	1 �s0w   Y  t |¡\}}| t|ƒ¡ | |¡ �qg }g }tt|
ƒt|ƒt|ƒtˆƒƒD ]w\}}}}tdd„ |D ƒƒ}t jj||d��( t jjtdd„ |D ƒƒtdd„ |D ƒƒtdd„ |D ƒƒd|d�}W d  ƒ n	1 �s w   Y  g }d}|D ]} | j�r¾| || ¡ |d7 }�q«| d¡ �q«t|ƒ}| |¡ | |¡ �q]| ¡  | ¡  dd„ }!g }"t | ƒD ]E\}#}|!||# ||# ˆ|# |	|# ||# |
|# ||# ||# ||# ƒ	}$t|t jjƒ�r&dd„ }%|%||j!|$|j"ƒ|_"|" |¡ �qç|" |$¡ �qç|�r4|"d S t|"ƒS ) aØ  Accept callables (functions or :class:`nn.Module<torch.nn.Module>`\ s) and returns graphed versions.

    Each graphed callable's forward pass runs its source callable's
    forward CUDA work as a CUDA graph inside a single autograd node.

    The graphed callable's forward pass also appends
    a backward node to the autograd graph. During backward, this node runs the
    callable's backward work as a CUDA graph.

    Therefore, each graphed callable should be a drop-in replacement for its source callable
    in an autograd-enabled training loop.

    See :ref:`Partial-network capture<partial-network-capture>` for detailed use and constraints.

    If you pass a tuple of several callables, their captures will use the same memory pool.
    See :ref:`Graph memory management<graph-memory-management>` for when this is appropriate.

    Arguments:
        callables (torch.nn.Module or Python function, or tuple of these): Callable or callables to graph.
            See :ref:`Graph memory management<graph-memory-management>` for when passing a tuple of callables
            is appropriate.  If you pass a tuple of callables, their order in the tuple must be the same order
            they'll run in the live workload.
        sample_args (tuple of Tensors, or tuple of tuples of Tensors): Samples args for each callable.
            If a single callable was passed, ``sample_args`` must be a single tuple of argument Tensors.
            If a tuple of callables was passed, ``sample_args`` must be tuple of tuples of argument Tensors.
        num_warmup_iters (int): The number of warmup iterations. Currently, ``DataDistributedParallel`` needs
            11 iterations for warm up. Default: ``3``.
        allow_unused_input (bool): If False, specifying inputs that were not used when computing outputs
            (and therefore their grad is always zero) is an error. Defaults to False.
        pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or
            :meth:`other_Graph_instance.pool()<torch.cuda.CUDAGraph.pool>`) that hints this graph may share memory
            with the indicated pool.  See :ref:`Graph memory management<graph-memory-management>`.
    .. note::
        The ``requires_grad`` state of each Tensor in ``sample_args`` must match the state
        that's expected for the corresponding real input in the training loop.

    .. warning::
        This API is in beta and may change in future releases.

    .. warning::
        ``sample_args`` for each callable must contain only Tensors. Other types are not allowed.

    .. warning::
        Returned callables do not support higher order differentiation (e.g., double backward).

    .. warning::
        In any :class:`~torch.nn.Module` passed to :func:`~make_graphed_callables`, only parameters
        may be trainable. Buffers must have ``requires_grad=False``.

    .. warning::
        After you pass a :class:`torch.nn.Module` through :func:`~make_graphed_callables`,
        you may not add or remove any of that Module's parameters or buffers.

    .. warning::
        :class:`torch.nn.Module`\s passed to :func:`~torch.cuda.make_graphed_callables` must not have module hooks
        registered on them at the time they are passed. However, registering hooks on modules *after* passing them
        through :func:`~torch.cuda.make_graphed_callables` is allowed.

    .. warning::
        When running a graphed callable, you must pass its arguments in the same order and format
        they appeared in that callable's ``sample_args``.

    .. warning::
        The automatic mixed precision is supported in :func:`~torch.cuda.make_graphed_callables` only with disabled
        caching. The context manager `torch.cuda.amp.autocast()` must have `cache_enabled=False`.
    z_make_graphed_callables does not support the autocast caching. Please set `cache_enabled=False`.FTr   z§Modules must not have hooks registered at the time they are passed. However, registering hooks on modules after passing them through make_graphed_callables is allowed.c                 s   s   � | ]}|j d u V  qdS )FN©Úrequires_grad©Ú.0Úbr	   r	   r
   Ú	<genexpr>  ó   € z)make_graphed_callables.<locals>.<genexpr>zœIn any :class:`~torch.nn.Module` passed to :func:`~make_graphed_callables`, only parameters may be trainable. All buffers must have ``requires_grad=False``.c                 s   s   � | ]	}t |tjƒV  qd S r   )Ú
isinstancer)   ÚTensor)rA   Úargr	   r	   r
   rC   "  ó   € zfIn the beta API, sample_args for each callable must contain only Tensors. Other types are not allowed.c                 S   s   g | ]}t |ƒ‘qS r	   )Úlen)rA   Úargsr	   r	   r
   Ú
<listcomp>)  s    z*make_graphed_callables.<locals>.<listcomp>c                 S   s*   g | ]}t |tjjƒrt| ¡ ƒnd ‘qS )r	   )rE   r)   ÚnnÚModuleÚtupleÚ
parameters)rA   Úcr	   r	   r
   rK   *  s    ÿÿc                    s   g | ]
}ˆ | ˆ|  ‘qS r	   r	   ©rA   Úi©Zflatten_sample_argsZper_callable_module_paramsr	   r
   rK   .  s    ÿÿc                 S   ó   g | ]}t j ¡ ‘qS r	   ©r)   r*   r   ©rA   Ú_r	   r	   r
   rK   3  ó    c                 S   rT   r	   rU   rV   r	   r	   r
   rK   4  rX   Nc                 s   ó   � | ]}|j r|V  qd S r   r>   ©rA   Úor	   r	   r
   rC   C  rD   c                 s   rY   r   r>   rQ   r	   r	   r
   rC   D  rD   c                 s   s    � | ]}|j rt |¡V  qd S r   ©r?   r)   Ú
empty_likerZ   r	   r	   r
   rC   E  s   € ÿ
ÿ)ÚoutputsÚinputsÚgrad_outputsZonly_inputsZallow_unused)r   c                 s   s$   � | ]}|j rt |¡nd V  qd S r   r\   rZ   r	   r	   r
   rC   h  ó   € 
ÿc                 s   rY   r   r>   rZ   r	   r	   r
   rC   n  rD   c                 s   rY   r   r>   rQ   r	   r	   r
   rC   o  rD   c                 s   s   � | ]	}|d ur|V  qd S r   r	   rZ   r	   r	   r
   rC   p  rH   é   c	           
         s8   G ‡‡‡‡‡‡‡	fdd„dt jjƒ‰ ‡ ‡‡fdd„}	|	S )Nc                       s@   e Zd Ze‡‡‡‡fdd„ƒZeejjj‡ ‡‡fdd„ƒƒZ	dS )zOmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphedc                    s`   t ˆƒD ]}ˆ|  ¡ ||  ¡ krˆ|  || ¡ qˆ  ¡  tˆtƒs'J ‚tdd„ ˆD ƒƒS )Nc                 s   s   � | ]}|  ¡ V  qd S r   ©ÚdetachrZ   r	   r	   r
   rC   ž  s   € zjmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.forward.<locals>.<genexpr>)ÚrangeÚdata_ptrÚcopy_r   rE   rN   )Úctxr_   rR   )Ú	fwd_graphÚlen_user_argsÚstatic_input_surfaceÚstatic_outputsr	   r
   Úforward–  s   €zWmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.forwardc                    sr   t |ƒt ˆƒks
J ‚tˆ|ƒD ]\}}|d ur$| ¡ | ¡ kr$| |¡ qˆ  ¡  tˆtƒs0J ‚tdd„ ˆD ƒƒS )Nc                 s   s$   � | ]}|d ur|  ¡ n|V  qd S r   rc   r@   r	   r	   r
   rC   ®  ra   zkmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.backward.<locals>.<genexpr>)rI   Úziprf   rg   r   rE   rN   )rh   ÚgradsÚgÚgrad)Ú	bwd_graphÚstatic_grad_inputsÚstatic_grad_outputsr	   r
   Úbackward   s   
€ÿzXmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.Graphed.backwardN)
r"   r#   r$   Ústaticmethodrm   r)   ÚautogradÚfunctionZonce_differentiableru   r	   )rr   ri   rj   rs   rt   rk   rl   r	   r
   ÚGraphed•  s    	ry   c                     s(   t j| Ž }ˆ jt|ƒˆ Ž }t  |ˆ¡S r   )r   Úarg_tree_leavesÚapplyrN   Ztree_unflatten)Ú	user_argsZflatten_user_argsÚout)ry   Úmodule_paramsÚoutput_unflatten_specr	   r
   Úfunctionalized²  s   
zVmake_graphed_callables.<locals>.make_graphed_autograd_function.<locals>.functionalized)r)   rw   ZFunction)
ri   rr   r~   rj   r   rk   rl   rt   rs   r€   r	   )
ry   rr   ri   rj   r~   r   rs   rt   rk   rl   r
   Úmake_graphed_autograd_functionŠ  s   $z>make_graphed_callables.<locals>.make_graphed_autograd_functionc                    s   ‡ ‡‡‡fdd„}|S )Nc                     s   ˆ j ˆkr	ˆ| Ž S ˆ| Ž S r   )Útraining)r|   ©ÚfuncÚgraph_training_stateÚgraphedÚorig_fwdr	   r
   Únew_fwdÎ  s   
zEmake_graphed_callables.<locals>.make_graphed_forward.<locals>.new_fwdr	   )r„   r…   r†   r‡   rˆ   r	   rƒ   r
   Úmake_graphed_forwardÍ  s   z4make_graphed_callables.<locals>.make_graphed_forward)#r)   Úis_autocast_enabledÚis_autocast_cache_enabledÚRuntimeErrorrE   rN   rn   rL   rM   rI   Ú_backward_hooksZ_forward_hooksZ_forward_pre_hooksÚallÚbuffersr   rz   Úappendre   r   r*   r0   r,   r+   Ztree_leavesrw   rq   r'   Ztree_flattenÚreversedr?   ÚreverseÚ	enumerater‚   rm   )&Z	callablesZsample_argsÚnum_warmup_itersZallow_unused_inputr   Zjust_one_callablerP   rJ   Zflatten_argZper_callable_len_user_argsZ"per_callable_static_input_surfacesZ
fwd_graphsZ
bwd_graphsZmempoolr„   rk   rW   r^   Úgrad_inputsZper_callable_static_outputsZ"per_callable_output_unflatten_specri   Zflatten_outputsÚspecZ per_callable_static_grad_outputsZper_callable_static_grad_inputsrl   rr   r~   rt   rs   Zgrad_idxrG   r�   ÚretrR   r†   r‰   r	   rS   r
   Úmake_graphed_callables¾   sì   Eÿ
þÿ
ÿþ
þ
ÿÿù	òÿ

ÿüÿûÿ
3÷r˜   )r=   FN)r1   r9   r)   Ztorch.utilsr   Ú_utilsr   ÚhasattrÚ_CÚ__dict__Útorch._Cr   r   r   r   r   r   r'   r˜   r	   r	   r	   r
   Ú<module>   s$   ÿ	GKÿ