o
    °ƒðjÄ  ã                   @   sŠ   d dl mZ d dlZd dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZmZmZmZ d dlmZ eeƒZG dd	„ d	ƒZG d
d„ deƒZdS )é    )Ú	getLoggerN)ÚFusion)ÚAttentionMaskFormat)ÚFusionUtilsÚNumpyHelper)Ú	NodeProtoÚTensorProtoÚhelperÚnumpy_helper)Ú	OnnxModelc                   @   sR   e Zd ZdZdefdd„Zdefdd„Zdd	„ Zd
d„ Z	de
de
dB fdd„ZdS )ÚAttentionMaskú:
    Fuse Attention subgraph into one Attention node.
    Úmodelc                 C   s2   || _ i | _i | _t|ƒ| _tj| _| ¡ | _	d S ©N)
r   Úmask_indiceÚmask_castedr   Úutilsr   ÚMaskIndexEndÚmask_formatÚget_opset_versionÚopset_version)Úselfr   © r   ú«/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/onnxruntime/transformers/fusion_attention.pyÚ__init__   s   
zAttentionMask.__init__r   c                 C   s
   || _ d S r   )r   )r   r   r   r   r   Úset_mask_format    s   
zAttentionMask.set_mask_formatc                 C   s*   || j v r|| j | ksJ ‚|| j |< d S r   )r   )r   ÚmaskÚ
mask_indexr   r   r   Úset_mask_indice#   s   
zAttentionMask.set_mask_indicec                 C   s    t | jƒdks	J ‚tt| jƒƒS ©Nr   )Úlenr   ÚnextÚiter)r   r   r   r   Úget_first_mask(   s   zAttentionMask.get_first_maskÚmask_2dÚreturnNc              	   C   s`  | j tjkrd S || jv r| j| S | j |¡r!| j |¡\}}n
| j |¡\}}d}|r2|| j	|< | j tj
kr?|| j|< |S | j d¡}| jdk rltjd|g|g| j dd¡d�}|j t ddg¡t d	d
¡g¡ n7d}| j |¡d u r‡| j tj|tjdgdgdd�¡ tjd||g|g| j dd¡d�}|j t d	d
¡g¡ | j |¡ || j|< |S )NTr   é   Ú	ReduceSumÚMaskReduceSum©ÚinputsÚoutputsÚnameÚaxesé   Úkeepdimsr   Úort_const_1_reduce_sum_axesF©r,   Ú	data_typeÚdimsÚvalsÚraw)r   r   ÚNoMaskr   r   Úfind_graph_inputr   Úcast_graph_input_to_int32Úcast_input_to_int32r   r   Úcreate_node_namer   r	   Ú	make_nodeÚ	attributeÚextendÚmake_attributeÚget_initializerÚadd_initializerÚmake_tensorr   ÚINT64Úadd_node)r   r$   ÚcastedÚ
input_nameÚ
_cast_nodeÚoutput_nameÚmask_index_nodeÚ	axes_namer   r   r   Úprocess_mask,   sV   




ü$ûÿ	ü
zAttentionMask.process_mask)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r   r   r   r#   ÚstrrJ   r   r   r   r   r      s    
r   c                (       s,  e Zd ZdZdddddgfdededed	edB d
ededee	 f‡ fdd„Z
dedeeef fdd„Zdedeeef fdd„Zdefdd„Zde	fdd„Zde	de	de	fdd„Zde	d e	d!e	fd"d#„Zd$ed%edB d&edB d'e	dedB f
d(d)„Zd*ed+ed,ed$ed%edB d&edB deeeef fd-d.„Z	/	/		/	/	/	/	dAd*ed+ee	B dB d,ee	B dB d$ed%edB d&edB deded0e	d1e	de	d2ede	de	d3e	d4e	d5ededB f$d6d7„Z	/		/	/	/	/	dBd8e	dB d*ed+ed,ed$ed%ed&ededed9e	d0e	d:e	d;ede	de	d3e	d4e	d<edB dedB f&d=d>„Zd?d@„ Z‡  ZS )CÚFusionAttentionr   NFÚSkipLayerNormalizationÚLayerNormalizationr   Úhidden_sizeÚ	num_headsÚattention_maskÚuse_multi_head_attentionÚ!disable_multi_head_attention_biasÚsearch_op_typesc           	         sh   |rdnd}t ƒ  |||¡ || _|| _|r|nt|ƒ| _|| _|| _d | _d| _	d| _
d | _d| _d S )NÚMultiHeadAttentionÚ	AttentionT)Úsuperr   rS   rT   r   rU   rV   rW   Úmask_filter_valueÚnum_heads_warningÚhidden_size_warningÚshape_inferÚshape_infer_done)	r   r   rS   rT   rU   rV   rW   rX   Úattention_op_name©Ú	__class__r   r   r   l   s   

zFusionAttention.__init__Úconcatr%   c                 C   s‚   t |jƒdkr;| j |jd ¡}| j |jd ¡}t|tjƒr;|jdkr;t|tjƒr;|jdkr;|d |d |d  fS | j| j	fS )aU  
        Detect num_heads and hidden_size from Concat node in the following subgraph:

        SkipLayerNormalization or EmbedLayerNormalization
                        /        |
                     MatMul    Shape
                        |        |
                       Add     Gather(indices=0)
                        |        |
                        |      Unsqueeze
                        |        |
                        |     Concat (*, -1, 12, 64)
                        |     /
                       Reshape
                          |
                       Transpose
        é   é   é   r.   r   )
r    Úinputr   Úget_constant_valueÚ
isinstanceÚnpÚndarrayÚsizerT   rS   )r   rd   rT   Ú	head_sizer   r   r   Ú)get_num_heads_and_hidden_size_from_concat†   s   
ÿ

ÿ
z9FusionAttention.get_num_heads_and_hidden_size_from_concatÚ	reshape_qc                 C   s*  | j  |jd ¡}|du r1| j  |d¡}|dur"|jdkr"|  |¡S t d|jd ¡ | j| j	fS t
|tjƒrIt|ƒdksI|d dksI|d dkrUt d	|¡ | j| j	fS |d }|d }|| }| jdkry|| jkry| jryt d
| j|¡ d| _| j	dkr‘|| j	kr‘| jr‘t d| j	|¡ d| _||fS )zÌDetect num_heads and hidden_size from a reshape node.

        Args:
            reshape_q (NodeProto): reshape node for Q

        Returns:
            Tuple[int, int]: num_heads and hidden_size
        r.   NÚConcatz%s is not initializer.re   rf   r   rg   zGq_shape_value=%s. Expected value are like [0, 0, num_heads, head_size].z>--num_heads is %d. Detected value is %d. Using detected value.Fz@--hidden_size is %d. Detected value is %d. Using detected value.)r   ri   rh   Ú
get_parentÚop_typero   ÚloggerÚdebugrT   rS   rj   rk   rl   r    r]   Úwarningr^   )r   rp   Úq_shape_valuerd   rT   rn   rS   r   r   r   Úget_num_heads_and_hidden_size¥   s:   


ÿÿÿz-FusionAttention.get_num_heads_and_hidden_sizeÚadd_qkc                 C   s�   | j s| jjdd�| _d| _ | jd u rd S | j |jd ¡}| j |jd ¡}|d u s/|d u r7t d|¡ d S ||krCt d|¡ d S |jd S )NT)Úupdater   r.   zone of the inputs of %s is Nonez)the shape of two inputs of %s is not same)r`   r   Úinfer_runtime_shaper_   Úget_edge_shaperh   rt   ru   )r   ry   Úinput_0_shapeÚinput_1_shaper   r   r   Úget_add_qk_strÓ   s   

zFusionAttention.get_add_qk_strc                    s’   ˆ d ‰t t‡fdd„| jƒƒ}t|ƒdkrˆS t|ƒdks J ‚| j d¡}tjd‡ fdd„t| j	ƒD ƒˆg|dd	�}| j 
|¡ | j| j|< ˆS )
NÚ_maskc                    s   | j d ˆ kS r   )Úoutput)Únode)Úmask_output_namer   r   Ú<lambda>î   s    z0FusionAttention.reshape_add_qk.<locals>.<lambda>r.   r   rq   c                    s   g | ]}ˆ ‘qS r   r   )Ú.0Ú_)ry   r   r   Ú
<listcomp>ö   s    z2FusionAttention.reshape_add_qk.<locals>.<listcomp>©r*   r+   r,   Úaxis)ÚlistÚfilterÚnodes_to_addr    r   r:   r	   r;   ÚrangerT   ÚappendÚthis_graph_nameÚnode_name_to_graph_name)r   ry   Úconcat_nodeÚconcat_node_nameÚconcat_add_qk_fp32r   )ry   rƒ   r   Úreshape_add_qkè   s    ûzFusionAttention.reshape_add_qkÚpast_kÚpast_vc                 C   sú   | j  d¡}| j  d¡}|d  dd¡}|d  dd¡}tjd|g|g|dgd�}tjd|g|g|dgd�}| j |¡ | j |¡ | j| j|< | j| j|< | j  d¡}	| dd	¡ dd¡ d
d¡}
tjd||g|
g|	dd�}| j |¡ | j| j|	< |
S )zýConcatenate past_k and past_v inputs to create past_kv input.

        Args:
            past_k (str): name of past K value
            past_v (str): name of past V value

        Returns:
            kv_output_name (str): name of past KV value
        Ú	UnsqueezeÚ_5dÚ.r†   r   )r*   r+   r,   r-   rq   z.valuez.kvÚ_valueÚ_kvrˆ   )	r   r:   Úreplacer	   r;   rŒ   rŽ   r�   r�   )r   r•   r–   Úunsqueeze_k_nameÚunsqueeze_v_nameÚ	k_5d_nameÚ	v_5d_nameÚk_5dÚv_5dr’   Úkv_output_nameÚ	concat_kvr   r   r   r¤     sD   ûû	û	zFusionAttention.concat_kvÚpresent_k_nameÚpresent_v_nameÚkv_nodec                 C   s   d\}}| j  |¡}| j  |¡}|du r(tjtjddd�|d�}| j  || j¡ |du r@tjtjddd�|d�}| j  || j¡ | j  d¡}| j  d¡}	t	j
d||g|g|dd	�}
t	j
d||g|g|	dd	�}| j |
¡ | j |¡ | j| j|< | j| j|	< dS )
a?  Split kv_node containing present KV values into separate present K and present V values.

        Args:
            present_k_name (str): name of output to store present K value in
            present_v_name (str): name of output to store present V value in
            kv_node (str): name of present KV values
        )Úindex_0Úindex_1Nr   Úint64)Údtype©r,   r.   ÚGatherrˆ   )r   r?   r
   Ú
from_arrayrk   Úarrayr@   r�   r:   r	   r;   rŒ   rŽ   r�   )r   r¥   r¦   r§   Úk_indexÚv_indexÚk_dimÚv_dimÚgather_k_nameÚgather_v_nameÚ	present_kÚ	present_vr   r   r   Úsplit_kv8  s:   ûû	zFusionAttention.split_kvÚq_addÚk_addÚv_addÚname_prefixc                 C   sô   | j  |jd ¡p| j  |jd ¡}t |¡}t |¡}t |¡}|d ur<| j  |jd ¡p6| j  |jd ¡}	t |	¡}|d urW| j  |jd ¡pQ| j  |jd ¡}
t |
¡}tj|||fdd�}dt |j	¡ }|d }| j
||j|g|d� |S )Nr.   r   ©r‰   rg   Ú	_qkv_bias©r,   r2   r3   r4   )r   r?   rh   r   Úto_arrayrk   Ú
zeros_likeÚstackÚprodÚshaper@   r2   )r   r¹   rº   r»   r¼   Úq_biasÚqbÚkbÚvbÚk_biasÚv_biasÚqkv_biasÚqkv_bias_dimÚ	bias_namer   r   r   Úcreate_combined_qkv_biase  s(   $


$
$
üz(FusionAttention.create_combined_qkv_biasÚq_matmulÚk_matmulÚv_matmulc           #      C   s"  | j  d¡}|jd |jd kr|jd |jd ksJ ‚| j  |jd ¡}| j  |jd ¡}	| j  |jd ¡}
t |¡}t |	¡}t |
¡}|j|jkrR|j|jksTJ ‚|jd }tj|||fdd� 	|d| f¡}|d }| j
||j|jd |jd g|d� |d }tjd|jd |g|g|d	�}| j| j|< |g}|d
 }| j
|tjdgdgdd� |d }| j
|tjdg|gdd� |d }| j
|tjdgd| gdd� |d }| j
|tjdgd| gdd� |d }| j
|tjdgdgdd� |d }tjd||||g|g| j  d¡d	�}| j| j|j< |d }tjd||||g|g| j  d¡d	�}| j| j|j< |d }tjd||||g|g| j  d¡d	�}| j| j|j< |}|} |}!| |||g¡ | j�r|du�r–| j  |jd ¡�rodnd}"t t | j  |j|" ¡¡¡�r–||jd|" < |}| |¡ | j| j|j< |du�rÎ| j  |jd ¡�r§dnd}"t t | j  |j|" ¡¡¡�rÎ||jd|" < |} | |¡ | j| j|j< |du�r| j  |jd ¡�rßdnd}"t t | j  |j|" ¡¡¡�r||jd|" < |}!| |¡ | j| j|j< | j |¡ || |!fS )a©  Create packed QKV MatMul node before MultiHeadAttention node.
           This is for the scenario where an Attention node should be created but cannot be created
           because past_key and past_value are separate inputs and not one concatenated input.

        Args:
            q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size)
            k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size)
            v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size)
            q_add (NodeProto): name of Add from Q path
            k_add (NodeProto): name of Add from K path
            v_add (NodeProto): name of Add from V path

        Returns:
             q_output (NodeProto): Slice node for Q
             k_output (NodeProto): Slice node for K
             v_output (NodeProto): Slice node for V
        ÚMatMulr   r.   r½   rg   Ú_qkv_weightr¿   Ú_qkv_outr)   Ú_q_start_indexFr1   Ú_k_start_indexÚ_v_start_indexrf   Ú_end_of_qkv_indexÚ_qkv_last_axiséÿÿÿÿÚ_q_outÚSliceÚ_k_outÚ_v_outN)r   r:   rh   r?   r   rÀ   rÄ   rk   rÂ   Úreshaper@   r2   r	   r;   r�   r�   r   rB   r,   r=   rW   ÚanyrŽ   rŒ   )#r   rÏ   rÐ   rÑ   r¹   rº   r»   Úmatmul_node_nameÚq_weightÚk_weightÚv_weightÚqwÚkwÚvwÚdÚ
qkv_weightÚqkv_weight_nameÚqkv_matmul_outputÚ
qkv_matmulÚ	qkv_nodesÚq_slice_nameÚk_slice_nameÚv_slice_nameÚend_of_qkv_nameÚqkv_last_axis_nameÚq_slice_outputÚq_sliceÚk_slice_outputÚk_sliceÚv_slice_outputÚv_sliceÚq_outputÚk_outputÚv_outputÚinitializer_inputr   r   r   Úcreate_packed_qkv_matmul_nodeƒ  s²   ,



"üü

ü

ü

ü
 

 

 

z-FusionAttention.create_packed_qkv_matmul_nodeÚ r�   Úkey_padding_maskÚunidirectionalr¶   r·   Ú
packed_qkvc                 C   s,  |dksJ ‚|dkr|| dkrt  d||¡ dS dd„ | j ¡ jD ƒ}| j d¡}g }|rM|  ||||||¡\}}}| |jd |jd |jd g¡ n`t	|t
ƒr~t	|t
ƒr~| jrl| |jd |jd |jd g¡ nA| |jd |jd |jd g¡ n/t	|tƒr«t	|tƒr«||v r«||v r«| jrŸ| |jd ||g¡ n| |jd ||g¡ ndS | js¾|  ||||¡}| |¡ n| d¡ |rÑ|rÑ| |
|||g¡ n|
sÕ|rÜ| |
|g¡ |	g}|rê|rê| ||g¡ tjd|||d	�}d
|_|j t d|¡¡ |�r|j t dt|ƒ¡¡ |  d¡ |S )aº  Create a MultiHeadAttention node.

        Args:
            q_matmul (NodeProto): name of MatMul from Q path - (batch_size, sequence_length, hidden_size)
            k_matmul (NodeProto): name of MatMul from K path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size)
            v_matmul (NodeProto): name of MatMul from V path - (batch_size, sequence_length, hidden_size) or (batch_size, num_heads, past_sequence_length, head_size)
            q_add (NodeProto): name of Add from Q path
            k_add (NodeProto): name of Add from K path
            v_add (NodeProto): name of Add from V path
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.
            hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning.
            output (str): output name of MHA
            key_padding_mask (str): name of key padding mask
            add_qk (str): name of add after Q x K'
            unidirectional (bool): whether to apply causal attention mask automatically or not
            past_k (str): name of past K value - (batch_size, num_heads, past_sequence_length, head_size)
            past_v (str): name of past V value - (batch_size, num_heads, past_sequence_length, head_size)
            present_k (str): name of present K value - (batch_size, num_heads, sequence_length, head_size)
            present_v (str): name of present V value - (batch_size, num_heads, sequence_length, head_size)
            packed_qkv (bool): whether to combine MatMuls from Q, K, V paths
                               Note: This is for the scenario where an Attention node should be created but cannot be created
                               because past_key and past_value are separate inputs and not one concatenated input.

        Returns:
            Union[NodeProto, None]: the node created or None if failed.
        r   ú9input hidden size %d is not a multiple of num of heads %dNc                 S   s   h | ]}|j ’qS r   r¬   )r…   r‚   r   r   r   Ú	<setcomp>H  ó    zBFusionAttention.create_multihead_attention_node.<locals>.<setcomp>rZ   rþ   rY   r)   úcom.microsoftrT   r   )rt   ru   r   Úgraphrh   r:   rý   r=   r�   rj   r   rW   rO   rÎ   rŽ   r	   r;   Údomainr<   r>   ÚintÚincrease_counter)r   rÏ   rÐ   rÑ   r¹   rº   r»   rT   rS   r�   rÿ   ry   r   r•   r–   r¶   r·   r  Úgraph_input_namesÚmha_node_nameÚ
mha_inputsrô   rö   rø   rÍ   Úmha_outputsÚmha_noder   r   r   Úcreate_multihead_attention_node  sl   /
ú$$$ÿþ
ü
z/FusionAttention.create_multihead_attention_noder   Úfirst_inputÚ
add_qk_strÚcausalÚscalec           5      C   sj  |dksJ ‚|	dkr|	| dkrt  d|	|¡ dS d}|du r)|du r)|du r)d}| j |jd ¡}| j |jd ¡}| j |jd ¡}d\}}}|r‹| j |jd ¡p\| j |jd ¡}| j |jd ¡pn| j |jd ¡}| j |jd ¡p€| j |jd ¡}|r‰|r‰|r‰|s‹dS |du r›t|jd › d�ƒ dS t |¡}t |¡}t |¡}|j|jks²J ‚|jd }|jd }|jd }||  krÌ|ksÏJ ‚ J ‚|	dkrÞ|	|krÞt  	d	|	|¡ d} |j|jkrèd} t
 |jdd… ¡}!t
 |jdd… ¡}"t
 |jdd… ¡}#d}$| �rt
j|||fdd
�}%|!|" |# }$nt
j|||fdd
�}%d|! }$d}&d}'|�r‹t |¡}(t |¡})t |¡}*t
 |(j¡}+t
 |)j¡},t
 |*j¡}-|+|,  k�r_|!k�sbJ ‚ J ‚|-|#k�siJ ‚| �r}t
j|(|)|*fdd
�}'|+|, |- }&nt
j|(|)|*fdd
�}'d|+ }&| j d¡}.| j�s¥| j|.d |j|t|$ƒg|%d� |�r·| j|.d |jt|&ƒg|'d� | j�rð|�rÅt  d¡ dS |jd |jd |jd |.d g}/|du�rà|/ |¡ tjd|/|g|.d�}0|  d¡ nr|
|.d |�rû|.d ndg}/|du�r	|/ |¡ n|/ d¡ |�o|}1|1�r!|  ||¡}2|/ |2¡ |�r1|1�s,|/ d¡ |/ |¡ |g}3|�rT|�rT| dd¡ dd¡ dd¡}4|3 |4¡ |  |||4¡ tjd|/|3|.d�}0|  d¡ d|0_|0j t d|¡g¡ |�r~|0j t dd¡g¡ |du�rŽ|0j t d|¡g¡ | �rŸ|0j t d|!|"|#g¡g¡ | jdu�r³|0j t dt | jƒ¡g¡ |0S )a>  Create an Attention node.

        Args:
            mask_index (str | None): mask input
            q_matmul (NodeProto): MatMul node in fully connection for Q
            k_matmul (NodeProto): MatMul node in fully connection for K
            v_matmul (NodeProto): MatMul node in fully connection for V
            q_add (NodeProto): Add bias node in fully connection for Q
            k_add (NodeProto): Add bias node in fully connection for K
            v_add (NodeProto): Add bias node in fully connection for V
            num_heads (int): number of attention heads. If a model is pruned, it is the number of heads after pruning.
            hidden_size (int): hidden dimension. If a model is pruned, it is the hidden dimension after pruning.
            first_input (str): first input name
            output (str): output name
            add_qk_str (str): name of Add node after Q x K'
            causal: whether it is uni-directional mask.
            past_k (str): name of input for past K value
            past_v (str): name of input for past V value
            present_k (str): name of output to store present K value
            present_v (str): name of output to store present V value
            scale: scale before softmax

        Returns:
            Union[NodeProto, None]: the node created or None if failed.
        r   r  NTFr.   )NNNzl is not an initializer. Please set do_constant_folding=True in torch.onnx.export to unblock attention fusionz„Input hidden size (%d) is not same as weight matrix dimension of q,k,v (%d). Please provide a correct input hidden size or pass in 0r½   rg   rZ   rÓ   r¿   r¾   zVMultiHeadAttention does not support relative_position_bias: cannot fuse the attention.rY   r)   rþ   z.keyÚ_keyr™   r†   r  rT   r   r  Úqkv_hidden_sizesr\   )!rt   ru   r   r?   rh   Úprintr   rÀ   rÄ   rv   rk   rÃ   ÚconcatenaterÂ   r:   rV   r@   r2   r  r�   rŽ   r	   r;   r	  r¤   rœ   r¸   r  r<   r=   r>   r\   Úfloat)5r   r   rÏ   rÐ   rÑ   r¹   rº   r»   rT   rS   r  r�   r  r  r•   r–   r¶   r·   r  Úhas_biasrâ   rã   rä   rÅ   rÉ   rÊ   rå   ræ   rç   Ú
qw_in_sizeÚ
kw_in_sizeÚ
vw_in_sizeÚis_qkv_diff_dimsÚqw_out_sizeÚkw_out_sizeÚvw_out_sizeÚqkv_weight_dimré   rÌ   rË   rÆ   rÇ   rÈ   Úq_bias_shapeÚk_bias_shapeÚv_bias_shapeÚattention_node_nameÚattention_inputsÚattention_nodeÚpast_existsÚpast_kvÚattention_outputsÚ
present_kvr   r   r   Úcreate_attention_nodeŒ  s  .
$$$ÿ





ü


 
üü
ü

üý






ü

ÿz%FusionAttention.create_attention_nodec           ;      C   sV  |}|}|j dkr| j |dd¡}|d ur|}nd S | j |g d¢g d¢¡}d }|d ur4|\}	}	}
}}n| j |g d¢g d¢¡}|d urK|\}	}}}nd S g }t|jƒD ]\}}||vr]qT||d jd krgqT| |¡ qTt|ƒdkrud S |d }| j |d	d¡}|d ur¹||jd  }|d ur§t|ƒd
kr§|d }|j dkr¥|jd }n,d S |d ur·t|ƒdkr·|jd }nd S |j dkrÑ|| }|D ]}|j dkrÐ|jd }qÄ|| }|j dkræt|jƒdkræ|jd }|| }dd„ |D ƒ}| 	d¡dkrúd S | j |g d¢g d¢¡}|d u �rt
 d¡ d S |\}	}	}}d}d}d}d}g d¢g d¢fg d¢g d¢fg d¢g d¢fg d¢g d¢fg d¢g d¢fg d¢g d ¢fd!œ}d } | ¡ D ]6\}!}"| j ||"d |"d ¡} | d u �rj�qS|!d"k�rrd#}n|!d$k�rzd#}n|!d%k�r‚d#}n|!d&k�r‰d#} | d u �r–t
 d'¡ d S d }#d }$d }%d }&|�r¨| \}	}%}$}	n$|�r²| \}	}#}%}$n|�r»| \}	}	}$n|�rÆ| \}	}#}$}&}	n| \}	}#}	}$|&�pÐ|$}&| j |&g d¢g d(¢¡}'|'d u �rú| j |&g d)¢g d*¢¡}'|'d u �rút
 d+¡ d S |'d, }(|'d- })|'d. }*|$}+|�r'| j |$d	d/gdd g¡},|,d u �r#t
 d0¡ d S |,\}+}	| j |+g d¢|�r3dndddd g¡}-|-d u �rW| j |$g d1¢g d2¢¡}-|-d u �rWt
 d3¡ d S |-d- }.|-d. }/d }0d4}1|�r‡| j |%g d5¢g d¢fg d6¢g d¢fg d7¢g d8¢fg|¡\}	}0}	n_|�r»| j |%g d9¢g d8¢fg d6¢g d¢fg|¡\}	}0}	|#d u�rº|  |#¡}1|1d u �rºt
 d:|#¡ d S n+|�r¿n'| j |#g d;¢g d<¢fg d=¢g d>¢fg d?¢g d@¢fg dA¢g dB¢fg|¡\}	}0}	|�sõ|0d u �rõt
 dC¡ d S |�s0t|0ƒdk�r0| j |0d ¡\}	}2|2d u �s"t|2tjƒ�r"|2jdk�r"t|2ƒdk�r$d S t|2ƒdDk�r0t|2ƒ| _|jd |k�r%|*jd |k�r'|/jd |k�r)|�sV| j |0d. jd ¡nd }3|d u �r_|
n|}4|  |(¡\}5}6|5dk�sr|6dk�ryt
 dE¡ d S | j|3|*|/||)|.||5|6||4jd |1dF�}7|7d u �r”d S | j |7¡ | j| j|7j< |d u�rá|jd }8dG|8 }9| jdH|8 t j!dgdd|5t"|6|5 ƒgddI�}:| j #t$ %dJ|4jd |:jg|9gdK|8 ¡| j¡ |9|jd< | j& '|4||g¡ | j& '| ¡ | j& '| j(�sù|'n|'d d.… ¡ | j& '| j(�s	|-n|-d d.… ¡ | j& '| j(�s|n|d d.… ¡ d#| _)d S d S d S d S )LNrR   ÚAddr   )r-  rÒ   ÚReshapeÚ	TransposerÒ   )NNr   r   r   )r-  ÚEinsumr/  rÒ   )r.   Nr   r   r.   ÚMulrf   é   rQ   re   c                 S   s   g | ]}|j ‘qS r   )rs   )r…   Úchildr   r   r   r‡   É  r  z(FusionAttention.fuse.<locals>.<listcomp>rÒ   rg   )r/  r.  r-  rÒ   )r.   r   r   Nz&fuse_attention: failed to match v pathF)ÚSoftmaxr-  ÚDivrÒ   )r   r   Nr   )r4  r-  r1  rÒ   )r4  ÚWhererÒ   r5  )r   r   rf   r   )r4  r-  r6  rÒ   )r   r   r   rf   )r4  r5  rÒ   )r   r   r   )r4  r-  rÒ   r1  ÚSqrt)r   r   Nr   r.   )Úpath1Úpath2Úpath3Úpath4Úpath5Úsdpar:  Tr;  r<  r=  z'fuse_attention: failed to match qk path)r   r   r   N)r5  r/  r.  r-  rÒ   )r   r   r   r   Nz&fuse_attention: failed to match q pathéýÿÿÿéþÿÿÿrÚ   r7  z/fuse_attention: failed to match mul sqrt q path)r/  r/  r.  r-  rÒ   )r.   r   r   r   Nz&fuse_attention: failed to match k pathrþ   )ÚExpandr.  ÚEqual)rA  r—   r—   )ÚCastr@  r.  rA  )r   r   r   r   )rB  rA  r—   r—   z6fuse_attention: failed to verify shape inference of %s)r1  ÚSubrB  r—   r—   )Nr   r.   r   r   )r1  rC  r—   r—   )Nr   r.   r   )r6  rB  rC  r@  r—   r—   )Nr   r   r.   r   r   )r6  rB  rC  rB  r@  r—   r—   )Nr   r   r.   r   r   r   z)fuse_attention: failed to match mask pathiðØÿÿzmFailed to detect num_heads and hidden_size for Attention fusion. Please specify those parameters in argument.)r   rÏ   rÐ   rÑ   r¹   rº   r»   rT   rS   r  r�   r  Úedge_modified_Úshape_modified_tensorr1   r.  Úreshape_modified_)*rs   r   Úmatch_parentÚmatch_parent_pathÚ	enumeraterh   r�   rŽ   r    Úcountrt   ru   ÚitemsÚmatch_parent_pathsr   Úget_constant_inputrj   rk   rl   rm   r  r\   rU   rJ   rx   rv   r,  rŒ   r�   r�   r,   r@   r   rB   r  rC   r	   r;   Únodes_to_remover=   rV   Úprune_graph);r   r‚   Úinput_name_to_nodesÚoutput_name_to_nodeÚnormalize_nodeÚ
start_nodeÚadd_before_layernormrí   Úeinsum_noder†   Úreshape_qkvÚtranspose_qkvÚ
matmul_qkvÚother_inputsÚ_iÚ
node_inputÚ
root_inputÚmul_before_layernormÚmul_childrenÚlayernorm_nodeÚchildrenr3  Úparent_nodeÚchildren_typesÚv_nodesÚadd_vÚmatmul_vÚ
is_distillÚis_distill_addÚis_no_mask_attentionÚis_sdpaÚqk_pathsÚqk_nodesÚkÚvry   Ú	matmul_qkÚwhere_qkÚafter_qÚq_nodesrp   Úadd_qÚmatmul_qÚafter_kÚmul_k_nodesÚk_nodesÚadd_kÚmatmul_kÚ
mask_nodesr  Úmul_valr   Úattention_last_nodeÚq_num_headsÚq_hidden_sizeÚnew_nodeÚunique_indexÚnew_edgeÚshape_tensorr   r   r   Úfusev  sÞ  
ýÿ



€


ú	








ý



ÿ
ý

ýù	þ
ú


€û
÷


ÿ
0 ÿô


ûüù
	   
¼zFusionAttention.fuse)rþ   rþ   Frþ   rþ   rþ   rþ   F)rþ   Frþ   rþ   rþ   rþ   N)rK   rL   rM   rN   r   r  r   ÚboolrŠ   rO   r   r   Útuplero   rx   r   r”   r¤   r¸   rÎ   rý   r  r  r,  r‚  Ú__classcell__r   r   rb   r   rP   g   s(   	øþýüûúùø.7-þýüû
úþýüûúù
ø îþ
ý
üûúùø	÷
öõôóòñðïî
í íþýüûúùø	÷
öõôóòñðïîí
ì krP   )Úloggingr   Únumpyrk   Úfusion_baser   Úfusion_optionsr   Úfusion_utilsr   r   Úonnxr   r   r	   r
   Ú
onnx_modelr   rK   rt   r   rP   r   r   r   r   Ú<module>   s   V