o
    °ƒðj™( ã                   @   sf  d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ ddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZmZmZmZmZm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z' ddl(m)Z* ddl+m,Z, ddl-m.Z/ ddl0m1Z1m2Z2 e 3d¡Z4G dd„ deƒZ5dˆde6e7 dB dej8fdd„Z9dej8fdd„Z:dej8fdd„Z;d‰de7de<fdd „Z=d‰de7de<de<fd!d"„Z>d#e7d$e<d%e<de%fd&d'„Z?d(ejd)efd*d+„Z@d(ejd)efd,d-„ZAd(ejd)efd.d/„ZB	0	1		dŠd2ed3ed4e7d5eCd6eDdB d7eDdB fd8d9„ZEd:ed;efd<d=„ZF	1d‹d(ed5eCde6e fd>d?„ZGd@dA„ ZHdBdC„ ZIdDdE„ ZJdFefdGdH„ZKdFedIe<dJe<de<fdKdL„ZLdFefdMdN„ZMdOedPe7fdQdR„ZNdg fdOedSeCdTe6eC fdUdV„ZOdOefdWdX„ZPdOedPe7fdYdZ„ZQ		[	\dŒdOed]e7d^eCd_eCd`eCf
dadb„ZRdFefdcdd„ZSdFefdedf„ZTdgefdhdi„ZUd‰dje7de<fdkdl„ZV	d‰dje7dme7de<de<fdndo„ZWdpdq„ ZXe5jYfdej8dre5fdsdt„ZZdej8dOee!B duej[dvej[dweCdxeCdye6e6eC  deDe7ef fdzd{„Z\d|d}„ Z]		~d�dej8de6e7 dB d€e<fd�d‚„Z^dˆdej8de6e7 dB fdƒd„„Z_dŽde6e7 dB de6e7 dB fd…d†„Z)e`d‡k�r1e)ƒ  dS dS )�aë  
This converts GPT2 or T5 model to onnx with beam search operator.

Example 1: convert gpt2 model with beam search:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx

Example 2: convert gpt2 model with beam search containing specific cuda optimizations:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx --use_gpu                       --past_present_share_buffer --use_decoder_masked_attention

Example 3: convert gpt2 model with beam search with mixed precision and enable SkipLayerNorm strict mode:
    python convert_generation.py -m gpt2 --output gpt2_beam_search.onnx --use_gpu -p fp16 --use_sln_strict_mode

Example 4: convert T5 model with beam search in two steps:
    python -m models.t5.convert_to_onnx -m t5-small
    python convert_generation.py -m t5-small --model_type t5                     --decoder_onnx ./onnx_models/t5-small_decoder.onnx                       --encoder_decoder_init_onnx ./onnx_models/t5-small_encoder.onnx          --output ./onnx_models/t5_small_beam_search.onnx

Example 5: convert T5 model with beam search. All in one step:
    python convert_generation.py -m t5-small --model_type t5 --output t5_small_beam_search.onnx

Example 6: convert T5 model with beam search containing specific cuda optimizations. All in one step:
    python convert_generation.py -m t5-small --model_type t5 --output t5_small_beam_search.onnx           --use_gpu --past_present_share_buffer --use_decoder_masked_attention

Example 7: convert MT5 model with external data file like mt5-base-beamsearch.onnx.data in below example.
    python convert_generation.py -m google/mt5-base --model_type mt5 --output mt5-base-beamsearch.onnx -e

Example 8: convert gpt2 model with greedy search:
    python convert_generation.py -m gpt2 --output gpt2_greedy_search.onnx --num_beams 1 --num_return_sequences 1

Example 9: convert gpt2 model with sampling:
    python convert_generation.py -m gpt2 --output gpt2_sampling.onnx --num_beams 1 --num_return_sequences 1 --top_p 0.6
é    N)ÚEnum)ÚPath)ÚAny)Ú	PrecisionÚsetup_logger)ÚNumpyHelper)Ú
GraphProtoÚ
ModelProtoÚTensorProto)Ú	OnnxModel)Ú
GPT2ConfigÚGPT2LMHeadModelÚGPT2TokenizerÚ	MT5ConfigÚMT5ForConditionalGenerationÚT5ConfigÚT5ForConditionalGenerationÚT5Tokenizer)ÚGraphOptimizationLevelÚInferenceSessionÚSessionOptionsÚget_available_providers)Úmain)ÚPRETRAINED_GPT2_MODELS)Úexport_onnx_models)ÚPRETRAINED_MT5_MODELSÚPRETRAINED_T5_MODELSÚ c                   @   s    e Zd ZdZdZdZdd„ ZdS )ÚGenerationTypeÚbeam_searchÚgreedy_searchÚsamplingc                 C   s   | j S ©N)Úvalue)Úself© r%   ú­/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/onnxruntime/transformers/convert_generation.pyÚ__str___   s   zGenerationType.__str__N)Ú__name__Ú
__module__Ú__qualname__Ú
BEAMSEARCHÚGREEDYSEARCHÚSAMPLINGr'   r%   r%   r%   r&   r   Z   s
    r   ÚargvÚreturnc                 C   sB  t  ¡ }| d¡}|jdddtdd tt t ¡ d� |jdd	td
g d¢dd g d¢¡ d� |jdd	tt	j
 dd¡dd� |jdd	tddd� |jdd	tddd� |jdd	ddd� |jd	d� | d¡}|jddtdd� |jd d!d	ttjjtjjtjjgd"d� |jd#d$d	d%d&gd'd(� |jd)d*d	dd+d� |jd	d,� |jd-d.d	dd/d� |jd	d0� |jd1d2d	dd3d� |jd	d4� |jd5d6d	dd7d� |jd	d8� |jd9d:d	dd;d� |jd	d<� |jd=d	dd>d� |jd	d?� | d@¡}|jdAd	ddBd� |jd	dC� |jdDd	ddEd� |jd	dF� |jdGd	ddH� |jd	dI� |jdJtd	dKdLdM� |jdNd	ddOd� |jd	dP� |jdQd	ddRd� |jd	dS� |jdTd	ddUd� |jd	dV� |jdWd	ddXd� |jd	dY� |jdZd	dd[d� |jd	d\� |jd]d	dd^d� |jd	d_� |jd`d	ddad� |jd	db� | dc¡}|jddtd	dedfdM� |jdgtd	dhdidM� |jdjtd	dkdldM� |jdmtd	dedndM� |jdotd	dedpdM� |jdqtd	dedrdM� |jdstd	dtdudM� |jdvtd	dtdwdM� |jdxtd	tdyƒ dzdM� |jd{td	ded|dM� |jd}td	d~ddM� |jd€td	dKd�dM� |jd‚td	dƒd„dM� |jd…td	dƒd†dM� |jd‡td	dƒdˆdM� | d‰¡}|jdŠd	dd‹d� |jd	dŒ� |jd�d	ddŽd� |jd	d�� |jd�d	dd‘d� |jd	d’� |jd“d	dd”d� |jd	d•� |jd–d	dd—d� |jd	d˜� |jd™d	tdedšd� |jd›d	ddœd� |jd	d�� | | ¡}|S )žzªParse arguments

    Args:
        argv (Optional[List[str]], optional): _description_. Defaults to None.

    Returns:
        argparse.Namespace: Parsed arguments.
    zInput optionsz-mú--model_name_or_pathTzEPytorch model checkpoint path, or pretrained model name in the list: ú, )ÚrequiredÚtypeÚhelpz--model_typeFÚgpt2)r5   Út5Úmt5z*Model type (default is gpt2) in the list: )r2   r3   ÚdefaultÚchoicesr4   ú--cache_dirÚ.Úcache_modelsz%Directory to cache pre-trained models)r2   r3   r8   r4   z--decoder_onnxr   zLPath of onnx model for decoder. Specify it when you have exported the model.z--encoder_decoder_init_onnxzgPath of ONNX model for encoder and decoder initialization. Specify it when you have exported the model.z	--verboseÚ
store_truezPrint more information)r2   Úactionr4   )ÚverbosezOutput optionsú--outputz,Output path for onnx model with beam search.z-pú--precisionzTPrecision of model to run. fp32 for full precision, fp16 for half or mixed precisionz-bú--op_block_listÚ*ÚautozÿDisable certain onnx operators when exporting model to onnx format. When using defaultvalue for gpt2 type of model fp16 precision, it will be set to ["Add", "LayerNormalization", "SkipLayerNormalization", "FastGelu"]. Other situation, it will be set to [])r2   Únargsr8   r4   z-eú--use_external_data_formatz!save external data for model > 2G)Úuse_external_data_formatz-sz--run_shape_inferencezrun shape inference)Úrun_shape_inferencez-dpvsz--disable_pad_vocab_sizez³Do not pad logits MatMul weight to be a multiple of 8 along the dimension where dim value is the vocab size. The logits MatMul may hence be of poor performance for fp16 precision.)Údisable_pad_vocab_sizez-dsgdz,--disable_separate_gpt2_decoder_for_init_runz™Do not create separate decoder subgraphs for initial and remaining runs. This does not allow for optimizations based on sequence lengths in each subgraph)Ú*disable_separate_gpt2_decoder_for_init_runz-iz--disable_shared_initializersz™do not share initializers in encoder and decoder for T5 or in the init decoder and decoder for GPT2. It will increase memory usage of t5/mt5/gpt2 models.)Údisable_shared_initializersz--encoder_decoder_initzbAdd decoder initialization to encoder for T5 model. This is legacy format that will be deprecated.)Úencoder_decoder_initz6Beam search parameters that stored in the output modelz--output_sequences_scoreszoutput sequences scores)Úoutput_sequences_scoresz--output_token_scoreszoutput token scores)Úoutput_token_scoresz--early_stopping)r2   r>   )Úearly_stoppingz--no_repeat_ngram_sizer   zNo repeat ngram size)r3   r2   r8   r4   z--vocab_maskz\Enable vocab_mask. This mask applies only to every generated token to filter some bad words.)Ú
vocab_maskz--past_present_share_bufferzWUse shared buffer for past and present, currently work for gpt2 greedy/sampling search.)Úpast_present_share_bufferz--use_decoder_masked_attentionzëUses `DecoderMaskedSelfAttention` or `DecoderMaskedMultiHeadAttention` to optimize the decoding Attention computation. Must be used with `past_present_share_buffer`. Currently, only Attention head sizes of 32, 64 and 128 are supported.)Úuse_decoder_masked_attentionz--prefix_vocab_maskzeEnable prefix_vocab_mask. This mask can be used to filter bad words in the first generated token only)Úprefix_vocab_maskz--custom_attention_maskz]Enable custom_attention_mask. This mask can be used to replace default encoder attention mask)Úcustom_attention_maskz--presence_maskz!Presence mask for custom sampling)Úpresence_maskz--seedzRandom seed for sampling op)ÚseedzYBeam search parameters not stored in the output model, for testing parity and performancez--min_lengthé   zMin sequence lengthz--max_lengthé2   zMax sequence lengthz--num_beamsé   z	Beam sizez--num_return_sequencesz&Number of return sequence <= num_beamsz--length_penaltyz<Positive. >1 to penalize and <1 to encourage short sentence.z--repetition_penaltyz-Positive. >1 to penalize and <1 to encourage.z--temperatureç      ð?z6The value used to module the next token probabilities.z--top_pzTop P for samplingz--filter_valueÚInfzFilter value for Top P samplingz--min_tokens_to_keepzAMinimum number of tokens we keep per batch example in the output.z--presence_penaltyç        z%presence penalty for custom sampling.z--customz&If 1 customized top P logic is appliedz--vocab_sizeéÿÿÿÿzIVocab_size of the underlying model used to decide the shape of vocab maskz--eos_token_idzKcustom eos_token_id for generating model with existing onnx encoder/decoderz--pad_token_idzKcustom pad_token_id for generating model with existing onnx encoder/decoderz0Other options for testing parity and performancez--use_sln_strict_modez_Enable strict mode for SLN in CUDA provider. This ensures a better accuracy but will be slower.)Úuse_sln_strict_modeú	--use_gpuz)use GPU for inference. Required for fp16.)Úuse_gpuz--disable_parityzdo not run parity test)Údisable_parityz--disable_perf_testzdo not run perf test)Údisable_perf_testz--torch_performanceztest PyTorch performance)Útorch_performancez--total_runsz4Number of times of inference for latency measurementz--save_test_dataz-save test data for onnxruntime_perf_test tool)Úsave_test_data)ÚargparseÚArgumentParserÚadd_argument_groupÚadd_argumentÚstrÚjoinr   r   r   ÚosÚpathÚset_defaultsr   ÚFLOAT32r#   ÚFLOAT16ÚintÚfloatÚ
parse_args)r.   ÚparserÚinput_groupÚoutput_groupÚmodel_groupÚbeam_parameters_groupÚ
test_groupÚargsr%   r%   r&   Úparse_argumentsc   sš  	
ÿû	ú	ûûûü
üù
úûûûûûü
üüûüüüüüüüÿûûûûûûûûûûûû
üüüüüûü
rz   ry   c                 C   s¾   | j }d|d| jdd| jddddd	g}| jr| d
| jg¡ | jr&| d¡ | jr.| d¡ t| j	ƒr?| dg¡ | | j	¡ | jt
jjkrM| jsMJ dƒ‚| jrXt d|› �¡ t|d� dS )zqConvert GPT-2 model to onnx

    Args:
        args (argparse.Namespace): arguments parsed from command line
    r0   r@   z--optimize_onnxrA   z--test_runsÚ1z--test_casesÚ10z--overwriter:   r_   rF   rB   zEfp16 or mixed precision model cannot run in CPU. Please add --use_gpuzarguments for convert_to_onnx:)r.   N)Úmodel_name_or_pathÚdecoder_onnxÚ	precisionÚ	cache_dirÚextendr`   ÚappendrG   ÚlenÚop_block_listr   ro   r#   r?   ÚloggerÚinfoÚconvert_gpt2_to_onnx)ry   Ú
model_nameÚ	argumentsr%   r%   r&   Úgpt2_to_onnx÷  s8   ô


rŠ   c                 C   s�   t | j| jt| jƒj| j| j| jt	j
jk| jddddd| j| j| jt	j
jkd�}t d|d › �¡ t d|d › �¡ |d | _|d | _dS )	znConvert T5 model to onnx

    Args:
        args (argparse.Namespace): arguments parsed from command line
    FT)r}   r€   Ú
output_dirr`   rG   Úoptimize_onnxr   r?   Úuse_decoder_start_tokenÚ	overwriteÚdisable_auto_mixed_precisionÚuse_int32_inputsÚ
model_typerL   Úforce_fp16_iozonnx model for encoder: r   zonnx model for decoder: rW   N)Úexport_t5_onnx_modelsr}   r€   r   ÚoutputÚparentr`   rG   r   r   ro   r#   r‘   rL   r…   ÚdebugÚencoder_decoder_init_onnxr~   )ry   Úpathsr%   r%   r&   Ú
t5_to_onnx$  s*   
ñ
r™   TÚ	onnx_pathrG   c                 C   sP   ddl m} tj| dd�}|j|ddd�}|r!tj|| |d� d	S t d¡ d	S )
zÇShape inference on an onnx file, which will be overwritten.

    Args:
        onnx_path (str): Path of onnx model
        use_external_data_format(bool): output tensors to external data or not.
    r   )ÚSymbolicShapeInferenceT©Úload_external_dataF)Ú
auto_mergeÚguess_output_rank©Úsave_as_external_dataz4Failed to run symbolic shape inference on the model.N)	Ú&onnxruntime.tools.symbolic_shape_inferr›   ÚonnxÚ
load_modelÚinfer_shapesr   Úsaver…   Úwarning)rš   rG   r›   ÚmodelÚoutr%   r%   r&   Úshape_inferenceB  s   rª   c                 C   s¤  t j| dd�}|jjd j}t|ƒ}| ¡ }||v sJ ‚|| }|jdkr'dS d}| |j	d ¡}|du rR| 
|dd¡}	|	du rBdS | |	j	d ¡}|du rPdS d}|jtjjkr[dS t|jƒd	krddS |jd }
|
d
 dkrqdS t |
d
 ¡d
 }||
 }|jrÆ|r¢tj|jd |ftjd�}tjt |¡|fdd�}||jd< ntj||jd ftjd�}tjt |¡|fdd�}||jd< | ¡ |_ndS tj|| |d� dS )zâPad the logits MatMul weight in the provided decoder model, which will be overwritten.

    Args:
        onnx_path (str): Path of onnx model
        use_external_data_format(bool): output tensors to external data or not.
    Trœ   r   ÚMatMulFrW   NÚ	Transposeé   é   ©Údtype©Úaxisr    )r£   r¤   Úgraphr”   Únamer   Úoutput_name_to_nodeÚop_typeÚget_initializerÚinputÚmatch_parentÚ	data_typer
   ÚDataTypero   rƒ   ÚdimsÚmathÚceilÚraw_dataÚnpÚzerosÚfloat16Úconcatenater   Úto_arrayÚtobytesr¦   )rš   rG   Údecoder_model_protoÚlogits_output_nameÚdecoder_modelrµ   Úmatmul_nodeÚpad_along_axis_1Úlogits_weightÚtranspose_before_matmulÚactual_vocab_sizeÚpadded_vocab_sizeÚpaddingÚpadding_dataÚweight_with_paddingr%   r%   r&   Úpad_weights_of_logits_matmulT  sN   


rÒ   Ú
model_pathr`   r^   c                    sx   t ƒ }tj|_|rddgndg}|r3dtƒ vrtdƒ‚t d¡ |r3ddi}d|i‰ ‡ fdd„|D ƒ}t| ||d	�}|S )
a„  Create OnnxRuntime session.

    Args:
        model_path (str): onnx model path
        use_gpu (bool): use GPU or not
        use_sln_strict_mode (bool): use strict mode for skip layer normalization or not

    Raises:
        RuntimeError: CUDAExecutionProvider is not available when --use_gpu is specified.

    Returns:
        onnxruntime.InferenceSession: The created session.
    ÚCUDAExecutionProviderÚCPUExecutionProviderz5CUDAExecutionProvider is not available for --use_gpu!zuse CUDAExecutionProviderÚ"enable_skip_layer_norm_strict_modeTc                    s$   g | ]}|ˆ v r|ˆ | fn|‘qS r%   r%   )Ú.0r´   ©Úprovider_optionsr%   r&   Ú
<listcomp>»  s    ÿz&create_ort_session.<locals>.<listcomp>)Ú	providers)	r   r   ÚORT_DISABLE_ALLÚgraph_optimization_levelr   ÚRuntimeErrorr…   r†   r   )rÓ   r`   r^   Úsess_optionsÚexecution_providersÚcuda_provider_optionsÚort_sessionr%   rØ   r&   Úcreate_ort_session¢  s   


ÿrã   r³   r   c              	   C   sî  |t jjk}t| jƒ}|d }|dksJ ‚g d¢dd„ t|ƒD ƒ }t| jƒt|ƒkr:tdt|ƒ› dt| jƒ› �ƒ‚t|ƒD ]E\}}| j| j|kr[td|› d	|› d| j| j› �ƒ‚t	j
}|dkrj|rgt	jnt	j}| j| jjj}	|	|krƒtd|› d
|› d|	› �ƒ‚q>t d¡ dgdd„ t|ƒD ƒ }
t| jƒt|
ƒkr­tdt|
ƒ› dt| jƒ› �ƒ‚t|
ƒD ]>\}}| j| j|krÎtd|› d	|› d| j| j› �ƒ‚|rÓt	jnt	j}| j| jjj}||krïtd|› d
|› d|› �ƒ‚q±t d¡ dS )aæ  Verify GPT-2 subgraph

    Args:
        graph (onnx.GraphProto): onnx graph of GPT-2
        precision (Precision): Precision (FLOAT16 or FLOAT32) of the model.

    Raises:
        ValueError: Number of inputs not expected.
        ValueError: Input name is not expected.
        ValueError: Input data type is not expected.
        ValueError: Number of outputs not expected.
        ValueError: Output name is not expected.
        ValueError: Output data type is not expected.
    é   rW   )Ú	input_idsÚposition_idsÚattention_maskc                 S   ó   g | ]}d |› �‘qS )Úpast_r%   ©r×   Úir%   r%   r&   rÚ   Ø  ó    z(verify_gpt2_subgraph.<locals>.<listcomp>ú Number of inputs expected to be ú. Got úInput ú is expected to be ú$ is expected to have onnx data type z:Verifying GPT-2 graph inputs: name and data type are good.Úlogitsc                 S   rè   )Úpresent_r%   rê   r%   r%   r&   rÚ   é  rì   ú!Number of outputs expected to be úOutput z;Verifying GPT-2 graph outputs: name and data type are good.N)r   ro   r#   rƒ   r¸   ÚrangeÚ
ValueErrorÚ	enumerater´   r
   ÚINT32ÚFLOATr3   Útensor_typeÚ	elem_typer…   r†   r”   )r³   r   Ú
is_float16Úinput_countÚlayer_countÚexpected_inputsrë   Úexpected_inputÚexpected_typeÚ
input_typeÚexpected_outputsÚexpected_outputÚoutput_typer%   r%   r&   Úverify_gpt2_subgraphÃ  s>   
"ÿ
"ÿ
r  c              	   C   s<  |t jjk}|rtjntj}t| jƒ}|d d }|dksJ ‚ddg}t|ƒD ]}| d|› �¡ | d|› �¡ q't|ƒD ]}| d|› �¡ | d	|› �¡ q>t| jƒt|ƒkrit	d
t|ƒ› dt| jƒ› �ƒ‚t
|ƒD ]?\}}| j| j|krŠt	d|› d|› d| j| j› �ƒ‚|dk r‘tjn|}	| j| jjj}
|
|	kr¬t	d|› d|	› d|
› �ƒ‚qmdg}t|ƒD ]}| d|› �¡ | d|› �¡ q´t| jƒt|ƒkrßt	dt|ƒ› dt| jƒ› �ƒ‚t
|ƒD ]8\}}| j| j|k�rt	d|› d|› d| j| j› �ƒ‚| j| jjj}||k�rt	d|› d|› d|› �ƒ‚qãdS )áð  Verify T5 decoder subgraph

    Args:
        graph (onnx.GraphProto): onnx graph of T5 decoder
        precision (Precision): Precision (FLOAT16 or FLOAT32) of the model.

    Raises:
        ValueError: Number of inputs not expected.
        ValueError: Input name is not expected.
        ValueError: Input data type is not expected.
        ValueError: Number of outputs not expected.
        ValueError: Output name is not expected.
        ValueError: Output data type is not expected.
    r­   rY   rW   rå   Úencoder_attention_maskÚpast_key_self_Úpast_value_self_Úpast_key_cross_Úpast_value_cross_rí   rî   rï   rð   rñ   rò   Úpresent_key_self_Úpresent_value_self_rô   rõ   N)r   ro   r#   r
   rú   rƒ   r¸   rö   r‚   r÷   rø   r´   rù   r3   rû   rü   r”   )r³   r   rý   Ú
float_typerþ   rÿ   r   rë   r  r  r  r  r  r  r%   r%   r&   Úverify_t5_decoder_subgraphû  sH   
"ÿ"
ÿür  c              	   C   s¼  |t jjk}d| jd jv }g d¢}|r|dd… }t| jƒt|ƒkr2tdt|ƒ› dt| jƒ› �ƒ‚t|ƒD ]9\}}| j| j|krStd|› d	|› d| j| j› �ƒ‚t	j
}| j| jjj}||krotd|› d
|› d|› �ƒ‚q6|r¤t| jƒd dks}J ‚t| jƒd }	|	dksŠJ ‚g }
t|	ƒD ]}|
 d|› �¡ |
 d|› �¡ q�nSt d¡ t| jƒd d dks¶J ‚t| jƒd d }	|	dksÅJ ‚ddg}
t|	ƒD ]}|
 d|› �¡ |
 d|› �¡ qÍt|	ƒD ]}|
 d|› �¡ |
 d|› �¡ qät| jƒt|
ƒk�rtdt|
ƒ› dt| jƒ› �ƒ‚t|
ƒD ]B\}}| j| j|k�r2td|› d	|› d| j| j› �ƒ‚|�r8t	jnt	j}| j| jjj}||k�rUtd|› d
|› d|› �ƒ‚�qt d¡ dS )r  Úcrossr   )Úencoder_input_idsr	  Údecoder_input_idsNr­   rí   rî   rï   rð   rñ   rW   Úpresent_key_cross_Úpresent_value_cross_zZThis format is deprecated. Please export T5 encoder in new format with only cross outputs.rY   rò   Úencoder_hidden_statesr  r  rô   rõ   zMT5 encoder graph verified: name and data type of inputs and outputs are good.)r   ro   r#   r”   r´   rƒ   r¸   r÷   rø   r
   rù   r3   rû   rü   rö   r‚   r…   r§   rú   r†   )r³   r   rý   Ú
new_formatr   rë   r  r  r  rÿ   r  r  r  r%   r%   r&   Ú'verify_t5_encoder_decoder_init_subgraphG  s\   "ÿþ
"
ÿr  Úshared_é   Úgraph1Úgraph2Úshared_prefixÚmin_elementsÚsignature_cache1Úsignature_cache2c                 C   s  i }i }g }g }	g }
| j D ]N}|jrt|jƒ|ksq|j D ]=}|jr)t|jƒ|ks*qt ||||¡rZ||j ||j< | |¡ |j|vrX||j }|||j< |	 |¡ |
 |¡  nqqt d|
› �¡ | j	D ]}t
t|jƒƒD ]}|j| |
v rƒtd|j| › �ƒ‚qpqg|j	D ]}t
t|jƒƒD ]}|j| |
v r¤td|j| › �ƒ‚q‘qˆ|	D ]}|j  |¡ q¨|jD ]}|j|v rÁ||j |_q´|j	D ]4}t
t|jƒƒD ]*}|j| |v rø||j|  }t d|j› d|› d|j| › d|› �¡ ||j|< qÎqÅ|D ]}| j  |¡ qü| jD ]}|j|v �r||j |_�q| j	D ]7}t
t|jƒƒD ],}|j| |v �rO||j|  }t d|j› d|› d|j| › d|› �¡ ||j|< �q$�q|	D ]	}||j |_�qU|	D ] }tj |¡j}tj |j|j|¡}| j |¡ |j |¡ �qa|	S )	a…  Remove initializers with same value from two graphs.

    Args:
        graph1 (GraphProto): the first graph to process
        graph2 (GraphProto): the second graph to process
        shared_prefix (str): add prefix to the shared initializers among two graphs
        min_elements (int, optional): minimal number of elements for initializers to be considered. Defaults to 1024.
        signature_cache1 (dict): Optional dictionary to store data signatures of tensors in graph1 in order to speed up comparison
        signature_cache2 (dict): Optional dictionary to store data signatures of tensors in graph2 in order to speed up comparison
    zshared initializers:zname is found in graph 1: zname is found in graph 2: zgraph 2 rename node z input z from z to zgraph 1 rename node )Úinitializerr¼   Úsumr   Úhas_same_valuer´   r‚   r…   r–   Únoderö   rƒ   r¸   rÞ   ÚremoveÚ
value_infor£   Únumpy_helperrÄ   ÚshapeÚhelperÚmake_tensor_value_inforº   )r  r  r  r  r   r!  Úmapping_initializers_1Úmapping_initializers_2Úshared_initializers_1Úshared_initializers_2Úshared_initializers_namesÚinitializer1Úinitializer2Úshared_namer%  Újr"  r'  Únew_namer)  r%   r%   r&   Úremove_shared_initializers£  sŽ   







÷€
ÿÿ
ÿÿ

€
*
€ü
€
*
€ür6  Úencoder_modelrÈ   c                 C   s`   t | ƒ}t |ƒ}| d¡ | d¡ i i }}| |¡ | |¡ t|jj|jjd||d�}|S )NÚe_Úd_Ús_)r  r   r!  )r   Úadd_prefix_to_namesÚremove_duplicated_initializerr6  r¨   r³   )r7  rÈ   ÚencoderÚdecoderr   r!  Úinitializersr%   r%   r&   Úget_shared_initializers  s   




ûr@  c                 C   s‚   g }| j D ]}|jrt|jƒ|ksq| |¡ q|D ]}| j  |¡ q|D ]}tj |¡j}tj	 
|j|j|¡}| j |¡ q%|S )a^  Remove initializers of a graph, when they have number of elements larger than a threshold.

    Args:
        graph (GraphProto): the graph.
        min_elements (int, optional): minimal number of elements for initializers to be considered. Defaults to 1024.

    Returns:
        List[TensorProto]: initializers that are removed from the graph.
    )r"  r¼   r#  r‚   r&  r£   r(  rÄ   r)  r*  r+  r´   rº   r'  )r³   r  Úmoved_initializersÚtensorr"  r)  r'  r%   r%   r&   Úmove_initializers  s   
rC  c                 C   sô   | j dkrtd| j› d�ƒ‚| j dkr| j}n^| j dkr | j}nU| j dkr)| j}nL| j dkr2| j}nC| j dkr;| j}n:| j d	krD| j}n1| j d
krM| j	}n(| j dkrV| j
}n| j dkr_| j}n| j dkrh| j}ntd| j› d| j › d�ƒ‚| j|fS )zÄ
    Convert attribute to kwarg format for use with onnx.helper.make_node.
        :parameter attribute: attribute in AttributeProto format.
        :return: attribute in {key: value} format.
    r   z
attribute z does not have type specified.rW   r­   rä   rY   é   é   é   r®   é	   é
   z has unsupported type r;   )r3   r÷   r´   Úfrë   ÚsÚtÚgÚfloatsÚintsÚstringsÚtensorsÚgraphs)Ú	attributer#   r%   r%   r&   Ú_attribute_to_pair>  s0   











rS  c                 C   sD   i }| j D ]}t|ƒ\}}| ||i¡ q| jr | d| ji¡ |S )NÚdomain)rR  rS  ÚupdaterT  )r%  ÚkwargsÚattrÚkeyr#   r%   r%   r&   Ú	kwargs_ofc  s   
rY  c                 C   s   t dd„ | jjjjD ƒƒS )Nc                 S   s   g | ]}|j r
|j n|j‘qS r%   )Ú	dim_paramÚ	dim_value)r×   Údr%   r%   r&   rÚ   n  s    zshape_of.<locals>.<listcomp>)Útupler3   rû   r)  Údim)Úvir%   r%   r&   Úshape_ofm  s   r`  Úsubgc              
   C   sê  d}d}g }t | jƒD ],\}}||kr1t|ƒ}tjj|j|jjj	|d |d |d d|d gd�}| 
|g¡ q| 
tjjdtjjdgd	�g¡ |  d
¡ | j 
|¡ g }t | jƒD ],\}}||kr€t|ƒ}tjj|j|jjj	|d |d |d d|d gd�}| 
|g¡ qZ|  d¡ | j 
|¡ g }| jD ]P}	|	}
|	jdkrát|	ƒ}| ddi¡ g }| 
|	j¡ t|ƒdk rÅ| 
dg¡ t|ƒdk s¹t|ƒdk rÑ| 
dg¡ tjjd||	jfd|	ji|¤Ž}
| 
|
g¡ q—|  d¡ | j 
|¡ | S )Nrä   rW   r   r­   Úmax_seq_lenrY   ©rü   r)  Úpast_sequence_length©r)  r¸   r”   Ú	AttentionrQ   rE  r   rF  r´   r%  )rø   r¸   r`  r£   r*  r+  r´   r3   rû   rü   r�   r
   rù   Ú
ClearFieldr”   r%  r¶   rY  rU  rƒ   Ú	make_node)ra  Úinput_past_0Úoutput_past_0Ú
new_inputsrë   r_  r)  Únew_outputsÚ	new_nodesr%  Únew_noderV  Únisr%   r%   r&   Ú1update_decoder_subgraph_past_present_share_bufferq  sZ   ý 
ý


ÿ 
rp  Úis_beam_searchÚswitch_attentionc                 C   s¢  |r@g }t | jƒD ]
\}}| |g¡ q	| tjjdtjjdgd�g¡ | tjjdtjjg d¢d�g¡ |  d¡ | j |¡ |rÏg d¢}g }| j	D ]x}|j
dkr½t|ƒ}	|	 ¡ D ]}
|
d	krd  d
S |
|vrx|
dkrut d|
› d�¡ |	|
= qZg }| |j¡ |r­t|ƒdk r•| dg¡ t|ƒdk s‰t|ƒdk r¡| dg¡ t|ƒdk r­| dg¡ tjjd||jfd|ji|	¤Ž}| |g¡ qK|  d¡ | j	 |¡ dS )aS  Update the Attention nodes to DecoderMaskedSelfAttention.

    Args:
        subg (GraphProto): GraphProto of the decoder subgraph
        is_beam_search (bool): Boolean specifying if the sampling algo is BeamSearch
        switch_attention (bool): Boolean specifying if `Attention` is to be switched with `DecoderMaskedSelfAttention`
    Ú
beam_widthrW   re  Úcache_indirection©Ú
batch_sizers  rb  r¸   ©rQ   Ú	num_headsÚscaleÚmask_filter_valuerT  rf  Úqkv_hidden_sizesFÚunidirectionalzRemoving attribute: zB from Attention node while switching to DecoderMaskedSelfAttentionrF  r   r®   rG  ÚDecoderMaskedSelfAttentionr´   r%  T)rø   r¸   r�   r£   r*  r+  r
   rù   rg  r%  r¶   rY  Úcopyr…   r§   rƒ   rh  r”   r´   )ra  rq  rr  rk  Ú_ir_  Ú'decoder_masked_attention_supported_attrrm  r%  rV  Úkro  r%   r%   r&   Ú4update_decoder_subgraph_use_decoder_masked_attention¢  sl   
 ýÿÿ
	


ÿ€ÿýüû
r‚  c                 C   sæ  t ƒ }g }dd„ t| jƒD ƒ}i }i }| jD ]'}|jD ]}|r0||vr)|g||< q||  |¡ q|jD ]}|r<|||< q4q| jD �],}|jdk�rn|jd rT|jd sUqA|jd |jd }	}
d}d|
v r�| jD ]}|jdkr|jd |
kr|jd j} nqin| j	D ]}|j
|
kr�|} nq„|du r•qAtj |¡}|jdk�rn| ¡ d	v �rn|jd |v �rn||	 }|jd
kr¾|jd s¿qA|jd |v rú|jd  d¡sÖ|jd  d¡rú| ¡ dkrú| |jd ¡ | |¡ t||jd  ƒdkrú| |¡ qA|jd |v�rqA||jd  }|jdk�r|jd �sqA||jd  }|jdk�r*|jd �s+qA|jd |v �rn|jd  d¡�sE|jd  d¡�rn| ¡ dk�rn| |jd ¡ | |||g¡ t||jd  ƒdk�rn| |¡ qAqA||fS )az  Correct graph which originally use dim of past_seq_len from input_ids's shape which is fixed to max_seq_len after
       shared past/present buffer

    Args:
        subg (GraphProto): GraphProto of the decoder subgraph
    return:
        tensor_names_to_rename : set of tensor names which is equal to past_sequence_length
        nodes_to_remove : list of node to remove
    c                 S   s   i | ]\}}|j |“qS r%   ©r´   )r×   ÚindexÚinpr%   r%   r&   Ú
<dictcomp>  ó    z+find_past_seq_len_usage.<locals>.<dictcomp>ÚGatherrW   r   NÚ	Constant_ÚConstant>   rW   r­   ÚShaper
  r  r­   ÚReshaper¬   )Úsetrø   r¸   r%  r‚   r”   r¶   rR  rK  r"  r´   r£   r(  rÄ   ÚsizeÚitemÚ
startswithÚaddrƒ   r�   )ra  Útensor_names_to_renameÚnodes_to_removeÚgraph_input_namesÚinput_name_to_nodesrµ   r%  Ú
input_nameÚoutput_nameÚshape_tensor_nameÚshape_index_nameÚini_gather_indicesÚ
const_noderB  Úgather_indices_arrÚ
shape_nodeÚreshape_nodeÚtranspose_noder%   r%   r&   Úfind_past_seq_len_usageø  s–   


€
€þ
€

þþý

þý
€r   r¨   Úpast_seq_len_namec                 C   s–   d}t tdd„ | jjjƒƒ}|D ]"}t|jƒdk r&|j d¡ t|jƒdk s|j |¡ |j |¡ q| jjj tj	j
|tjg d¢d�¡ |  ¡  | S )Nrt  c                 S   ó
   | j dkS ©NÚMultiHeadAttention©r¶   ©r%  r%   r%   r&   Ú<lambda>d  ó   
 z.add_cache_indirection_to_mha.<locals>.<lambda>r®   r   ©rv  rs  Úmax_sequence_lengthre  )ÚlistÚfilterr¨   r³   r%  rƒ   r¸   r‚   r£   r*  r+  r
   rù   Útopological_sort)r¨   r¡  Úcache_indirection_nameÚ	mha_nodesr%  r%   r%   r&   Úadd_cache_indirection_to_mhaa  s   ÿ
ÿÿr°  r°   Úskip_node_idxsc              
   C   sR  d}g }t tdd„ | jjjƒƒ}t|ƒD ]†\}}||v rqd}|jD ]}	|	jdkr.|	j} nq"|}
|
dkrJ| jjj	D ]}|j|j
d krI|j}
 nq:d}| jjj
D ]}|j|j
d krf|jjjjd	 j} nqQt|jƒdk r{|j d
¡ t|jƒdk sn|› d|d	 › �}|j |¡ | tjj||
d|d|gd�¡ q| jjj |¡ |  ¡  | S )NÚoutput_cross_qkc                 S   r¢  r£  r¥  r¦  r%   r%   r&   r§  z  r¨  z&add_output_qk_to_mha.<locals>.<lambda>r   rx  rä   Útarget_sequence_lengthrW   r­   r   Ú_rv  Úsequence_lengthre  )r«  r¬  r¨   r³   r%  rø   rR  r´   rë   r"  r¸   rº   r3   rû   r)  r^  r[  rƒ   r”   r‚   r£   r*  r+  r�   r­  )r¨   r°   r±  Úoutput_qk_basenameÚ
output_qksr¯  Úidxr%  rx  ÚattÚoutput_qk_dtyperë   r³  Úoutput_qk_namer%   r%   r&   Úadd_output_qk_to_mhav  sP   

þþþÿ
ýÿr¼  c                    sš  d}d}d}t tdd„ | jjjƒƒd }|  |g d¢g d¢¡}|  |d	d
gddg¡}|d ur1|}n|d ur8|}nt d¡ d S |d }|jdkrÂ|d }	|  |	ddgddg¡‰ ˆ d u rbt d¡ d S |  |	g d¢g d¢¡}
|
d u rxt d¡ d S |
d }ˆ |
dd … kr‹t d¡ d S t t‡ fdd„| jjjƒƒd }| jjj 	|¡ | jjj 	ˆ d ¡ | jjj 	ˆ d ¡ ||	j
d< ||j
d< nª|  |g d¢g d¢¡}|d u rØt d¡ d S |d }|  |g d¢g d¢¡‰ˆd u ròt d¡ d S ˆd }|dd … ˆdd … k�r
t d¡ d S t t‡fdd„| jjjƒƒd }| jjj 	|¡ t t‡fd d„| jjjƒƒd }| jjj 	|¡ | jjj 	ˆd ¡ | jjj 	ˆd ¡ | jjj 	ˆd! ¡ | jjj 	ˆd" ¡ ||j
d< ||j
d< | jjj
 tjj|tjdgd#�¡ tjjd$|g|g|  d$¡d%�}tjj|tjg d#�}tjjd&|g|g|  d&¡tjd'�}tjj|tjg d#�}| jjj ||g¡ | jjj ||g¡ |  ¡  | |fS )(Nrd  Úpast_seq_len_int32Úpast_seq_len_int64c                 S   r¢  )NÚLayerNormalizationr¥  ©Únr%   r%   r&   r§  Ð  r¨  z*fix_past_sequence_length.<locals>.<lambda>r   )ÚAddrˆ  ÚTileÚExpandÚ	UnsqueezeÚRange)r   rW   rW   r   r   r   rÂ  ÚSlicerW   zBCannot identify base path for fixing past_sequence_length subgraphr]   rÆ  rˆ  r‹  zDCannot identify gather path for fixing past_sequence_length subgraph)rÂ  rˆ  r‹  ©rW   r   r   zACannot identify add path for fixing past_sequence_length subgraphz]Gather path and add path do not share the same nodes for calculating the past_sequence_lengthc                    s   | j d ˆ d jd kS ©Nr   rW   ©r”   r¸   rÀ  )Úgather_pathr%   r&   r§    r‡  )rÅ  rÂ  rˆ  r‹  rŒ  r¬   )r­   r   r   r   r   r   zGCannot identify input_ids path for fixing past_sequence_length subgraph)rÅ  rˆ  r‹  rŒ  r¬   )rW   r   r   r   r   zFCannot identify past_key path for fixing past_sequence_length subgraphr­   ziThe input_ids path and past_key path do not share the same nodes for calculating the past_sequence_lengthc                    s   | j d ˆ d jd kS rÉ  rÊ  rÀ  ©Úpast_key_pathr%   r&   r§  '  r‡  c                    s   | j d ˆ d jd kS )Nr   éþÿÿÿrW   rÊ  rÀ  rÌ  r%   r&   r§  )  r‡  rä   rY   re  ÚSqueeze©ÚinputsÚoutputsr´   ÚCast©rÑ  rÒ  r´   Úto)r«  r¬  r¨   r³   r%  Úmatch_parent_pathr…   r†   r¶   r&  r¸   r‚   r£   r*  r+  r
   rù   rh  Úcreate_node_nameÚINT64r�   r'  r­  )r¨   r¡  r½  r¾  r%  Úbase_path_hfÚbase_path_oaiÚ	base_pathÚ	base_nodeÚ
range_nodeÚadd_pathÚadd_nodeÚconstant_in_gatherÚinput_ids_pathÚunsqueeze_nodeÚconstant_in_reshapeÚsqueeze_nodeÚsqueeze_outputÚ	cast_nodeÚcast_outputr%   )rË  rÍ  r&   Úfix_past_sequence_lengthª  sÔ   "ýý

ý
ý

 
ý
ý
ÿ ÿ


ÿüûrè  c                 C   sò  d}d}| j jj tjj|tjdgd�tjj|tjg d¢d�g¡ t	t
dd„ | j jjƒƒ}t|ƒD ]Á\}}d}|jD ]}|jd	krF|j} nq:d
|d › �}	tjj|	tjd|ddgd�}
|d dkrj| j jj |
¡ tjjd|jd |jd |jd ddt|jƒdkrˆ|jd ndt|jƒdkr•|jd nd||||jd g|jd t|jƒdkr®|jd ndt|jƒdkr»|jd nd|d dkrÄ|	ndg|j dd¡d||d dd�}|d dkrá|j d¡ | j jj |¡ | j jj |g¡ q1|  ¡  | S )Nrs  rt  rW   re  r©  c                 S   r¢  r£  r¥  r¦  r%   r%   r&   r§  b  r¨  z(replace_mha_with_dmmha.<locals>.<lambda>r   rx  Úoutput_cross_qk_r­   rv  zencode_sequence_length / 2ÚDecoderMaskedMultiHeadAttentionr   rY   rE  rF  rä   r¤  úcom.microsoft)rÑ  rÒ  r´   rT  rx  Ú	output_qkrQ   )r¨   r³   r¸   r�   r£   r*  r+  r
   rù   r«  r¬  r%  rø   rR  r´   rë   rú   r”   r‚   rh  rƒ   Úreplacer&  r­  )r¨   r¡  rs  rt  r¯  r¸  r%  rx  r¹  Úqk_output_nameÚ	qk_outputÚ
dmmha_noder%   r%   r&   Úreplace_mha_with_dmmhaS  sl   
ÿþÿ


þÿõüçrñ  rW   r]   Ú	attn_maskÚkv_num_headsÚ
world_sizeÚwindow_sizec           1      C   sP  |   tjjdtjdgdgd�¡ tjjd|dg|d g|  d¡d�}tjjd|d dgdg|  d¡d�}tjjd	dgd
g|  d	¡tjd�}tjjd|g|d g|  d¡d�}tjjd|d dgdg|  d¡dd�}	tjjd	dgdg|  d	¡tjd�}
| j	j
j |||||	|
g¡ ttdd„ | j	j
jƒƒ}t|ƒD �]Œ\}}|  |g d¢g d¢¡}|  |ddgddg¡}d\}}}|d urÂ|\}}}n|d urÊ|\}}|  |g d¢g d¢¡}|  |ddgddg¡}d\}}}|d urï|\}}}n|d ur÷|\}}|  |ddgddg¡}|  |dgdg¡}d\}}|d u�r|\}}n	|d u�r"|d }d}|d u�r>|d u�r>|jD ]}|jdk�r<|j}�q1d}|jD ]}|jdk�rN|j}�qC|jd |jd k�od|jd |jd k}|d u�or|d u�or|d u} |d u �o€|d u �o€|d u }!d \}"}#}$|�r–| �s�|!�r–t |  |jd ¡¡}%t |  |jd ¡¡}&t |  |jd ¡¡}'|%jd! }(tj|%|&|'fdd"� |(d#|( ¡})tjj|)d$|› �d%�})|   |)¡ tjjd|jd |)jg|)j› d&�g|  d¡d�}*| j	j
j |*g¡ | j	j
j |¡ | j	j
j |¡ | j	j
j |¡ |*jd }"| �r•t |  |jd ¡¡}+t |  |jd ¡¡},t |  |jd ¡¡}-|+jd! }(tj|+|,|-fdd"� d#|( ¡}.tjj|.d'|› �d%�}.|   |.¡ tjjd|*jd |.jg|.j› d&�gd(�}/| j	j
j |/g¡ | j	j
j |¡ | j	j
j |¡ | j	j
j |¡ |/jd }"n|jd }"|jd }#|jd }$tjjd)|"|#|$|jd* |jd+ |jd |
jd |d u�rÆ|jd nd,|d u�rÑ|jd# nd,g	|j|j  d-d)¡d.|| |dk�rè|| n|| |t!|d u�oõ|d uƒ|d/�
}0| j	j
j |¡ | j	j
j |0g¡ |d u�r| j	j
j |¡ |d u�r%| j	j
j |¡ q˜| S )0NÚonerW   ©r´   rº   r¼   ÚvalsÚ	ReduceSumÚ	_row_sumsrÐ  ÚSubÚseqlens_k_int64rÓ  Ú	seqlens_krÔ  r‹  Ú_shaperˆ  Útotal_seq_len_int64r   )rÑ  rÒ  r´   r²   Útotal_seq_lenc                 S   r¢  r£  r¥  r¦  r%   r%   r&   r§     r¨  z&replace_mha_with_gqa.<locals>.<lambda>)ÚRotaryEmbeddingrÂ  r«   )r   r   r   r  r«   )NNNrÈ  rÂ  r­   ©NNÚinterleavedrx  )r   r   r   r]   r±   rä   ÚQKV_Weight_rƒ  Ú_outputÚ	QKV_Bias_)rÑ  rÒ  ÚGroupQueryAttentionrE  rF  r   r¤  rë  )	rÑ  rÒ  r´   rT  rx  ró  Úlocal_window_sizeÚ	do_rotaryÚrotary_interleaved)"Úadd_initializerr£   r*  Úmake_tensorr
   rØ  rh  r×  rù   r¨   r³   r%  r�   r«  r¬  rø   rÖ  rR  r´   rë   r¸   r   rÄ   r·   r)  rÀ   ÚstackÚreshaper(  Ú
from_arrayr&  r”   rí  rp   )1r¨   rò  ró  rô  rõ  Úreduce_sum_nodeÚsub_nodeÚseqlen_k_cast_noder�  Úgather_nodeÚtotal_seqlen_cast_noder¯  r¸  r%  Úq_path_1Úq_path_2Úq_rotaryÚq_addÚq_matmulÚk_path_1Úk_path_2Úk_rotaryÚk_addÚk_matmulÚv_path_1Úv_path_2Úv_addÚv_matmulr  r¹  rx  Úroot_input_is_sameÚall_paths_have_biasÚall_paths_have_no_biasÚq_input_to_attentionÚk_input_to_attentionÚv_input_to_attentionÚqwÚkwÚvwr^  Ú
qkv_weightÚpacked_matmul_nodeÚqbÚkbÚvbÚqkv_biasÚpacked_add_nodeÚgqa_noder%   r%   r&   Úreplace_mha_with_gqaš  sH  üÿü
üûü
ûû
úÿ$





€
€*

 
ü


ý
€


÷ì

€r4  c              	      s®  d}dd„ ˆj D ƒ}|dk r$||  d¡s$|d7 }|dk r$||  d¡rd}tˆjƒ| d }d| | ‰ ‡ ‡fdd„t|ƒD ƒ}td	|› �ƒ tˆj ˆ  ƒ}td
|› �ƒ |d }|d }|d }	d}
ˆjD ]_}|jdkrÆ|j d |v rÆtd|j	› d|j› �ƒ |
d7 }
||j d  }d|› �}dgdt|jƒ  }| 
|¡ |j |¡ |j tj dd¡g¡ tj |tj||d|	g¡}ˆj |g¡ qg|
|krÕtd|› d|
› �ƒ‚d S )NrW   c                 S   ó   g | ]}|j ‘qS r%   rƒ  ©r×   Úgir%   r%   r&   rÚ   ‹  ó    zBupdate_decoder_subgraph_output_cross_attention.<locals>.<listcomp>rä   Úpastr­   c                    s"   i | ]}ˆj |d  ˆ   j|“qS )r­   )r¸   r´   )r×   Úlayer©Úinput_cross_past_0ra  r%   r&   r†  ’  s   " zBupdate_decoder_subgraph_output_cross_attention.<locals>.<dictcomp>z    -- past_key_cross_inputs = zpast_key_cross_0_shape is r   rê  z'    -- add cross QK output from: node: z with output: ré  r   rì  z#Did not add cross QK for all layersz vs )r¸   r�  rƒ   r”   rö   Úprintr`  r%  r¶   r´   r‚   r�   rR  r£   r*  Úmake_attributer+  r
   rú   r÷   )ra  Úinput_self_past_0r”  Úoutput_self_present_0Ú
num_layersÚpast_key_cross_inputsÚinput_past_key_cross_0_shapeÚbatch_size_dimÚnum_heads_dimÚcross_seq_len_dimÚnum_layer_output_qkr%  r:  Úcross_attention_out_nameÚappended_namesÚcross_attentionr%   r;  r&   Ú.update_decoder_subgraph_output_cross_attentionˆ  sH   ÿ



ý€ÿrK  c              	   C   s$  d}dd„ | j D ƒ}|dk r$||  d¡s$|d7 }|dk r$||  d¡rd}tt| j ƒ| d ƒ}d| | }g }g }| jD ]}|jdkrK| |g¡ q>t|ƒ|k rTd	S d }	| jD ]}|jd
krd|}	 nqYg d¢}
d}t| ƒ\}}t|ƒdkr¹|D ]}td|› d|› d�ƒ qy|D ]}td|j› d|j	› �ƒ q‰t
jjddgdgdd�}t
jjddg|gdtjd�}| ||g¡ | jD ]ÿ}t|jƒdkrï|	d urï|jd |	j d krït
jjddgdgdtjd�}|jd |j d< | |g¡ |jdk�r›t|ƒ}| ¡ D ]
}||
v�r||= qý|j d |j d |j d g}| t|j ƒdk�r%|j d ndg¡ | t|j ƒdk�r8|j d ndg¡ | t|j ƒdk�rK|j d ndg¡ | t|j ƒd k�r^|j d  ndg¡ | dg¡ | d!g¡ | d"g¡ | t|j ƒdk�rƒ|j d ndg¡ d|d#< t
jjd$||jfd%|j	i|¤Ž}||v�r»t|j ƒD ]\}}||v �r³||j |< �q¥| |g¡ q¼|  d&¡ | j |¡ d'd„ | j D ƒ}g }t| j ƒD ]0\}}||k�rÿ||k �rÿt|ƒ}t
jj|j	|jjj|d |d d(|d gd)�}| |g¡ �qÖd|v�r| t
jjdt
jjdgd*�g¡ d!|v�r1| t
jjd!t
jjdgd*�g¡ d"|v�rG| t
jjd"t
jjg d+¢d*�g¡ |  d,¡ | j  |¡ g }t| jƒD ]+\}}||k�r}t|ƒ}t
jj|j	|jjj|d |d d(|d gd)�}| |g¡ �qY|  d-¡ | j |¡ d.S )/NrW   c                 S   r5  r%   rƒ  r6  r%   r%   r&   rÚ   ´  r8  zSupdate_decoder_subgraph_share_buffer_and_use_decoder_masked_mha.<locals>.<listcomp>rä   r9  rY   r­   r¤  FÚRelativePositionBiasrw  Ú#past_sequence_length_squeezed_int64r   zFound tensor name `z` to be renamed to `ú`zFound node to remove: type = z	, name = rÏ  rd  Úpast_sequence_length_squeezedÚ!node_past_sequence_length_squeezerƒ  rÓ  Ú&node_past_sequence_length_squeeze_cast)r´   rÕ  Úpast_sequence_length_int64Úpast_sequence_length_castr   rD  rE  rF  rs  rt  rQ   rê  r´   r%  c                 S   r5  r%   rƒ  )r×   r…  r%   r%   r&   rÚ   !  r8  rb  rc  re  ru  r¸   r”   T)r¸   r�  rp   rƒ   r%  r¶   r�   r   r=  r´   r£   r*  rh  r
   rØ  r”   rY  r~  rø   rg  r`  r+  r3   rû   rü   rù   )ra  r?  r”  Úoutput_self_past_0rA  r<  rm  Ú	old_nodesr%  Úrel_pos_bias_noder€  Útarget_squeezed_past_seq_namer’  r“  Úname_to_renameÚnrrä  ræ  rV  r�  ro  r„  r´   Úorig_input_namesrk  rë   r_  r)  rl  r%   r%   r&   Ú?update_decoder_subgraph_share_buffer_and_use_decoder_masked_mha±  s
  ÿ

€

þüû
*û
€ý&&&&&ýüû


€€
ý
ÿ
 
ýÿÿ
	
ý
r[  Úmodel_protoc                 C   sÜ  t | ƒ}| ¡ }g }g }| ¡ D ]É}|jdkrÙd|jd v r&d|jd v r&q||jd  }||jd  }||jd  }| |jd ¡}	| |jd ¡}
| |jd ¡}|	rY|
rY|s\ dS t |	¡}t |
¡}t |¡}tj	|||gdd�}|j
d	d
d�}tjj|d |	jdkrŠtjntj|jd |jd g| ¡  ¡ d�}| jj |g¡ tjjd	|jd |d g|d g|d�}|jd |jd< d|jd< d|jd< | |g¡ | |||g¡ q| |¡ | |¡ | ¡  | ¡  dS )Nrê  Úpast_key_crossrW   Úpast_value_crossr­   r   Fr±   r«   Ú
MatMul_QKV)Úname_prefixÚ_weightr÷  Ú_outrÐ  r   T)r   rµ   Únodesr¶   r¸   r·   r   rÄ   rÀ   rÃ   r×  r£   r*  r  rº   r
   rú   ro   r)  ÚflattenÚtolistr³   r"  r�   rh  r”   Ú	add_nodesÚremove_nodesÚupdate_graphr­  )r\  Ú
onnx_modelrµ   Únodes_to_addr“  r%  r  r  r"  Úq_weightÚk_weightÚv_weightr)  r*  r+  r,  Úmatmul_node_nameÚweightrÉ   r%   r%   r&   Úpack_qkv_for_decoder_masked_mhaP  sZ   




üü

€

rp  Údecoder_onnx_pathc                 C   sŒ   t j| dd�}tt|jjƒƒD ],}|jj| jdks#|jj| jdkr;|jj| jjj	j
d }| d¡r8| ¡  d|_qtj|| |d� dS )aQ  Update the input shapes for the inputs "input_ids" and "position_ids" and make the sequence length dim value 1 for each of them.
       The decoder model will be over-written.

    Args:
        decoder_onnx_path (str): Path of GPT-2 decoder onnx model
        use_external_data_format(bool): output tensors to external data or not.
    Trœ   rå   ræ   rW   rZ  r    )r£   r¤   rö   rƒ   r³   r¸   r´   r3   rû   r)  r^  ÚHasFieldÚClearr[  r   r¦   )rq  rG   rÆ   rë   Úshape_dim_protor%   r%   r&   Ú*update_input_shapes_for_gpt2_decoder_model‹  s   	
€ýru  Úinit_decoder_onnx_pathc                 C   sÀ  t j| dd�}|jjd j}t|ƒ}| ¡ }||v sJ ‚|| }|jdkr'dS | |g d¢g d¢¡}|du rA| |g d	¢g d
¢¡}|du r_| |g d¢g d¢¡}|du r_| |g d¢g d¢¡}|du redS |d }	|	jdk}
|
s·d}| |	g d¢|dddg¡}|du r’d}| |	g d¢|dddg¡}|du r¤d}| |	g d¢|ddg¡}|du r¶d}| |	g d¢|ddg¡}nBd}| |	g d¢|ddg¡}|du r×d}| |	g d¢|ddg¡}|du rèd}| |	ddg|dg¡}|du rùd}| |	ddg|dg¡}|du rÿdS |dk�rdnd}|
�s| 	|	d|¡}n| 	|	d|¡}|du �r!dS |d }|d }t j
jdtjdgdgd�}t j
jdtjdgdgd�}t j
jdtjdgdgd�}t j
jdtjdgdgd�}| |¡ | |¡ | |¡ | |¡ d|jd  }t j
jd|jd ddddg|g| dd¡d �}|
�s—|jd n|jd! }d|jd  }t j
jd|ddddg|g| dd"¡d �}| |¡ | |¡ | ||jd |¡ | |	||¡ | ¡  tj|||d#� dS )$a„  Generates the initial decoder GPT2 subgraph and saves it for downstream use.
       The initial decoder model will be saved to init_decoder_onnx_path.

    Args:
        decoder_onnx_path (str): Path of GPT-2 decoder onnx model
        init_decoder_onnx_path (str): Path of GPT-2 init decoder onnx model
        use_external_data_format(bool): output tensors to external data or not.
    Trœ   r   r«   F)rÓ  r¿  rÂ  rÂ  rÓ  r«   rÓ  ÚFastGelurÓ  r«   rÓ  r¿  rÂ  )r   r   r   rW   r   r   r   r   r   r   r   r   r   N)
rÓ  ÚSkipLayerNormalizationrÓ  r«   rÓ  rw  rÓ  r«   rÓ  rx  )
r   r   rW   r   r   r   r   r   r   r   )r¿  rÂ  rÂ  r«   rw  r«   r¿  rÂ  )r   r   rW   r   r   r   r   r   )rx  r«   rw  r«   rx  )r   rW   r   r   r   r]   rx  )rÂ  rÓ  r«   rf  rW   )rÂ  r«   rf  )rÓ  r«   rf  rf  rÂ  rÎ  ÚSliceLastTokenStartsr÷  ÚSliceLastTokenEndsÚSliceLastTokenAxesÚSliceLastTokenStepsÚedge_modified_rÇ  ÚGatherLastToken_0_rÐ  rä   ÚGatherLastToken_1_r    )r£   r¤   r³   r”   r´   r   rµ   r¶   rÖ  r¹   r*  r  r
   rù   r  rh  r×  rß  Úreplace_node_inputr­  r¦   )rq  rv  rG   Úinit_decoder_model_protorÇ   Úgpt2_init_decoder_modelrµ   Úlogits_matmul_nodeÚ"logits_matmul_to_residual_add_pathÚresidual_add_nodeÚis_skiplayernorm_pathÚ&residual_add_to_attention_parent_indexÚresidual_add_to_attention_pathÚ residual_add_to_add_parent_indexÚadd_before_residual_addÚ	attentionÚmatmul_after_attentionÚslice_startsÚ
slice_endsÚ
slice_axesÚslice_stepsÚslice_0_output_nameÚslice_node_0Úadd_before_residual_add_outputÚslice_1_output_nameÚslice_node_1r%   r%   r&   Úgenerate_gpt2_init_decoder«  sX  
ïò
ô÷

ý
ýýý€ýýýýÿý
üüüü



û
öÿû
ö

ýr–  c           	      C   sÒ   t dƒ}t |jƒ}t |jƒ}t |jƒ}| jjD ]%}|jjjj	D ]}| 
d¡r;|j||||fv r;t|jƒ}| ¡  ||_qq| jjD ]%}|jjjj	D ]}| 
d¡re|j||||fv ret|jƒ}| ¡  ||_qIqAdS )zoMake dim_proto numeric.

    Args:
        model: T5 encoder and decoder model.
        config: T5 config.
    rW   rZ  N)ri   rx  Úd_modelÚd_kvr³   r”   r3   rû   r)  r^  rr  rZ  rp   rs  r[  r¸   )	r¨   Úconfigrµ  rx  Úhidden_sizeÚ	head_sizerB  Ú	dim_protor[  r%   r%   r&   Úmake_dim_proto_numeric_t5Ö	  s>   


ü
€÷ü
€÷ÿr�  Úgeneration_typec           -      C   s´  | j dk}|tjk}|tjk}|tjk}| j}t d|› �¡ t| j	ƒdkrM| j	d dkrM|rJ| j
tjjkrJg d¢| _	t d| j	› �¡ t d¡ ng | _	|sQ|re|sWtd	ƒ‚| jr^td
ƒ‚| jretdƒ‚|rp|rp| jsptdƒ‚| jry|sytdƒ‚| jrƒ| jsƒtdƒ‚|rÅ| jr™tj | j¡r™t d| j› �¡ nN| js²| j› d| j
› d�}tt| jƒj|ƒ ¡ | _t d| j› d| j› d�¡ t| ƒ n"| jrÙ| jrÙt d| j› d| j› �¡ nt d| j› d�¡ t| ƒ d}| j �s| j
tjjk�r|�r|sÿ|sÿ|�rt d| j› d�¡ t!| j| j"ƒ}|�st #d¡ d}	d}
| j$�sg|�rg|�s,|�s,|�rgt d| j› d�¡ d | j
› d�}tt| jƒj|ƒ ¡ }
t%| j|
| j"ƒ}	|	�sXt #d!¡ |	�rgt&| j| j"ƒ�sgtd"ƒ‚|�sq| j'�sq|	�r”t d#| j› d�¡ t(| j| j"ƒ |	�r”t d#|
› d�¡ t(|
| j"ƒ |�r¡t)j*| j| j+d$�}n| j d%k�r±t,j*| j| j+d$�}n	t-j*| j| j+d$�}| j.�rÆt d&|› �¡ |j/}|�rÏ|j/n|j0}|j1}| j1d'k�rÞ| j1}| j/d'k�rç| j/}| j0d'k�rð| j0}t2j3| jd(d)�}| j › d*�|j4_5d}| j dk�r)t6|j4| j
ƒ |	�r(t2j3|
d(d)�}| j › d+�|j4_5t6|j4| j
ƒ nt7|j4| j
ƒ d}|�r:g d,¢}n
|�s@|�rDg d-¢}| j8�rN| 9d.¡ n| 9d/¡ | j:�r]| 9d0¡ n| 9d/¡ | j;�rl| 9d1¡ n| 9d/¡ |�r�| j<�r‚| j=�r‚| 9d2¡ n| 9d/¡ | j>�r�| 9d3¡ d4g}| j�rœ| 9d5¡ | j�r­| j�s¨J d6ƒ‚| 9d7¡ d}|�rÁt2j?j@d8||d9| j › �d:�}n#|�rÓt2j?j@d;||d<| j › �d:�}n|�rät2j?j@d=||d>| j › �d:�}d?|_Ad}|�rt2j? Bd@|¡t2j? BdA|¡t2j? BdB| jC¡t2j? BdC| jD�r	dnd¡t2j? BdD| j dk�rdnd¡g}nw|�rCt2j? Bd@|¡t2j? BdA|¡t2j? BdD| j dk�r7dnd¡t2j? BdB| jC¡g}nP|�r“t2j? Bd@|¡t2j? BdA|¡t2j? BdD| j dk�r^dnd¡t2j? BdB| jC¡t2j? BdE| jE¡t2j? BdF| jF¡t2j? BdG| jG¡t2j? BdH| jH¡t2j? BdI| j<¡t2j? BdJ| jI¡g
}|�r¡| Jt2j? BdK|¡g¡ |jK J|¡ g }| j dLv �rf| j'�rÄt dM| j› d�¡ t(| j| j"ƒ t2j3| jd(d)�}t|j4jLƒdNk�r×dOndP}| j › dQ|› �|j4_5tM|j4| j
ƒ tN||ƒ tN||ƒ |�r%| j�sÿtdRƒ‚t dS¡ tO|j4ƒ�rt dT¡ nt dU¡ tP|ƒ�r t dV¡ nt dW¡ | jQ�s@tR||ƒ}t t|ƒ› dXdYdZ„ |D ƒ› d[�¡ |jSdk�sJJ d\ƒ‚|jK Jt2j? BdO|j4¡t2j? Bd]|j4¡t2j? Bd^|jS¡g¡ n€|	�r®| jQ�s„tR||ƒ}t t|ƒ› dXd_dZ„ |D ƒ› d`�¡ |�r‘t da¡ tT|j4ƒ | j�r¡tU|j4|dƒ�s¡tdbƒ‚|jK 9t2j? Bdc|j4¡¡ ntV|j4ƒ}t t|ƒ› dd�¡ |�rÊt de¡ tT|j4ƒ | j�rÚtU|j4|d(ƒ�sÚtdfƒ‚|jK 9t2j? Bd]|j4¡¡ t2j? WdgtXjYdhdig¡}t2j? WdjtXjYdg¡}t2j? WdktXjYdg¡}t2j? WdltXjYdg¡}t2j? WdmtXjYdg¡}t2j? WdntXjZdg¡}t2j? WdotXjZdg¡}d} |�r<|||||||g} n|�sB|�rH||||g} | j8�r[t2j? Wd.tXjY|g¡}!|  9|!¡ | j:�rot2j? Wd0tXjYdh|g¡}"|  9|"¡ | j;�rƒt2j? Wd1tXjYdhdig¡}#|  9|#¡ | j<�r›| j=�r›t2j? Wd2tXjYdh|g¡}$|  9|$¡ |�r±| j>�r±t2j? Wd3tXjYdg¡}%|  9|%¡ d}&|�rÂt2j? Wd4tXjYg dp¢¡}&n|�sÈ|�rÓt2j? Wd4tXjYdhdjg¡}&|&g}'| j�rêt2j? Wd5tXjZdhdmg¡}(|' 9|(¡ | j�r t2j? Wd7tXjZdqdhdl|g¡})|' 9|)¡ t2j? [|g|�s| j › dr�n| j › ds�| |'|¡}*t2j?j\|*dt|j]du�}+| j"�rHddvl^m_}, |, `t2ja¡|, `dw¡k �r=t #dx¡ tbjc|+| jd(d(dy� nt2 c|+| j¡ t dz| j› �¡ dS ){zˆConvert model according to command line arguments.

    Args:
        args (argparse.Namespace): arguments parsed from command line
    r5   z**** past_present_share_buffer=rW   r   rD   )rÂ  r¿  rx  rw  z**** Setting op_block_list to zI**** use --op_block_list if you want to override the block operator list.z<Currently only gpt2 with greedy search/sampling is supportedzLoutput_sequences_scores currently is not supported in greedy search/samplingzHoutput_token_scores currently is not supported in greedy search/samplingzi`use_decoder_masked_attention` MUST be turned on to use `past_present_share_buffer` in case of BeamSearchzS`past_present_share_buffer` MUST be turned on to use `use_decoder_masked_attention`z?`use_decoder_masked_attention` option is only supported on GPUsz)skip convert_to_onnx since path existed: Ú_past_z.onnxzConvert GPT model z	 to onnx z ...z,skip convert_to_onnx since paths specified: z and zConvert model z to onnx ...Fz=Pad logits MatMul weights for optimal MatMul perf in fp16 on z. The file will be overwritten.z]Tried and failed to pad logits MatMul weights. Performance may be sub-optimal for this MatMulNz*Creating an initial run GPT2 decoder from z. Úgpt2_init_past_zuTried and failed to generate the init decoder GPT2 model. Performance may be sub-optimal for the initial decoding runzGCould not update the input shapes for the non-initial decoder subgraph.z Run symbolic shape inference on ©r€   r6   zConfig=r]   Trœ   z decoderz init decoder©rå   Ú
max_lengthÚ
min_lengthÚ	num_beamsÚnum_return_sequencesÚlength_penaltyÚrepetition_penalty©rå   r£  r¤  r¨  rP   r   rS   rç   rU   rV   Ú	sequencesÚsequences_scoresz8--output_token_scores requires --output_sequences_scoresÚscoresÚ
BeamSearchÚBeamSearch_rÐ  ÚGreedySearchÚGreedySearch_ÚSamplingÚ	Sampling_rë  Úeos_token_idÚpad_token_idÚno_repeat_ngram_sizerO   r‘   ÚtemperatureÚtop_pÚfilter_valueÚmin_tokens_to_keepÚcustomÚpresence_penaltyÚ
vocab_size©r6   r7   zSymbolic shape inference on r­   r=  zencoder and decoder initú zMpast_present_share_buffer is only supported with use_decoder_masked_attentionzl*****update t5 decoder subgraph to share past/present buffer and use decoder_masked_multihead_attention*****z4*****update t5 decoder subgraph successfully!!!*****zF*****DecoderMaskedMultiHeadAttention is not applied to T5 decoder*****z9*****pack qkv for decoder masked mha successfully!!!*****z3*****pack qkv for decoder masked mha failed!!!*****z shared initializers (c                 S   r5  r%   rƒ  rê   r%   r%   r&   rÚ   ?  r8  z,convert_generation_model.<locals>.<listcomp>z>) in encoder and decoder subgraphs are moved to the main graphz%decoder_start_token_id should be >= 0r>  Údecoder_start_token_idc                 S   r5  r%   rƒ  rê   r%   r%   r&   rÚ   [  r8  zC) in decoder and init decoder subgraphs are moved to the main graphzY*****update init decoder subgraph to make past and present share buffer******************zLCould not update the init decoder subgraph to use DecoderMaskedSelfAttentionÚinit_decoderz: initializers from the decoder are moved to the main graphzT*****update decoder subgraph to make past and present share buffer******************zGCould not update the decoder subgraph to use DecoderMaskedSelfAttentionrå   rv  rµ  r£  r¤  r¥  r¦  r§  r¨  )rv  r¦  r£  zmax_length - sequence_lengthz beam searchz greedy searchzonnxruntime.transformers)Úproducer_nameÚopset_imports)Úversionz1.12.0z0Require onnx >= 1.12 to save large (>2GB) model!)r¡   Úall_tensors_to_one_filezmodel save to )dr‘   r   r+   r,   r-   rQ   r…   r†   rƒ   r„   r   r   ro   r#   ÚNotImplementedErrorrM   rN   rR   r÷   r`   r~   rk   rl   Úexistsr}   r   r”   r•   Úas_posixrŠ   r—   r™   rI   rÒ   rG   r§   rJ   r–  ru  rH   rª   r   Úfrom_pretrainedr€   r   r   r?   r³  r´  r¼  r£   r¤   r³   r´   r  r  rP   r‚   rS   rT   rº  rU   rV   r*  rh  rT  r>  rµ  rO   r¶  r·  r¸  r¹  r»  r�   rR  r¸   r  r�  r[  rp  rK   r@  r¿  rp  r‚  rC  r+  r
   rù   rú   Ú
make_graphÚ
make_modelÚopset_importÚ	packagingrÃ  ÚparseÚ__version__r   r¦   )-ry   rž  Úis_gpt2Úis_beamsearchÚis_greedysearchÚis_samplingrQ   Úonnx_filenameÚlogits_matmul_weight_paddedÚgpt2_init_decoder_generatedÚgpt2_init_decoder_onnx_pathÚgpt2_init_decoder_onnx_filenamer™  r³  r´  r¼  rÈ   r‚  rÑ  rÒ  r%  Úattr_to_extendr?  r7  Úsuffixrå   r£  r¤  r¥  r¦  r§  r¨  Úgraph_inputsrP   rS   rç   rU   rV   rª  Úgraph_outputsr«  r¬  Ú	new_graphÚ	new_modelrÃ  r%   r%   r&   Úconvert_generation_modelû	  sÆ  
	



ÿ

ÿÿÿþþþÿÿÿþýýýýÿÿ€
	







ü
ü
üûüö

ÿ



ÿýÿ
ÿ


ÿ



ÿù	ü
ÿ
ÿ
ÿ

ýýý

ý
û	ý
ürÞ  rå   rç   r³  r´  Úbad_words_idsc                 C   s   | j rtj ¡ stdƒ‚| jtjjkr| 	¡  t 
| j rdnd¡}| |¡ t d¡ | |¡}| |¡}g }t| jƒD ]3}	t ¡ }
|j||| j| j| j| j| j||| j| j| j|r\|ndd| jpc| jd�}	| t ¡ |
 ¡ q<|jd }dd	lm} |||ƒS )
aœ  Test PyTorch performance of text generation.

    Args:
        args (argparse.Namespace): arguments parsed from command line
        model (Union[GPT2LMHeadModel, T5ForConditionalGeneration]): PyTorch model
        input_ids (torch.Tensor): input_ids
        attention_mask (torch.Tensor): Attention mask
        eos_token_id (int): EOS token ID
        pad_token_id (int): Padding token ID
        bad_words_ids (List[List[int]]): Words shall not be generated.

    Raises:
        RuntimeError: PyTorch with CUDA is not available for --use_gpu

    Returns:
        Dict[str, Any]: A dictionary with string with metric name, and value can be integer or string.
    z=Please install PyTorch with Cuda for testing gpu performance.zcuda:0ÚcpuFNT©rå   rç   r£  r¤  r¥  rO   rµ  r³  r´  r¦  r§  r¨  rß  Úreturn_dict_in_generateÚoutput_scoresr   ©Úget_latency_result)r`   ÚtorchÚcudaÚis_availablerÞ   r   r   ro   r#   ÚhalfÚdevicerÕ  Úset_grad_enabledrö   Ú
total_runsÚtimeÚgenerater£  r¤  r¥  rO   rµ  r¦  r§  r¨  rM   rN   r‚   r)  Úbenchmark_helperrå  )ry   r¨   rå   rç   r³  r´  rß  rê  Útorch_latencyr´  Ústartrv  rå  r%   r%   r&   Útest_torch_performance÷  sB   





ñ

rò  c                 C   sp   t j| jt jd�}t| jd ƒD ]%}d}t| jd ƒD ]}| | | |kr0|dkr0d|| |< q|d7 }qq|S )Nr¯   r   rW   )rÀ   Úonesr)  Úint32rö   )rå   r´  rç   rë   Úabs_posr4  r%   r%   r&   Úcreate_attention_mask9  s   
ürö  FÚ	sentencesÚ	is_greedyc           +      C   s  | j dksJ ‚tj| j| jd�}d|_|j|_tj| j| j|j	d�}|du r*g d¢}||ddd	�}|d
 }|d }d}|j
|dd�}	dd„ |	D ƒ}	| jrSt d|	¡ ng }	|j}
|
j	}|
j	}|
j}g }d}| jsÔtdƒ tdƒ |j||| j| j| j| j| j||| j| j| j|	rŠ|	ndd| jp‘| jd�}td
|ƒ tdƒ td|jƒ | jr¬td|jƒ | jrµtd|jƒ t |jƒD ]\}}|j!|dd�}| "|¡ t|› d|› �ƒ qºtdƒ tdƒ |�r| #¡  $¡  %t&j'¡t&j(| jgt&j'd�t&j(| jgt&j'd�t&j(| jgt&j)d�dœ}nB| #¡  $¡  %t&j'¡t&j(| jgt&j'd�t&j(| jgt&j'd�t&j(| jgt&j'd�t&j(| jgt&j'd�t&j(| jgt&j)d�t&j(| jgt&j)d�dœ}| j�rgt&j*|t&j'd�}| j�rc|	D ]}d||< �q[||d< | j+�rrt,||ƒ|d< |j-d }| j.�rŽt /d ¡ t&j*||ft&j'd�}||d!< | j0�rÌt1| j2ƒj3 4¡ }t d"|¡ dd#l5m6} t /d$|› d%�¡ |g}t |ƒD ]\}}t7j8 9|d&t:|ƒ ¡}|||ƒ �q¶t d'|¡ | j;�rØdS t d(¡ t<| j2| j=| j>ƒ}t d)¡ | ?d|¡}g }t@| jAƒD ]}tB B¡ }| ?d|¡}| "tB B¡ | ¡ �qødd*lCmD}  |j-d }| ||ƒ}!td+ƒ |d }"td|"ƒ | j�r7td|d, ƒ | j�rBtd|d- ƒ |�rm|"j-\}}#g }$t@|ƒD ]}|j!|"| dd�}|$ "|¡ td.|› d/|› �ƒ �qPn5|"j-\}}%}#g }$t@|ƒD ](}t@|%ƒD ] }&|j!|"| |& dd�}|$ "|¡ td.|› d0|&› d|› �ƒ �q�qy|�ré|j E|| jd1¡}'tF G|"¡}(tdƒ td2ƒ t|'ƒ t|ƒ tdƒ td3ƒ t|(ƒ t|$ƒ tdƒ ||$k})td4|)�râd5nd6ƒ |)|!d7< | jH�rütI| ||||||	ƒ}*td8|*ƒ td9|!ƒ |!S ):a9  Test GPT-2 model

    Args:
        args (argparse.Namespace): arguments parsed from command line
        sentences (Optional[List[str]], optional): input text. Defaults to None.

    Returns:
        Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
    r5   r¡  Úleft)r€   r´  N)zThe product is releasedzI enjoy walking in the parkzTest best way to investÚptT©Úreturn_tensorsrÏ   rå   rç   úwalk in park)Úadd_prefix_spacec                 S   ó   g | ]}|g‘qS r%   r%   ©r×   Úword_idr%   r%   r&   rÚ   m  r8  z"test_gpt_model.<locals>.<listcomp>rß  ú2--------------------------------------------------úCTest PyTorch model and beam search with huggingface transformers...rá  ú!huggingface transformers outputs:rª  r«  r¬  ©Úskip_special_tokensú: ú'Testing beam search with onnxruntime...r¯   r©  r¢  r   rP   zYUse prefix vocab mask with all ones in ORT, but no corresponding setting for Torch model.rS   Útest_data_dir©Úoutput_test_datazSaving test_data to z/test_data_set_* ...Útest_data_set_ú
ORT inputszCreating ort session......zRun ort session......rä  úORT outputs:rW   r­   úbatch z sequence: ú
 sequence r]   úTorch Sequences:úORT Sequences:zTorch and ORT result isÚsameÚ	differentÚparityúTorch LatencyÚORT)Jr‘   r   rÈ  r}   r€   Úpadding_sideÚ	eos_tokenÚ	pad_tokenr   r³  ÚencoderP   r…   r–   r™  r¼  ra   r=  rî  r£  r¤  r¥  rO   rµ  r¦  r§  r¨  rM   rN   rª  r«  r¬  rø   Údecoder‚   rà  ÚnumpyÚastyperÀ   rô  ÚarrayÚfloat32ró  rT   rö  r)  rS   r†   rd   r   r”   r•   rÇ  Úbert_test_datar  rk   rl   rj   ri   rb   rã   r`   r^   Úrunrö   rì  rí  rï  rå  r  ræ  Ú
LongTensorrc   rò  )+ry   r÷  rø  Ú	tokenizerr¨   rÑ  rå   rç   Ú	bad_wordsrß  r™  r³  r´  r¼  Útorch_decoded_sequencesÚbeam_outputsrë   ÚsequenceÚdecoded_sequencerP   Úbad_word_idrv  rS   r	  r  Ú
all_inputsÚdirrâ   ÚresultÚlatencyr´  rñ  rå  r”   rª  r£  Úort_decoded_sequencesÚnum_sequencesr4  Útorch_sequencesÚort_sequencesÚis_sameÚtorch_latency_outputr%   r%   r&   Útest_gpt_modelE  sB  ý

ñ

üù









ý
ý
ù
	
r5  c           )      C   s(  | j dv sJ ‚| jrt d¡ dS tj| j| jd�}d|_| j dkr,t	j| j| jd�}n	t
j| j| jd�}|du r=ddg}||d	d
d�}|d }|d }d}| |¡dd… }dd„ |D ƒ}| jrht d|¡ ng }|j}	|	j}
|	j}|	j}t d|
› d|› d|› �¡ g }| jsõtdƒ tdƒ |j||| j| j| j| j| j|
|| j| j| j|r«|ndd
| jp²| jd�}td|ƒ tdƒ td|jƒ | jrÍtd|jƒ | jrÖtd|j ƒ t!|jƒD ]\}}|j"|d
d�}| #|¡ t|› d|› �ƒ qÛtdƒ tdƒ t$j%|t$j&d �}| j�r|D ]}d!||< �q| '¡  (¡  )t$j&¡t$j*| jgt$j&d �t$j*| jgt$j&d �t$j*| jgt$j&d �t$j*| jgt$j&d �t$j*| jgt$j+d �t$j*| jgt$j+d �d"œ}| j�r]||d#< | j,�rht-||ƒ|d< | j.�r�t/| j0ƒj1 2¡ }t d$|¡ d!d%l3m4} |g}t!|ƒD ]\}}t5j6 7|d&t8|ƒ ¡}|||ƒ �q‡t d'|¡ t9| j0| j:| j;ƒ}g }t<| j=ƒD ]}t> >¡ }| ?d|¡}| #t> >¡ | ¡ �q³|j@d! }d!d(lAmB} |||ƒ}td)ƒ |d! } td| ƒ | j�ròtd|d* ƒ | j�rýtd|d+ ƒ | j@\}}!}"g }#t<|ƒD ](}t<|!ƒD ] }$|j"| | |$ d
d�}|# #|¡ td,|› d-|$› d|› �ƒ �q�q	| j�sz|j C|| jd¡}%tD E| ¡}&tdƒ td.ƒ t|%ƒ t|ƒ tdƒ td/ƒ t|&ƒ t|#ƒ tdƒ ||#k}'td0|'�rsd1nd2ƒ |'|d3< | jF�r�tG| ||||
||ƒ}(td4|(ƒ td5|ƒ |S )6a=  Test T5 or MT5 model

    Args:
        args (argparse.Namespace): arguments parsed from command line
        sentences (Optional[List[str]], optional): input text. Defaults to None.

    Returns:
        Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
    r½  zLSkipping parity test as prefix vocab mask is not implemented by Hugging FaceNr¡  rù  r6   z4translate English to French: The product is releasedz¬summarize: research continues to show that pets bring real health benefits to their owners. Having a dog around can lead to lower levels of stress for both adults and kids.rú  Trû  rå   rç   rý  r]   c                 S   rÿ  r%   r%   r   r%   r%   r&   rÚ   I  r8  z!test_t5_model.<locals>.<listcomp>rß  zeos_token_id:z, pad_token_id:z, vocab_size:r  r  rá  r  rª  r«  r¬  r  r  r  r¯   r   r¢  rP   r	  r
  r  r  rä  r  rW   r­   r  r  r  r  zTorch and ORT result is r  r  r  r  r  )Hr‘   rS   r…   r–   r   rÈ  r}   r€   r  r   r   r  rP   r™  r³  r´  r¼  ra   r=  rî  r£  r¤  r¥  rO   rµ  r¦  r§  r¨  rM   rN   rª  r«  r¬  rø   r  r‚   rÀ   ró  rô  rà  r  r  r  r   rT   rö  rd   r   r”   r•   rÇ  r!  r  rk   rl   rj   ri   rã   r`   r^   rö   rì  rí  r"  r)  rï  rå  r  ræ  r#  rc   rò  ))ry   r÷  r$  r¨   rÑ  rå   rç   r%  rß  r™  r³  r´  r¼  r&  r'  rë   r(  r)  rP   r*  r	  r  r+  r,  râ   r.  r´  rñ  r-  rv  rå  r”   rª  r0  r£  r/  r4  r1  r2  r3  r4  r%   r%   r&   Útest_t5_model  s   


þþþ

ñ

ù




ý
ù
	
r6  c                 C   sr  t | ƒ}t|jƒ |jdv rB|jr tj |j¡s td|j› �ƒ‚|j	r2tj |j	¡s2td|j	› �ƒ‚|jr8|j	r>|j	rB|jsBtdƒ‚|j
dkoK|jdk}|jdkr}|r}|jdkrv|jdk rvt|tjƒ t d	¡ |jd
kss|jss|jrudS nt|tjƒ nt|ƒ t d¡ |jdv r’t||d�}nt|||d�}|r·|jr®t d|j› d|j› d�¡ |S t d|j› �¡ |S )a/  Main entry function

    Args:
        argv (Optional[List[str]], optional): _description_. Defaults to None.
        sentences (Optional[List[str]], optional): input text. Defaults to None.

    Raises:
        ValueError: Path does not exist: --encoder_decoder_init_onnx
        ValueError: Path does not exist: --decoder_onnx
        ValueError: --decoder_onnx and --encoder_decoder_init_onnx are not used together for T5

    Returns:
        Union[Dict[str, Any], None]: A dictionary with string with metric name, and value can be integer or string.
    r½  z1Path does not exist: --encoder_decoder_init_onnx z$Path does not exist: --decoder_onnx zB--decoder_onnx shall use together with --encoder_decoder_init_onnxrW   r5   r\   rZ   z�The test for gpt2_sampling onnx model is limited to non-custom model with small top_p(e.g <=0.01) value. The result should be the same as gpt2 greedy search.g{®Gáz„?Nzstart testing model...)r÷  )r÷  rø  zOutput files: r1   z.datazOutput file: )rz   r   r?   r‘   r—   rk   rl   rÆ  r÷   r~   r¥  r¦  r·  rÞ  r   r-   r…   r†   rº  rV   r,   r6  r5  rG   r”   )r.   r÷  ry   rø  r-  r%   r%   r&   r   Ù  sF   

ÿÿÿÿ

þr   Ú__main__r"   )T)r  r  NN)r  )r   rW   r]   )NFr  )aÚ__doc__re   Úloggingr½   rk   rí  Úenumr   Úpathlibr   Útypingr   r  rÀ   r£   ræ  rï  r   r   Úfusion_utilsr   r   r	   r
   ri  r   Útransformersr   r   r   r   r   r   r   r   Úonnxruntimer   r   r   r   Ú4onnxruntime.transformers.models.gpt2.convert_to_onnxr   r‡   Ú0onnxruntime.transformers.models.gpt2.gpt2_helperr   Ú2onnxruntime.transformers.models.t5.convert_to_onnxr   r“   Ú,onnxruntime.transformers.models.t5.t5_helperr   r   Ú	getLoggerr…   r   r«  ri   Ú	Namespacerz   rŠ   r™   Úboolrª   rÒ   rã   r  r  r  rp   Údictr6  r@  rC  rS  rY  r`  rp  r‚  r   r°  r¼  rè  rñ  r4  rK  r[  rp  ru  r–  r�  r+   rÞ  ÚTensorrò  rö  r5  r6  r(   r%   r%   r%   r&   Ú<module>   s:  %(
	   -N!8L_úÿþýüû
újþÿþ
ý%
1ÿÿÿ
þVi 4 *Jûÿþýü
û o)  ;#ýÿþý
ü  -'þÿ
þ   ÿþýüûú
ù

øBýÿ
þ
ý X $>
;
ÿ