o
    °ƒðjG€  ã                   @   sZ  d Z ddlZddlZddlZddlZddlmZ ddlZddlZddlm	Z	m
Z
mZmZmZmZmZmZmZmZmZ ddlmZ ddlmZmZ ddlmZmZmZmZ ddlmZ dd	l m!Z! e "d
¡Z#ej$dd�Z$dej%vrwe&e$ƒej%d< ddl'Z'ddl(m)Z)m*Z*m+Z+ dd„ Z,dd„ Z-de.de.fdd„Z/dd„ Z0dd„ Z1dd„ Z2e3dkr«e2ƒ  dS dS )a   Benchmarking the inference of pretrained transformer models.
PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py.
One difference is that random input_ids is generated in this benchmark.

For onnxruntime, this script will convert a pretrained model to ONNX, and optimize it when -o parameter is used.

Example commands:
    Export all models to ONNX, optimize and validate them:
        python benchmark.py -b 0 -o -v -i 1 2 3
    Run OnnxRuntime on GPU for all models:
        python benchmark.py -g
    Run OnnxRuntime on GPU for all models with fp32 optimization:
        python benchmark.py -g -o
    Run OnnxRuntime on GPU with fp16 optimization:
        python benchmark.py -g -o -p "fp16"
    Run TorchScript on GPU for all models:
        python benchmark.py -e torchscript -g
    Run TorchScript on GPU for all models with fp16:
        python benchmark.py -e torchscript -g -p "fp16"
    Run ONNXRuntime and TorchScript on CPU for all models with quantization:
        python benchmark.py -e torchscript onnxruntime -p "int8" -o
    Run OnnxRuntime with the ROCM provider and graph optimization script:
        python benchmark.py -g -m bert-base-cased --provider rocm --optimizer_info by_script --disable_embed_layer_norm
    Run OnnxRuntime with bfloat16 fastmath mode kernels on aarch64 platforms with bfloat16 support:
        python benchmark.py --enable_arm64_bfloat16_fastmath_mlas_gemm

It is recommended to use run_benchmark.sh to launch benchmark.
é    N)Údatetime)ÚConfigModifierÚOptimizerInfoÚ	PrecisionÚcreate_onnxruntime_sessionÚget_latency_resultÚinference_ortÚinference_ort_with_io_bindingÚoutput_detailsÚoutput_fusion_statisticsÚoutput_summaryÚsetup_logger)ÚFusionOptions)ÚMODEL_CLASSESÚMODELS)Úcreate_onnxruntime_inputÚexport_onnx_model_from_ptÚexport_onnx_model_from_tfÚload_pretrained_model)Úversion)ÚQuantizeHelperÚ F)ÚlogicalÚOMP_NUM_THREADS)Ú
AutoConfigÚAutoTokenizerÚLxmertConfigc           4      C   sè  dd l }g }| r'd| ¡ vr'd| ¡ vr'd| ¡ vr'd| ¡ vr't d¡ |S d}|dkr?tj}d}d	| ¡ vr?t d
¡ |S |tjkrMt d|› d�¡ |D �]¡}t| d }|
D �]•}|t|ƒkrf �n‹|d |… }t| d |_	t
 |¡}d|v r´t ¡ �, t|t| d t| d t| d |||||| |||||||ƒ\}} }!}"W d   ƒ n1 s¯w   Y  d|v rÜt|t| d t| d t| d |||||| |||||||ƒ\}} }!}"| sßqZt|| |d|||d�}#|#d u rïqZdd„ |# ¡ D ƒ}$g }%| rþdnd}&tj||d�}'t t|ƒt|ƒt|!|'jƒg¡}(t t|ƒ|'jg¡})|D ]Ì}*|*dk�r,�q#|D ]¿}+|"d u�r<|+|"k�r<�q.d|v �rDtjntj},t|!|*|+||'|,ƒ}-d|j||&||| ||||*|+| ¡ tt ¡ ƒdœ}.|'j	dv �r�t  d|› d|*d|'j!|'j!g› �¡ nt  d|› d|*|+g› �¡ |�r›t"|#|-|.|	|*|ƒ}/nG|# #|$|-¡}0|(g}1t$t|0ƒƒD ]}2|2dk�rÁt| d dk�rÁ|1 %|)¡ �qª|1 %|(¡ �qªd|v �rÐtj&ntj'}3t(|#|-|.|	|$|0|%|1|*|&|3|ƒ}/t  |/¡ | %|/¡ �q.�q#qZqO|S )Nr   ÚCUDAExecutionProviderÚMIGraphXExecutionProviderÚROCMExecutionProviderÚDmlExecutionProviderzŽPlease install onnxruntime-gpu or onnxruntime-directml package instead of onnxruntime, and use a machine with GPU for testing gpu performance.Útensorrté   ÚTensorrtExecutionProviderzhPlease install onnxruntime-gpu-tensorrt package, and use a machine with GPU for testing gpu performance.zOptimizerInfo is set to zA, graph optimizations specified in FusionOptions are not applied.é   Úpté   é   ÚtfT)Úenable_all_optimizationÚnum_threadsÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16c                 S   s   g | ]}|j ‘qS © )Úname)Ú.0Únode_argr-   r-   ú¤/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/onnxruntime/transformers/benchmark.pyÚ
<listcomp>Ý   s    z#run_onnxruntime.<locals>.<listcomp>ÚcudaÚcpu©Ú	cache_dirÚonnxruntime©Úenginer   Ú	providersÚdeviceÚ	optimizerÚ	precisionÚ
io_bindingÚ
model_nameÚinputsÚthreadsÚ
batch_sizeÚsequence_lengthÚcustom_layer_numr   ©ÚvitÚswinzRun onnxruntime on ú with input shape Úgpt))r7   Úget_available_providersÚloggerÚerrorr   ÚNOOPTÚwarningr   ÚlenÚ
model_typer   ÚparseÚtorchÚno_gradr   r   r   Úget_outputsr   Úfrom_pretrainedÚnumpyÚprodÚmaxÚhidden_sizeÚint64Úint32r   Ú__version__Úget_layer_numÚstrr   ÚnowÚinfoÚ
image_sizer   ÚrunÚrangeÚappendÚlonglongÚintcr	   )4Úuse_gpuÚproviderÚmodel_namesÚmodel_classÚconfig_modifierr=   r*   Úbatch_sizesÚsequence_lengthsÚrepeat_timesÚinput_countsÚoptimizer_infoÚvalidate_onnxr6   Úonnx_dirr+   Ú	overwriteÚdisable_ort_io_bindingÚuse_raw_attention_maskÚmodel_fusion_statisticsÚmodel_sourceÚ(enable_arm64_bfloat16_fastmath_mlas_gemmÚargsr7   ÚresultsÚwarm_up_repeatr?   Úall_input_namesÚ
num_inputsÚinput_namesÚfusion_optionsÚonnx_model_fileÚis_valid_onnx_modelÚ
vocab_sizeÚmax_sequence_lengthÚort_sessionÚort_output_namesÚoutput_buffersr;   ÚconfigÚmax_last_state_sizeÚmax_pooler_sizerB   rC   Úinput_value_typeÚ
ort_inputsÚresult_templateÚresultÚort_outputsÚoutput_buffer_max_sizesÚiÚ	data_typer-   r-   r1   Úrun_onnxruntimeY   sh  ÿÿÿ

ÿ






ïûû


ïûù	
ýÿ
ú	
òÿú
ô
·ý€Nr’   c                    s°  g }| rt j ¡ st d¡ |S t  d¡ |D �]=}tj||	|d�}| |¡ t	||||d�}|j
dv r:|d g}n
tj||d�}|j}t d|› �¡ t d	| ¡ › �¡ |tjkr_| ¡  t  | red
nd¡}| |¡ |tjkrwt |¡}|D ]Û}|dkr€qy|D ]Ñ}|j
dv r²t d|› d|d|j|jg› �¡ t j|d|j|jf|tjkr«t jnt j|d�‰n&|d ur»||kr»q‚t d|› d||g› �¡ t jd|jd ||ft j|d�‰z^|	rât j  |ˆ¡n|
rét  !|¡n|‰ ˆ ˆƒ t"j#‡ ‡fdd„|dd�}|	�rdn|
�rdndt j$d| �rdndd|d|d|||| %¡ t&t' (¡ ƒdœ}| )t*||ƒ¡ t |¡ | +|¡ W q‚ t,�yS } zt -|¡ t j .¡  W Y d }~q‚d }~ww qyq|S )NzYPlease install PyTorch with Cuda, and use a machine with GPU for testing gpu performance.F)Útorchscriptr6   )r‡   r6   Úcustom_model_classrE   r   r5   zModel zNumber of parameters zcuda:0r4   zRun PyTorch on rH   r$   )ÚsizeÚdtyper;   r&   )ÚlowÚhighr•   r–   r;   c                      s   ˆ ˆƒS ©Nr-   r-   ©Ú	inferenceÚ	input_idsr-   r1   Ú<lambda>Ž  s    zrun_pytorch.<locals>.<lambda>©ÚrepeatÚnumberr“   Útorch2rR   ÚNAr3   r   r8   )/rR   r3   Úis_availablerK   rL   Úset_grad_enabledr   rU   Úmodifyr   rP   r   Úmodel_max_lengthÚdebugÚnum_parametersr   ÚFLOAT16Úhalfr;   ÚtoÚINT8r   Úquantize_torch_modelr`   ra   ÚrandnÚfloat16Úfloat32Úrandintr‚   ÚlongÚjitÚtraceÚcompileÚtimeitrŸ   r\   r]   r^   r   r_   Úupdater   rd   ÚRuntimeErrorÚ	exceptionÚempty_cache)rg   ri   rj   rk   r=   r*   rl   rm   rn   r“   r¡   r6   r+   rz   r?   r‡   ÚmodelÚ	tokenizerÚmax_input_sizer;   rB   rC   Úruntimesr�   Úer-   rš   r1   Úrun_pytorch:  s    



ü





ÿýû"ÿ
ò

€þÏü9rÀ   Údo_eager_modeÚuse_xlac                    s*   ddl m‰ dd l‰‡ ‡‡‡fdd„}|S )Nr   )Úwrapsc                    sT   ˆˆ ƒ‡ fdd„ƒ}ˆˆ ƒˆj ˆd�‡ fdd„ƒƒ}ˆdu r(ˆdu s&J dƒ‚|S |S )	Nc                     ó   ˆ | i |¤ŽS r™   r-   ©ry   Úkwargs©Úfuncr-   r1   Úrun_in_eager_mode°  ó   zFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_eager_mode)Úexperimental_compilec                     rÄ   r™   r-   rÅ   rÇ   r-   r1   Úrun_in_graph_mode´  s   zFrun_with_tf_optimizations.<locals>.run_func.<locals>.run_in_graph_modeTFzcCannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.)Úfunction)rÈ   rÉ   rÌ   ©rÁ   r(   rÂ   rÃ   rÇ   r1   Úrun_func¯  s   

ÿz+run_with_tf_optimizations.<locals>.run_func)Ú	functoolsrÃ   Ú
tensorflow)rÁ   rÂ   rÏ   r-   rÎ   r1   Úrun_with_tf_optimizationsª  s   rÒ   c                    sö  g }dd l ‰ˆjj |¡ | sˆj g d¡ | r$ˆj ¡ s$t d¡ |S | r`ˆj 	d¡}zˆj |d d¡ ˆjj
 |d d¡ ˆjjdd� W n ty_ } zt |¡ W Y d }~nd }~ww |tjksj|tjkrntdƒ‚|D �]}tj||	d�‰ | ˆ ¡ t|ˆ |	|dd	�‰tj||	d�}|j}|D ]ã}|dkr›q”|D ]Ù}|d ur¨||kr¨q�t d
|› d||g› �¡ dd l}| ¡ ‰‡ ‡fdd„t|| ƒD ƒ}ˆj|||fˆjd�‰zzt ddd�‡‡fdd„ƒ}t ddd�‡‡fdd„ƒ}t ddd�‡ ‡‡‡fdd„ƒ}|‰ˆ j!�r	|‰nt"ˆ t#ƒ�r|‰ˆƒ  t$j%‡fdd„|dd�}dˆj&d| �r)dndd|d|d|||| '¡ t(t) *¡ ƒd œ}| +t,||ƒ¡ t |¡ | -|¡ W q� t�yv } zt |¡ dd!l.m/} | 0¡ }| 1¡  W Y d }~q�d }~ww q”qp|S )"Nr   ÚGPUzVPlease install Tensorflow-gpu, and use a machine with GPU for testing gpu performance.Tz/gpu:0)r;   z+Mixed precision is currently not supported.r5   )r‡   r6   r”   Úis_tf_modelzRun Tensorflow on rH   c                    s   g | ]}ˆ  d ˆ jd ¡‘qS )r   r&   )r±   r‚   )r/   r�   )r‡   Úrngr-   r1   r2     s    z"run_tensorflow.<locals>.<listcomp>)Úshaper–   F)rÁ   rÂ   c                      s   ˆˆ dd�S )NF)Útrainingr-   r-   ©rœ   r»   r-   r1   Úencoder_forward  s   z'run_tensorflow.<locals>.encoder_forwardc                      s   ˆˆ ˆ dd�S )NF)Údecoder_input_idsr×   r-   r-   rØ   r-   r1   Úencoder_decoder_forward  rÊ   z/run_tensorflow.<locals>.encoder_decoder_forwardc                     s8   ˆj  ddˆ jg¡} ˆj  ddˆ jg¡}ˆˆ| |dd�S )Nr&   F)Úvisual_featsÚ
visual_posr×   )ÚrandomÚnormalÚvisual_feat_dimÚvisual_pos_dim)ÚfeatsÚpos)r‡   rœ   r»   r(   r-   r1   Úlxmert_forward  s   üz&run_tensorflow.<locals>.lxmert_forwardc                      s   ˆ ƒ S r™   r-   r-   )r›   r-   r1   r�   '  s    z run_tensorflow.<locals>.<lambda>r&   rž   rÑ   r¢   r3   r4   r   r8   )r3   )2rÑ   r‡   Ú	threadingÚ set_intra_op_parallelism_threadsÚset_visible_devicesÚtestÚis_built_with_cudarK   rL   Úlist_physical_devicesÚexperimentalÚset_memory_growthÚ
distributeÚOneDeviceStrategyr¸   r¹   r   r©   r¬   ÚNotImplementedErrorr   rU   r¥   r   r   r¦   r`   rÞ   ÚRandomrc   Úconstantr[   rÒ   Úis_encoder_decoderÚ
isinstancer   r¶   rŸ   r\   r]   r^   r   r_   r·   r   rd   Únumbar3   Úget_current_deviceÚreset)rg   ri   rj   rk   r=   r*   rl   rm   rn   r6   r+   rz   Úphysical_devicesr¿   r?   r¼   r½   rB   rC   rÞ   ÚvaluesrÙ   rÛ   rä   r¾   r�   r3   r;   r-   )r‡   r›   rœ   r»   rÕ   r(   r1   Úrun_tensorflowÄ  sª   
€ÿ

û




ò

€ûÂüIrù   c                  C   s’  t  ¡ } | jddddtg d¢tt ¡ ƒdd t ¡ ¡ d� | jd	dd
tdddgdd� | jddtd ttƒdd t¡ d� | jddddtdgg d¢dd� | jdddtt	j
 dd¡dd� | jddtt	j
 dd¡dd� | jdd dd!d"d#� | jd$dtd d%d� | jd&d'ttjttƒd(d)� | jd*dd!d+d#� | jd,dd!d-d#� | jd.d/ttjttƒd0d)� | jd1d2dd!d3d#� | jd4d5dd d6d7� | jd8d9dd d:d7� | jd;d<dd d=d7� | jd>d?ddd
gtg d@¢dAdB� | jdCdDddEtdFdG� | jdHdIdtd
gdJ� | jdKdLdtg dM¢dJ� | jdNdd!dOd#� | jddP� | jdQdRddtdSgdTdU� | jdVdtd dWd� | jdXdd!dYd#� | jddZ� t | ¡ |  ¡ }|S )[Nz-mz--modelsFú+)zbert-base-casedzroberta-baseÚgpt2z Pre-trained models in the list: z, )ÚrequiredÚnargsÚtypeÚdefaultÚchoicesÚhelpz--model_sourcer&   r%   r(   zExport onnx from pt or tfz--model_classz!Model type selected in the list: )rü   rþ   rÿ   r   r  z-ez	--enginesr7   )r7   rR   r¡   r“   rÑ   zEngines to benchmarkz-cz--cache_dirÚ.Úcache_modelsz%Directory to cache pre-trained models)rü   rþ   rÿ   r  z
--onnx_dirÚonnx_modelszDirectory to store onnx modelsz-gz	--use_gpuÚ
store_truezRun on gpu device)rü   Úactionr  z
--providerzExecution provider to usez-pz--precisionzfPrecision of model to run. fp32 for full precision, fp16 for half precision, and int8 for quantization)rþ   rÿ   r   r  z	--verbosezPrint more informationz--overwritezOverwrite existing modelsz-oz--optimizer_infozjOptimizer info: Use optimizer.py to optimize onnx model as default. Can also choose from by_ort and no_optz-vz--validate_onnxzValidate ONNX modelz-fz--fusion_csvz:CSV file for saving summary results of graph optimization.)rü   rÿ   r  z-dz--detail_csvz#CSV file for saving detail results.z-rz--result_csvz$CSV file for saving summary results.z-iz--input_counts)r&   r'   r$   zXNumber of ONNX model inputs. Please use 1 for fair comparison with Torch or TorchScript.)rü   rý   rÿ   rþ   r   r  z-tz--test_timeséd   z8Number of repeat times to get average inference latency.)rü   rÿ   rþ   r  z-bz--batch_sizes)rý   rþ   rÿ   z-sz--sequence_lengths)é   é   é   é    é@   é€   é   z--disable_ort_io_bindingz=Disable running ONNX Runtime with binded inputs and outputs. )rt   z-nz--num_threadsr   zThreads to use)rü   rý   rþ   rÿ   r  z--force_num_layersz%Manually set the model's layer numberz*--enable_arm64_bfloat16_fastmath_mlas_gemmzHEnable bfloat16 mlas gemm kernels on aarch64. Supported only for CPU EP )rx   )ÚargparseÚArgumentParserÚadd_argumentr^   Úlistr   ÚkeysÚjoinr   ÚosÚpathr   ÚFLOAT32r   ÚBYSCRIPTÚintÚset_defaultsr   Úadd_argumentsÚ
parse_args)Úparserry   r-   r-   r1   Úparse_argumentsF  sV  
øù
ú	øú	ûûú	üú	ûûûûøú	ûüù
ûü
r  c                  C   sÀ  t ƒ } t| jƒ | jtjkr| jst d¡ d S | jtj	kr-| jr-| j
dvr-t d¡ d S t| jƒdkrCt| jd  d dv rCdg| _td	d
„ | jD ƒƒ| _t d| › �¡ tj | j¡svzt | j¡ W n tyu   t d| j¡ Y nw d| jv }d| jv }d| jv }d| jv }d| jv }|r§t tj¡t d¡k r§t dtj› �¡ d S t| jƒ}g }| jD ]ß}t |¡ t  tj! "¡ ¡ |sÇ|sÇ|�r+| j#dgkrÒt $d¡ |rï|t%| j| j| j&|| j|| j'| j| j(dd| j| jƒ7 }|�r|t%| j| j| j&|| j|| j'| j| j(dd| j| jƒ7 }|�r+|t%| j| j| j&|| j|| j'| j| j(dd| j| jƒ7 }|�rG|t)| j| j| j&|| j|| j'| j| j(| j| jƒ7 }i }	|�r�z4| j* }
|t+| j| j
| j| j&|| j|| j'| j| j(| j#| j,| j-| j| j.| j| j/| j0|
|	| j1| j2| ƒ7 }W q± t3�y�   t 4d¡ Y q±w q±t5 6¡  7d¡}|	�rª| j8�p¤d|› d�}t9|	|ƒ t|ƒdk�r¿| j'dgk�r½t $d¡ d S | j:�pÈd|› d�}t;||ƒ | j<�p×d|› d�}t=||| ƒ d S )Nzfp16 is for GPU only)ÚmigraphxÚrocmzint8 is for CPU onlyr&   r   r$   )rF   Úswimr   c                 S   s   h | ]
}|d kr
t n|’qS )r   )Ú	cpu_count)r/   Úxr-   r-   r1   Ú	<setcomp>  s    zmain.<locals>.<setcomp>zArguments: z#Creation of the directory %s failedrR   r¡   r“   r7   rÑ   z2.0.0z2PyTorch version must be >=2.0.0 and you are using zB--input_counts is not implemented for torch or torchscript engine.TFÚ	Exceptionz%Y%m%d-%H%M%SÚbenchmark_fusion_z.csvzNo any result available.Úbenchmark_detail_Úbenchmark_summary_)>r  r   r+   r=   r   r©   rg   rK   rL   r¬   rh   rO   Úmodelsr   rm   Úsortedr*   r`   r  r  Úexistsr6   ÚmkdirÚOSErrorÚenginesr   rQ   rR   r\   r   Úforce_num_layersÚset_num_threadsr§   Ú
__config__Úparallel_inforo   rN   rÀ   rj   rl   Ú
test_timesrù   Úuse_mask_indexr’   rp   rq   rr   rs   rt   rw   rx   r%  r¹   r   r_   ÚstrftimeÚ
fusion_csvr   Ú
detail_csvr
   Ú
result_csvr   )ry   Úenable_torchÚenable_torch2Úenable_torchscriptÚenable_onnxruntimeÚenable_tensorflowrk   rz   r*   rv   ru   Ú
time_stampÚcsv_filenamer-   r-   r1   Úmain  s  


$ÿ








óóóõ
éÿä


r@  Ú__main__)4Ú__doc__r  Úloggingr  r¶   r   rV   ÚpsutilÚbenchmark_helperr   r   r   r   r   r   r	   r
   r   r   r   r   r   Úhuggingface_modelsr   r   Úonnx_exporterr   r   r   r   Ú	packagingr   Úquantize_helperr   Ú	getLoggerrK   r"  Úenvironr^   rR   Útransformersr   r   r   r’   rÀ   ÚboolrÒ   rù   r  r@  Ú__name__r-   r-   r-   r1   Ú<module>   sB   4

 bp  I #
ÿ