o
    °ƒðj$X  ã                   @   sÖ  d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d dlZd dlZd dlZd dlZd dlmZ d dlZe e¡ZG d	d
„ d
eƒZG dd„ deƒZG dd„ dƒZdejiZ ddddddi fdd„Z!d8dd„Z"d9dd„Z#dd„ Z$dd„ Z%dd„ Z&dd „ Z'd:d!d"„Z(ej)d fd#d$„Z*d%d&„ Z+d;d(d)„Z,d*e-e.e/ef  dB fd+d,„Z0G d-d.„ d.eƒZ1G d/d0„ d0e1ƒZ2G d1d2„ d2e1ƒZ3d<d4d5„Z4d6d7„ Z5dS )=é    N)ÚABCÚabstractmethod)ÚThreadPoolExecutor)Údatetime)ÚEnum)Úsleep)ÚAny)Úversionc                   @   s$   e Zd ZdZdZdZdZdd„ ZdS )Ú	PrecisionÚfp32Úfp16Úint8Úint4c                 C   ó   | j S ©N©Úvalue©Úself© r   ú«/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/onnxruntime/transformers/benchmark_helper.pyÚ__str__&   ó   zPrecision.__str__N)Ú__name__Ú
__module__Ú__qualname__ÚFLOAT32ÚFLOAT16ÚINT8ÚINT4r   r   r   r   r   r
       s    r
   c                   @   s    e Zd ZdZdZdZdd„ ZdS )ÚOptimizerInfoÚno_optÚby_ortÚ	by_scriptc                 C   r   r   r   r   r   r   r   r   1   r   zOptimizerInfo.__str__N)r   r   r   ÚNOOPTÚBYORTÚBYSCRIPTr   r   r   r   r   r    *   s
    r    c                   @   s$   e Zd Zdd„ Zdd„ Zdd„ ZdS )ÚConfigModifierc                 C   ó
   || _ d S r   ©Ú
num_layers)r   r*   r   r   r   Ú__init__6   ó   
zConfigModifier.__init__c                 C   s‚   | j d u rd S t|dƒr| j |_t d| j › �¡ t|dƒr+| j |_t d| j › �¡ t|dƒr?| j |_t d| j › �¡ d S d S )NÚnum_hidden_layersz6Modifying pytorch model's number of hidden layers to: Úencoder_layersz7Modifying pytorch model's number of encoder layers to: zdecoder_layers z7Modifying pytorch model's number of decoder layers to: )r*   Úhasattrr-   ÚloggerÚinfor.   Údecoder_layers)r   Úconfigr   r   r   Úmodify9   s   



þzConfigModifier.modifyc                 C   r   r   r)   r   r   r   r   Úget_layer_numF   r   zConfigModifier.get_layer_numN)r   r   r   r+   r4   r5   r   r   r   r   r'   5   s    r'   Úfloat32TéÿÿÿÿFc	              	      sX  t  ¡ }	|rt jj|	_nt jj|	_|rd|	_|dkr&||	_t 	d|	j› �¡ |r,d|	_
nd|	_
|t  ¡ v r9|g}
n=|rs|dkrDddg}
n2|dkrMd	dg}
n)|d
krVg d¢}
n |dks^|d u rcddg}
n|dkrlg d¢}
n
td|› �ƒ‚dg}
ˆ r�‡ fdd„|
D ƒ}
|r‰|	 dd¡ d }zt j| |	|
d�}W |S  ty«   t d| › d|
› �¡ Y |S w )NTr   z%Session option: intra_op_num_threads=é   ÚdmlÚDmlExecutionProviderÚCPUExecutionProviderÚrocmÚROCMExecutionProviderÚmigraphx)ÚMIGraphXExecutionProviderr=   r;   ÚcudaÚCUDAExecutionProviderÚtensorrt)ÚTensorrtExecutionProviderrA   r;   z)The execution provider is not supported: c                    s$   g | ]}|ˆ v r|ˆ | fn|‘qS r   r   )Ú.0Úname©Úprovider_optionsr   r   Ú
<listcomp>‰   s   $ z.create_onnxruntime_session.<locals>.<listcomp>z(mlas.enable_gemm_fastmath_arm64_bfloat16Ú1)Ú	providerszFailed to create session for z with providers=)ÚonnxruntimeÚSessionOptionsÚGraphOptimizationLevelÚORT_ENABLE_ALLÚgraph_optimization_levelÚORT_ENABLE_BASICÚenable_profilingÚintra_op_num_threadsr0   ÚdebugÚlog_severity_levelÚget_available_providersÚRuntimeErrorÚadd_session_config_entryÚInferenceSessionÚ	ExceptionÚ	exception)Úonnx_model_pathÚuse_gpuÚproviderÚenable_all_optimizationÚnum_threadsrQ   ÚverboseÚ(enable_mlas_gemm_fastmath_arm64_bfloat16rG   Úsess_optionsrJ   Úsessionr   rF   r   Úcreate_onnxruntime_sessionP   sN   





ýýrd   c                 C   s8   | rt jddd� d S t jdd� t d¡ tj¡ d S )NÚDEBUGz8[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s)ÚlevelÚfmtz%(message)s)rg   Útransformers)ÚcoloredlogsÚinstallÚloggingÚ	getLoggerÚsetLevelÚWARNING)r`   r   r   r   Úsetup_logger—   s   
þro   c                 C   sü   | rt j | ¡st  | ¡ |rt j |¡st  |¡ |r:|dkr+dt ¡ v s*J dƒ‚ntt ¡ ƒ g d¢¡r:J dƒ‚t 	dt
j› �¡ t 	dtj› �¡ t 	dtj› �¡ t t
j¡t d	¡ksbJ ‚t tj¡t d
¡ksoJ ‚t tj¡t d	¡ks|J ‚d S )Nr9   r:   zBPlease install onnxruntime-directml package to test GPU inference.)rA   r=   r?   zWPlease install onnxruntime-gpu package, or install ROCm support, to test GPU inference.zPyTorch Version:zTransformers Version:zOnnxRuntime Version:z1.10.0z4.12.0)ÚosÚpathÚexistsÚmakedirsrK   rU   ÚsetÚ
isdisjointr0   r1   ÚtorchÚ__version__rh   r	   Úparse)Ú	cache_dirÚ
output_dirr\   r]   r   r   r   Úprepare_environment¢   s(   

ÿÿþr{   c                 C   sŒ   t | ƒtt| ƒƒ d }tj| tjd�d }|d|  }t| ƒ|d›t | d¡d d›t | d¡d d›t | d¡d d›|d›|d›dœS )Ng     @�@)Údtypez.2féZ   é_   éc   )Ú
test_timesÚlatency_varianceÚlatency_90_percentileÚlatency_95_percentileÚlatency_99_percentileÚaverage_latency_msÚQPS)ÚsumÚfloatÚlenÚnumpyÚvarÚfloat64Ú
percentile)Úlatency_listÚ
batch_sizeÚ
latency_msr�   Ú
throughputr   r   r   Úget_latency_result¾   s   ùr’   c                 C   sv   t |dddd��!}g d¢}tj||d�}| ¡  | D ]}| |¡ qW d   ƒ n1 s,w   Y  t d|› �¡ d S )NÚaÚ Úascii©ÚmodeÚnewlineÚencoding)Úenginer	   rJ   ÚdeviceÚ	precisionÚ	optimizerÚ
io_bindingÚ
model_nameÚinputsÚthreadsr�   Úsequence_lengthÚcustom_layer_numr   r€   r†   r…   r�   r‚   rƒ   r„   ©Ú
fieldnamesz&Detail results are saved to csv file: )ÚopenÚcsvÚ
DictWriterÚwriteheaderÚwriterowr0   r1   )ÚresultsÚcsv_filenameÚcsv_fileÚcolumn_namesÚ
csv_writerÚresultr   r   r   Úoutput_detailsÎ   s   ÿår±   c                    sÜ  t |dddd��Ô}g d¢‰ g }|jD ]"}|jdgkr#| d|› �¡ q|jD ]}| d|› d|› �¡ q&qtj|ˆ | d�}| ¡  |jD ]�}d	D ]Š}	|jD ]„}
d
D ]}|j	D ]y}i }| D ]k}|d |krÈ|d |	krÈ|d |
krÈ|d |krÈ|d |krÈ‡ fdd„| 
¡ D ƒ}|s™| |¡ | t |d¡¡ nˆ D ]}|| || ks§J ‚q›|d }|d }|r¿|d |d|› d|› �< q]|d |d|› �< q]|rÐ| |¡ qWqRqNqIqEW d   ƒ n1 sßw   Y  t d|› �¡ d S )Nr“   r”   r•   r–   )rŸ   r    r£   rš   r	   rJ   r›   rœ   r�   rž   r¡   ÚbÚ_sr¤   )é   é   é   )TFr”   rŸ   r    rš   rž   r¡   c                    s   i | ]\}}|ˆ v r||“qS r   r   )rD   ÚkÚv©Úheader_namesr   r   Ú
<dictcomp>  s    z"output_summary.<locals>.<dictcomp>r�   r¢   r…   z'Summary results are saved to csv file: )r¦   Úbatch_sizesÚsequence_lengthsÚappendr§   r¨   r©   ÚmodelsÚenginesr_   ÚitemsÚupdateÚdictÚfromkeysrª   r0   r1   )r«   r¬   Úargsr­   Ú
data_namesr�   r¢   r¯   rŸ   Úinput_countÚengine_namerž   r¡   Úrowr°   Úheadersr·   r²   Úsr   r¹   r   Úoutput_summaryð   sZ   

ÿ



€
€èÿÿÿÿè6rÌ   c                 C   sÒ   t |dddd��O}ddddgttt|  ¡ ƒƒ ¡ ƒ¢}tj||d	�}| ¡  | D ]'}t	t
 ¡ ƒ| | d< tj| | d< tj| | d< || | d< | | | ¡ q(W d   ƒ n1 sZw   Y  t d
|› �¡ d S )Nr“   r”   r•   r–   Úmodel_filenamer   rh   rv   r¤   z(Fusion statistics is saved to csv file: )r¦   ÚlistÚnextÚiterÚvaluesÚkeysr§   r¨   r©   Ústrr   Únowrh   rw   rv   rª   r0   r1   )Úmodel_fusion_statisticsr¬   r­   r®   r¯   Úkeyr   r   r   Úoutput_fusion_statistics*  s&   üûûör×   c                    sd   i }t j‡ ‡fdd„d|d� t j‡ ‡fdd„d|d�}| |¡ | ddi¡ | t||ƒ¡ |S )Nc                      ó   ˆ  d ˆ ¡S r   ©Úrunr   ©Ú
ort_inputsÚort_sessionr   r   Ú<lambda>@  ó    zinference_ort.<locals>.<lambda>r´   ©ÚnumberÚrepeatc                      rØ   r   rÙ   r   rÛ   r   r   rÞ   A  rß   rž   F)Útimeitrâ   rÂ   r’   )rÝ   rÜ   Úresult_templateÚrepeat_timesr�   Úwarm_up_repeatr°   rŽ   r   rÛ   r   Úinference_ort>  s   
rç   c              
      s  i }ˆ  ¡ ‰ |D ]&}t || ¡ |	¡}t t|| jƒ|
¡}ˆ  ||j	j
d||j| ¡ ¡ qt|ƒdkr;t|||	ƒ t|ƒD ]\}}ˆ  ||| j	j
dtj|| j||  ¡ ¡ q?tj‡ ‡fdd„d|d� tj‡ ‡fdd„d|d�}| |¡ | ddi¡ | t||ƒ¡ |S )	Nr   c                      ó
   ˆ  ˆ ¡S r   ©Úrun_with_iobindingr   ©rž   rÝ   r   r   rÞ   u  ó   
 z/inference_ort_with_io_binding.<locals>.<lambda>r´   rà   c                      rè   r   ré   r   rë   r   r   rÞ   {  rì   rž   T)rž   rv   Ú
from_numpyÚtoÚIO_BINDING_DATA_TYPE_MAPÚgetrÓ   r|   Ú
bind_inputr›   ÚtypeÚshapeÚdata_ptrr‰   ÚallocateOutputBuffersÚ	enumerateÚbind_outputrŠ   r6   rã   râ   rÂ   r’   )rÝ   rÜ   rä   rå   Úort_output_namesÚort_outputsÚoutput_buffersÚoutput_buffer_max_sizesr�   r›   Ú	data_typeræ   r°   rE   Únp_inputÚ
input_typeÚiÚort_output_namerŽ   r   rë   r   Úinference_ort_with_io_bindingH  sL   ú	

ú	ýý
r  c                 C   s&   |D ]}|   tj|tj|d�¡ qd S )N)r|   r›   )r¾   rv   Úemptyr6   )rú   rû   r›   rÿ   r   r   r   rõ   …  s   ÿrõ   é{   c                 C   s<   t  | ¡ tj  | ¡ t | ¡ tj | ¡ tj | ¡ dS )z5Set random seed manually to get deterministic resultsN)ÚrandomÚseedrŠ   rv   Úmanual_seedr@   Úmanual_seed_all)r  r   r   r   Úset_random_seed�  s
   

r  Úreturnc               
   C   sÒ   ddl m} m}m}m}m}m}m} z>|ƒ  g }|ƒ }t|t	ƒs#W d S t
|ƒD ]#}	|||	ƒƒ}
t|
tƒr8 W d S | |	|||	ƒƒ|
j|
j|
jdœ¡ q'|ƒ  |W S  | yh } ztd|ƒ W Y d }~d S d }~ww )Nr   ©Ú	NVMLErrorÚnvmlDeviceGetCountÚnvmlDeviceGetHandleByIndexÚnvmlDeviceGetMemoryInfoÚnvmlDeviceGetNameÚnvmlInitÚnvmlShutdown)ÚidrE   ÚtotalÚfreeÚusedú-Error fetching GPU information using nvml: %s)Úpy3nvml.py3nvmlr  r  r  r  r  r  r  Ú
isinstanceÚintÚrangerÓ   r¾   r  r  r  Úprint)r  r  r  r  r  r  r  r°   Údevice_countrÿ   r1   Úerrorr   r   r   Úget_gpu_info™  s4   $



ûÿ	
€þr  c                   @   s@   e Zd Zd
dd„Zdd„ Zedeeee	f  dB fdd	„ƒZ
dS )ÚMemoryMonitorTc                 C   r(   r   )Úkeep_measuring©r   r   r   r   r   r+   À  r,   zMemoryMonitor.__init__c                 C   sB   dd l }d}	 t|| t ¡ ¡ ¡ jd ƒ}tdƒ | js 	 |S q)Nr   Té   ç{®Gázt?)	ÚpsutilÚmaxÚProcessrp   ÚgetpidÚmemory_infoÚrssr   r   )r   r$  Ú	max_usager   r   r   Úmeasure_cpu_usageÃ  s   ûzMemoryMonitor.measure_cpu_usager	  Nc                 C   s   t ƒ ‚r   )ÚNotImplementedErrorr   r   r   r   Úmeasure_gpu_usageÎ  s   zMemoryMonitor.measure_gpu_usage©T)r   r   r   r+   r+  r   rÎ   rÃ   rÓ   r   r-  r   r   r   r   r  ¿  s
    
$r  c                       s<   e Zd Zd‡ fdd„	Zdeeeef  dB fdd„Z‡  Z	S )	ÚCudaMemoryMonitorTc                    s   t ƒ  |¡ d S r   )Úsuperr+   r!  ©Ú	__class__r   r   r+   Ô  s   zCudaMemoryMonitor.__init__r	  Nc           
   
      s>  ddl m}m}m‰m}m‰m}m} g ‰g ‰ zo|ƒ  |ƒ }t|t	ƒs-t
 d|› �¡ W d S dd„ t|ƒD ƒ‰‡‡fdd„t|ƒD ƒ‰ 	 t|ƒD ]%}|ˆ|ƒƒ}t|tƒr`t
 d|› �¡  W d S tˆ| |jd	 ƒˆ|< qGtd
ƒ | jsunqC|ƒ  ‡ ‡fdd„t|ƒD ƒW S  |yž }	 zt
 d|	¡ W Y d }	~	d S d }	~	ww )Nr   r
  z*nvmlDeviceGetCount result is not integer: c                 S   ó   g | ]}d ‘qS ©r   r   ©rD   rÿ   r   r   r   rH   ë  ó    z7CudaMemoryMonitor.measure_gpu_usage.<locals>.<listcomp>c                    s   g | ]}ˆˆ |ƒƒ‘qS r   r   r5  )r  r  r   r   rH   ì  s    Tz%nvmlDeviceGetMemoryInfo returns str: r"  r#  c                    ó    g | ]}|ˆ | ˆ| d œ‘qS ©)Ú	device_idrE   Úmax_used_MBr   r5  ©Úgpu_nameÚmax_gpu_usager   r   rH   ø  ó    üýÿr  )r  r  r  r  r  r  r  r  r  r  r0   r  r  rÓ   r%  r  r   r   )
r   r  r  r  r  r  r  rÿ   r1   r  r   )r<  r=  r  r  r   r-  ×  s>   $


÷
ú€þz#CudaMemoryMonitor.measure_gpu_usager.  )
r   r   r   r+   rÎ   rÃ   rÓ   r   r-  Ú__classcell__r   r   r1  r   r/  Ó  s    &r/  c                       s.   e Zd Zd‡ fdd„	Zdd„ Zdd„ Z‡  ZS )	ÚRocmMemoryMonitorTc                    sl   t ƒ  |¡ d}tj |¡r|tjvrtj |¡ zdd l}|| _| j ¡  W d S  t	y5   d | _Y d S w )Nz/opt/rocm/libexec/rocm_smir   )
r0  r+   rp   rq   rr   Úsysr¾   Úrocm_smiÚinitializeRsmiÚImportError)r   r   Úrocm_smi_pathrB  r1  r   r   r+     s   
ÿzRocmMemoryMonitor.__init__c                 C   s(   | j d u rdS | j  |d¡d d d S )Nr7   ÚVRAMr   i   )rB  Ú
getMemInfo)r   Údevr   r   r   Úget_used_memory  s   
z!RocmMemoryMonitor.get_used_memoryc                    s¢   | j d u rd S | j d urt| j  ¡ ƒnd}dd„ t|ƒD ƒ‰dd„ t|ƒD ƒ‰ 	 t|ƒD ]}tˆ| |  |¡ƒˆ|< q,t d¡ | jsDnq(‡ ‡fdd„t|ƒD ƒS )Nr   c                 S   r3  r4  r   r5  r   r   r   rH     r6  z7RocmMemoryMonitor.measure_gpu_usage.<locals>.<listcomp>c                 S   s   g | ]}d |› �‘qS )ÚGPUr   r5  r   r   r   rH     s    Tr#  c                    r7  r8  r   r5  r;  r   r   rH   &  r>  )	rB  r‰   ÚlistDevicesr  r%  rI  Útimer   r   )r   r  rÿ   r   r;  r   r-    s   

ûúz#RocmMemoryMonitor.measure_gpu_usager.  )r   r   r   r+   rI  r-  r?  r   r   r1  r   r@    s    r@  r@   c              	   C   sD  d }|dkr	t }nt}|dƒ}| rÅ|d ur|}n| ¡ }|d u r"d S |d u r(|S tƒ ��}|ƒ }| |j¡}z| |¡}	|	 ¡ }
W d|_| ¡ }nd|_| ¡ }w |d u r]	 W d   ƒ d S t d|› d|› �¡ t	|ƒdkr£t	|ƒdkr«t	|ƒt	|ƒkr³d}t
|ƒD ]\}}|d }|| d }|| }t||ƒ}q‚|W  d   ƒ S W d   ƒ d S W d   ƒ d S W d   ƒ d S 1 s¾w   Y  d S |d urÌ|}n| ¡ }|d u rÖ|S tƒ �>}|ƒ }| |j¡}z| |¡}	|	 ¡ }
W d|_| ¡ }nd|_| ¡ }w t d|d	›d
|d	›d�¡ || W  d   ƒ S 1 �sw   Y  d S )Nr<   FzGPU memory usage: before=z  peak=r´   r   r:  zCPU memory usage: before=z.1fz
 MB, peak=z MB)r@  r/  r-  r   Úsubmitr°   r   r0   r1   r‰   rö   r%  r+  )Úis_gpuÚfuncÚmonitor_typeÚstart_memoryÚmemory_monitor_typeÚmonitorÚmemory_before_testÚexecutorÚ
mem_threadÚ	fn_threadÚ_r*  Úmax_usedrÿ   Úmemory_beforeÚbeforeÚafterr  r   r   r   Úmeasure_memory0  s‚   


ÿ
õ(ê
ò÷
ò÷
òé


ÿ
&õr]  c                  C   sL   g d¢} d}| D ]}t  |¡}|d u rq|r|d7 }||› d|› �7 }q|S )N)ÚORT_DISABLE_FUSED_ATTENTIONÚ!ORT_ENABLE_FUSED_CAUSAL_ATTENTIONÚ!ORT_DISABLE_FUSED_CROSS_ATTENTIONÚORT_DISABLE_TRT_FLASH_ATTENTIONÚ&ORT_DISABLE_MEMORY_EFFICIENT_ATTENTIONÚORT_TRANSFORMER_OPTIONSÚORT_CUDA_GEMM_OPTIONSr”   ú,ú=)rp   Úgetenv)Ú	env_namesÚenvrE   r   r   r   r   Úget_ort_environment_variablest  s   	
rj  r.  r   r4  )r  )r@   N)6r§   rk   rp   r  rA  rL  rã   Úabcr   r   Úconcurrent.futuresr   r   Úenumr   r   Útypingr   ri   rŠ   rv   rh   Ú	packagingr	   rK   rl   r   r0   r
   r    r'   r6   rï   rd   ro   r{   r’   r±   rÌ   r×   rç   Úlonglongr  rõ   r  rÎ   rÃ   rÓ   r  r  r/  r@  r]  rj  r   r   r   r   Ú<module>   sf   

ÿ	
÷
G
":

ô=
&2
+D