o
    °ƒðj‚   ã                   @  sÜ  d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
Z
d dlZd dlmZmZmZ d dlmZ d dlmZmZmZmZ d dlmZ d d	lmZmZmZ zd d
lmZ W n eyi   dZY nw z
d dlmZm Z  W n ey�   dZdZ Y nw zd dl!m"Z" W n ey•   dZ"Y nw dZ#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,i Z-dd„ e.eƒD ƒZ/G dd„ deƒZ0G dd„ deƒZ1G dd„ deƒZ2G dd „ d eƒZ3ejj4e
 5d!¡ejj6e
 5d"¡ejj7e
 5d#¡ejj8e
 5d$¡ejj9eejj:eejj;e iZ<ejj6e
j=d e
j>d%�e
j=d&e
j>d%�fejj4e
j=d'e
j?d%�e
j=d(e
j?d%�fejj8e
j=d e
j@d%�e
j=d)e
j@d%�fejj7e
j=d*e
jAd%�e
j=d+e
jAd%�fejj;e
j=d e d%�e
j=d,e d%�fejj:e
j=d-ed%�e
j=d.ed%�fiZBejj6e
j=d e
j>d%�e
j=d/e
j>d%�fejj4e
j=d0e
j?d%�e
j=d(e
j?d%�fejj8e
j=d e
j@d%�e
j=d1e
j@d%�fejj7e
j=d2e
jAd%�e
j=d+e
jAd%�fiZCejj6e
j=d e
j>d%�e
j=d(e
j>d%�fejj4e
j=d3e
j?d%�e
j=d4e
j?d%�fejj8e
j=d e
j@d%�e
j=d+e
j@d%�fejj7e
j=d5e
jAd%�e
j=d6e
jAd%�fejj;e
j=d ed%�e
j=d.ed%�fejj:e
j=d7ed%�e
j=d8ed%�fiZDd9d:œd;d<„ZEd´d=d>„ZFdµd@dA„ZGdBdC„ ZH	?			d¶d·dQdR„ZI	d¶d¸dTdU„ZJ		d´d¹d^d_„ZKdºd`da„ZLdºdbdc„ZMd»dgdh„ZNd¼dldm„ZOG dndo„ doƒZPG dpdq„ dqƒZQG drds„ dsƒZRdtdu„ ZSdvdw„ ZTdxdy„ ZUdzd{„ ZVd½d€d�„ZWd‚dƒ„ ZXd¾d…d†„ZYd¿dˆd‰„ZZdÀd‹dŒ„Z[dÁdŽd�„Z\dÂd’d“„Z]dÃd”d•„Z^dÂd–d—„Z_dÃd˜d™„Z`dÄdšd›„ZadÅd�dž„ZbdÆdŸd „ZcdÇd¡d¢„ZddÈd¥d¦„ZedÉd¨d©„ZfdÉdªd«„ZgdÊd¬d­„ZhdÊd®d¯„ZidÊd°d±„ZjdÊd²d³„ZkdS )Ëé    )ÚannotationsN)ÚEnum)ÚPath)Ú
ModelProtoÚTensorProtoÚexternal_data_helper)Úonnx_pb)Ú
make_graphÚ
make_modelÚ	make_nodeÚmake_tensor_value_info)ÚReferenceEvaluator)ÚGraphOptimizationLevelÚInferenceSessionÚSessionOptions©Úfloat8e4m3fn)Úint4Úuint4)Úto_array_extendedzonnx.quantizez0.1.0úai.onnxzcom.microsoftÚQuantizeLinearÚ_QuantizeLinear_InputÚDequantizeLinearÚ_DequantizeLinear_OutputÚ
_quantizedl        c                 C  s(   i | ]}t tt|ƒtƒrtt|ƒ|“qS © )Ú
isinstanceÚgetattrr   Úint)Ú.0Úkr   r   ú¦/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/onnxruntime/quantization/quant_utils.pyÚ
<dictcomp>9   s   ( r#   c                   @  ó(   e Zd ZdZdZdd„ Zedd„ ƒZdS )ÚQuantizationModer   é   c                 C  ó   | j S ©N©Úname©Úselfr   r   r"   Ú__str__D   ó   zQuantizationMode.__str__c                 C  ó    zt |  W S  ty   tƒ ‚w r(   )r%   ÚKeyErrorÚ
ValueError)Úmoder   r   r"   Úfrom_stringG   ó
   
ÿzQuantizationMode.from_stringN)Ú__name__Ú
__module__Ú__qualname__Ú
IntegerOpsÚ
QLinearOpsr-   Ústaticmethodr3   r   r   r   r"   r%   @   ó    r%   c                   @  r$   )ÚQuantizedValueTyper   r&   c                 C  r'   r(   r)   r+   r   r   r"   r-   S   r.   zQuantizedValueType.__str__c                 C  r/   r(   )r<   r0   r1   )Úvr   r   r"   r3   V   r4   zQuantizedValueType.from_stringN)r5   r6   r7   ÚInputÚInitializerr-   r:   r3   r   r   r   r"   r<   O   r;   r<   c                   @  sH   e Zd ZdZdZdZdZdZdZdZ	dd	„ Z
ed
d„ ƒZedd„ ƒZdS )Ú	QuantTyper   r&   é   é   é   é   é   c                 C  r'   r(   r)   r+   r   r   r"   r-   g   r.   zQuantType.__str__c                 C  r/   r(   )r@   r0   r1   )Útr   r   r"   r3   j   r4   zQuantType.from_stringc                 C  s€   | t jkrtjS | t jkrtjS | t jkrtjS | t jkr tj	S | t j
kr(tjS | t jkr0tjS | t jkr8tjS td| ›d�ƒ‚)NzUnexpected value qtype=Ú.)r@   ÚQInt8r   ÚINT8ÚQUInt8ÚUINT8ÚQUInt16ÚUINT16ÚQInt16ÚINT16ÚQFLOAT8E4M3FNÚFLOAT8E4M3FNÚQUInt4ÚUINT4ÚQInt4ÚINT4r1   r+   r   r   r"   Útensor_typeq   s   






zQuantType.tensor_typeN)r5   r6   r7   rH   rJ   rP   rN   rL   rT   rR   r-   r:   r3   ÚpropertyrV   r   r   r   r"   r@   ^   s    
r@   c                   @  r$   )ÚQuantFormatr   r&   c                 C  r'   r(   r)   r+   r   r   r"   r-   ˆ   r.   zQuantFormat.__str__c                 C  r/   r(   )rX   r0   r1   )Úformatr   r   r"   r3   ‹   r4   zQuantFormat.from_stringN)r5   r6   r7   Ú	QOperatorÚQDQr-   r:   r3   r   r   r   r"   rX   „   r;   rX   Úint8Úuint8Úint16Úuint16©Údtypeéÿ   i€ÿÿÿé   iÿÿ  i €ÿÿiÿ  é   iøÿÿÿé   éþ   i�ÿÿÿiþÿ  i€ÿÿiÀÿÿÿé@   i Àÿÿi @  éüÿÿÿrB   éÿÿÿÿ©Úzero_point_indexc                 G  sº   g }t |ƒD ]H\}}t t|ƒtj¡r| t |¡¡ nt|tjƒr(| |¡ n
t	d|› d|› �ƒ‚|| krN|d }|j
tjksF|j
tjkrNt	d|j
› �ƒ‚qt|ƒdkrYt|ƒS |d S )Nzarg z is not an array: ri   zzero_point cannot be r&   r   )Ú	enumerateÚnumpyÚ
issubdtypeÚtypeÚnumberÚappendÚarrayr   ÚndarrayÚ	TypeErrorra   Úfloat32Úfloat16ÚlenÚtuple)rk   ÚargsÚnew_argsÚiÚar=   r   r   r"   Ú_check_type·   s   €r}   c                 C  s�  | t v sJ d| › d�ƒ‚| tjjtjjtjjtjjfv r†|dkr(td|›d�ƒ‚|jt	j
kr2tj}n|jt	jkr<tj}n	td|j› d�ƒ‚tttdg dgtj d| g dg¡d	�td
g d¢dgƒgdtd|d ƒtd|d ƒgtd| d ƒgƒƒ}t|ƒ}t| d ||dœ¡d ƒS t |  }	t| ddd�\}
}|d urœt|
|ƒn|
}|d ur§t||ƒn|}t	 | t	j
¡|  ¡ | ¡}t	j||||d� t| |	¡ƒS )NúUnexpected data type ú> requested. Only INT8, UINT8, INT16, and UINT16 are supported.r   z2zero_point is expected to be null for float 8 not rG   zUnexpected dtype ÚConstantÚ
zero_point)Úvaluer   )ÚXÚscaler�   ÚYÚqurƒ   r„   )rƒ   r„   F)Úreduce_rangeÚ	symmetric)Úout) ÚONNX_TYPE_TO_NP_TYPEÚ
onnx_protor   rQ   ÚFLOAT8E4M3FNUZÚ
FLOAT8E5M2ÚFLOAT8E5M2FNUZÚNotImplementedErrorra   rm   ru   ÚFLOATrv   ÚFLOAT16r1   r
   r	   r   ÚonnxÚhelperÚmake_tensorr   r   r}   ÚrunÚget_qmin_qmax_for_qTypeÚmaxÚminÚasarrayÚastypeÚroundÚclip)ÚqTypeÚarrr„   r�   ÚlowÚhighÚ	onnx_typeÚ
onnx_modelÚrefra   ÚqminÚqmaxÚcliplowÚcliphighÚarr_fp32r   r   r"   Úquantize_nparrayÇ   sN   

ÿüÿü

þôÿr©   Fc                 C  s¨  |dks|dk rt d|› d|› �ƒ‚t | tjd| jd�¡} t |tjd|jd�¡}|dur;t|| tj|| jd� ƒ}|rOt t | ¡t |¡¡}| } |
 }||ks]J d| › d|› �ƒ‚tj||  tj	d�}tj|tj	d�tj|tj	d� }t || ¡}	|	dks†J dƒ‚|	t 
|j¡jk r£tjd	|jd�}	tjd|jd�}
|
|	gS |r»tjt || tjd
tj	d� ¡|jd�}
ntjt || |	  ¡|jd�}
|	 |j¡}	|
|	gS )aè  Calculate the scale s and zero point z for the quantization relation
    r = s(q-z), where r are the original values and q are the corresponding
    quantized values.

    r and z are calculated such that every value within [rmin,rmax] has an
    approximate representation within [qmin,qmax]. In addition, qmin <= z <=
    qmax is enforced. If the symmetric flag is set to True, the interval
    [rmin,rmax] is symmetrized to [-absmax, +absmax], where
    absmax = max(abs(rmin), abs(rmax)).

    :parameter rmin: minimum value of r
    :parameter rmax: maximum value of r
    :parameter qmin: minimum value representable by the target quantization data type
    :parameter qmax: maximum value representable by the target quantization data type
    :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False.
    :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None.
    :return: zero and scale [z, s]

    r   úBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:ú, qmmax:r`   Nzqmin=z > qmax=zscale issueç      ð?g       @)r1   rm   Úminimumrr   ra   Úmaximumr—   r™   ÚabsÚfloat64ÚfinfoÚtinyr›   rš   )ÚrminÚrmaxr¤   r¥   rˆ   Úmin_real_rangeÚabsmaxÚdrÚdqr„   r�   r   r   r"   Úcompute_scale_zpû   s4    ó ÿr¹   c           	        sè   d}| t vr?| tjkr2ddlm‰  ddlm} |}‡ fdd„tdƒD ƒ}tj	dd„ |D ƒtj
d	�}ntd
| › d�ƒ‚|t | < n| tjkrLddlm} |}|du rXtd| › d�ƒ‚t t |  ¡}tj	d|d	�}tj	|| |jd	�}||gS )ar  Calculate the scale s for a float8 type (E4M3FN).
    The function assumes the coefficient distribution and the float 8
    distribution are similar to two gaussian laws.

    :return: zero and scale [z, s]

    More details in notebook `quantization_fp8.ipynb
    <https://github.com/microsoft/onnxruntime/blob/main/docs/python/notebooks/quantization_fp8.ipynb>`_.
    Nr   ©Úfloat8e4m3_to_float32r   c                   s   g | ]}ˆ |ƒ‘qS r   r   )r    r{   rº   r   r"   Ú
<listcomp>K  s    z+compute_scale_zp_float8.<locals>.<listcomp>é   c                 S  s$   g | ]}t  |¡st  |¡s|‘qS r   )rm   ÚisnanÚisinf)r    Úfr   r   r"   r¼   M  s   $ r`   zQuantization to element_type=z not implemented.zUnexpected element_type rG   )ÚFLOAT8_DISTRIBUTIONSr   rQ   Úonnx.numpy_helperr»   Ú#onnx.reference.custom_element_typesr   Úrangerm   rr   ru   r1   rt   Ústdra   )	Úelement_typerÅ   Úzp_dtyper   Ú
all_valuesÚvaluesÚstd_f8Úzeror„   r   rº   r"   Úcompute_scale_zp_float8:  s*   

ÿ

rÌ   Údataúnumpy.ndarrayÚ
quant_typeúonnx.TensorProto.DataTyperˆ   Úboolr‡   rµ   úfloat | NoneÚrmin_overrideÚrmax_overrideÚreturnú#tuple[numpy.ndarray, numpy.ndarray]c                 C  sP  t | tjƒstdt| ƒ› d�ƒ‚|dur|}n
t| ƒr|  ¡ nd}|dur(|}n
t| ƒr0|  ¡ nd}tj|| j	d�}tj|| j	d�}tjd| j	d�}	|t
jkrh|rUtdƒ‚t | ¡}
t||
ƒ\}}	t||	dd	�S |t
jt
jt
jt
jt
jt
jfv r t|||d
�\}}t| ƒr‘t||||||ƒ\}}	ntjd|j	d�}t||	dd	�S td|› d�ƒ‚)a¯  
    Returns the zero_point and scale for the given data.

    :param data: The data for which to compute quantization parameters.
    :param quant_type: The quantization data type.
    :param symmetric: whether symmetric quantization is used or not.
    :parameter reduce_range: True if the quantization range should be reduced. Defaults to False.
    :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None.
    :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data).
    :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data).
    :return: zero point and scale
    z%Weight must be given as an array not rG   Ng        r`   r¬   z1Unsupported option reduce_range=True for float 8.r   rj   ©rˆ   z Unexpected value for quant_type=)r   rm   rs   rt   ro   rw   r˜   r—   rr   ra   r   rQ   ÚRuntimeErrorrÅ   rÌ   r}   rI   rK   rO   rM   rU   rS   r–   r¹   r1   )rÍ   rÏ   rˆ   r‡   rµ   rÓ   rÔ   r³   r´   r„   rÅ   r�   r¤   r¥   r   r   r"   Úcompute_data_quant_params_  s>   

úrÙ   ú2tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]c              
   C  sÚ   t | ||||||ƒ\}}|tjkrIt|| ||ƒ}	t|	 tj¡ ¡ d@ dkƒrDt 	| ¡}
t
d|
 ¡ › d|
 ¡ › d|	 ¡ › d|	 ¡ › d�	ƒ‚|||	fS |tjtjtjtjtjtjfv ret|| ||ƒ}	|||	fS td|› d�ƒ‚)al  
    :param data: data to quantize
    :param qType: data type to quantize to.
    :param symmetric: whether symmetric quantization is used or not.
    :parameter reduce_range: True if the quantization range should be reduced. Defaults to False.
    :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None.
    :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data).
    :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data).
    :return: minimum, maximum, zero point, scale, and quantized weights

    To pack weights, we compute a linear transformation

    - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and
    - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where
        `m = max(abs(rmin), abs(rmax))`

    and add necessary intermediate nodes to transform quantized weight to full weight using the equation

    :math:`r = S(q-z)`, where

    - *r*: real original value
    - *q*: quantized value
    - *S*: scale
    - *z*: zero point
    rc   z+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rG   )rÙ   r   rQ   r©   Úanyrš   rm   r]   Úravelr™   rØ   r˜   r—   rI   rK   rO   rM   rU   rS   r1   )rÍ   r�   rˆ   r‡   rµ   rÓ   rÔ   r�   r„   Úquantized_dataÚnp_datar   r   r"   Úquantize_data�  s@   ù
	
ÿÿÿ
ú
rß   Úweightúonnx.TensorProtor�   r„   Úaxisú
int | NoneÚquant_weight_nameú
str | Nonec                 C  s  t | ƒ}d}|du rt|| ¡ ||ƒ}n?|j| }t|jƒ}	d|	|< g }
t|ƒD ]$}| ||¡}|| }|| }t|| ¡ ||ƒ}|
 t 	|¡ 
|	¡¡ q(t |
|¡}|rW|n| j› t› �}|tjjkr¾t ¡ }||_|j | j¡ ||_| ¡  ¡  ¡ |_tdur¼t|ƒ}|j|jks”| ¡ | ¡ kr¼td|j› d| ¡ dd… › d| ¡ dd… › d| j› dt|ƒdd	… › d
�ƒ‚|S |tjjtjjfv rï|jtjtjfvrÙtd|› d�ƒ‚t t!| ¡ ƒƒ}tj"j#||| j|dd�}|S tj" $|¡}tj	||d� 
| j¡}tj% &||¡}|S )aG  
    Returns a quantized version of the given ONNX initializer.

    :param weight: The ONNX initializer to quantize.
    :param quant_type: The final quantized data type.
    :param zero_point: The zero-point value to use for quantization.
    :param scale: The scale value to use for quantization.
    :param axis: The quantization axis if quantizing per-channel. Defaults to None.
    :param quant_weight_name: The name of the quantized initializer.
                              If not specified, the quantized name is generated.
    :return: The quantized ONNX initializer.
    Nr&   zThe initializer of shape z! could not be created, expecting é
   z, got z and shape=z
raw=éÈ   rG   zQuantized weights for z. must be 8-bit before packing as 4-bit values.T)Úrawr`   )'Útensor_proto_to_arrayr©   rÜ   ÚshapeÚlistrÄ   Útakerq   rm   r™   ÚreshapeÚconcatenater*   ÚTENSOR_NAME_QUANT_SUFFIXr’   r   rQ   Ú	data_typeÚdimsÚextendÚflattenÚcopyÚtobytesÚraw_datar   rØ   ÚstrrU   rS   ra   r\   r]   ÚbytesÚpack_bytes_to_4bitr“   r”   Útensor_dtype_to_np_dtypeÚnumpy_helperÚ
from_array)rà   rÏ   r�   r„   râ   rä   Úweight_dataÚq_weight_dataÚchannel_countÚchannel_dimsÚquantized_channel_data_listr{   Úchannel_dataÚchannel_scaleÚchannel_zero_pointÚquantized_channel_dataÚq_weight_nameÚq_weight_initializerÚcheckÚpacked_dataÚquant_np_dtyper   r   r"   Úquantize_onnx_initializerÚ  sb   

ÿ
ÿÿÿþÿñür  c                 C  s¦   | t jjkr
tdƒ‚d}|rt | ¡}n|r| tv rt|  }nt | ¡}|s.td| › d�ƒ‚|\}}|dks:|dk rQtd|› d|› d|j	› d	|› d
|› d| › �ƒ‚|S )zÍ
    Return qmin and qmax, the minimum and maximum value representable by the given qType
    :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8
    :return: qmin, qmax
    z;This function is not implemented for float 8 as not needed.Nr~   r   r   rª   r«   z, dtype=z, reduce_range=z, symmetric=z, qType=)
r‹   r   rQ   r�   ÚONNX_INT_TYPE_REDUCED_RANGEÚgetÚONNX_INT_TYPE_SYMMETRIC_RANGEÚONNX_INT_TYPE_RANGEr1   ra   )r�   r‡   rˆ   Úqranger¤   r¥   r   r   r"   r–   )  s8   

ÿÿÿÿþþÿr–   c                 C  s   t | ||d�\}}|| S )z“
    Helper function to get the quantization range for a type.
        parameter qType: quantization type.
        return: quantization range.
    r×   )r–   )r�   r‡   rˆ   r¤   r¥   r   r   r"   Úget_qrange_for_qTypeI  s   r  r   Úrankútuple[bool, int]c                 C  s,   | dk r| | n| }|dko||k }||fS )zç
    Helper function that tries to return a normalized axis in the range [0, rank - 1].
    :parameter axis: The axis to normalize.
    :parameter rank: The tensor rank (number of dimensions).
    :return (is_valid, axis_norm)
    r   r   )râ   r  Ú	axis_normÚis_validr   r   r"   Únormalize_axisS  s   r  Úsrc_8bitrø   Ú	bytearrayc                 C  sš   t | ƒ}|dkrtƒ S |d d }t|ƒ}d}d}||d k r?| |d  d@ d> | | d@ B ||< |d7 }|d7 }||d k s||k rK| | d@ ||< |S )aB  
    Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values.
    Assumes that the source values are already in the appropriate int4 range.
    :parameter src_8bit: The 8-bit element values to pack.
    :return A bytearray with every two 8-bit src elements packed into a single byte.
    r   r&   rA   rd   rC   )rw   r  )r  Ú	num_elemsÚdst_sizeÚdstÚsrc_iÚdst_ir   r   r"   rù   _  s   $ýrù   c                   @  s    e Zd ZdZg g dfdd„ZdS )ÚQuantizedInitializerzJ
    Represents a linearly quantized weight input from ONNX operators
    Nc
           
      C  ó:   || _ || _|| _|| _|| _|| _|| _|| _|	| _d S r(   )	r*   ÚinitializerÚrminsÚrmaxsÚzero_pointsÚscalesrÍ   rÝ   râ   )
r,   r*   r   r!  r"  r#  r$  rÍ   rÝ   râ   r   r   r"   Ú__init__‚  s   
zQuantizedInitializer.__init__©r5   r6   r7   Ú__doc__r%  r   r   r   r"   r  }  s    ör  c                   @  s"   e Zd ZdZ				ddd„ZdS )ÚQuantizedValuezI
    Represents a linearly quantized value (input\output\intializer)
    Nc
           
      C  r  r(   )	Úoriginal_nameÚq_nameÚ
scale_nameÚzp_nameÚ
value_typerâ   Ú	node_typeÚ
node_qtypeÚ
scale_type)
r,   r*   Únew_quantized_namer+  Úzero_point_nameÚquantized_value_typerâ   r.  r/  r0  r   r   r"   r%  ¡  s   
zQuantizedValue.__init__)NNNNr&  r   r   r   r"   r(  œ  s    ör(  c                   @  s   e Zd ZdZdd„ ZdS )ÚBiasToQuantizez+
    Represents a bias to be quantized
    c                 C  s   || _ || _|| _d S r(   )Ú	bias_nameÚ
input_nameÚweight_name)r,   r5  r6  r7  r   r   r"   r%  ½  s   
zBiasToQuantize.__init__Nr&  r   r   r   r"   r4  ¸  s    r4  c                 C  sô   | j dkrtd| j› d�ƒ‚| j dkr| j}n^| j dkr | j}nU| j dkr)| j}nL| j dkr2| j}nC| j dkr;| j}n:| j d	krD| j}n1| j d
krM| j	}n(| j dkrV| j
}n| j dkr_| j}n| j dkrh| j}ntd| j› d| j › d�ƒ‚| j|iS )zÄ
    Convert attribute to kwarg format for use with onnx.helper.make_node.
        :parameter attribute: attribute in AttributeProto format.
        :return: attribute in {key: value} format.
    r   z
attribute z does not have type specified.r&   rA   rB   rC   rD   rE   re   é   é	   ræ   z has unsupported type rG   )ro   r1   r*   rÀ   r{   ÚsrF   ÚgÚfloatsÚintsÚstringsÚtensorsÚgraphs)Ú	attributer‚   r   r   r"   Úattribute_to_kwargÃ  s0   











rB  c                   s*   ‡ fdd„|D ƒ}t |ƒdkr|d S dS )zÃ
    Helper function to find item by name in a list.
        parameter item_name: name of the item.
        parameter item_list: list of items.
        return: item if found. None otherwise.
    c                   s   g | ]	}|j ˆ kr|‘qS r   r)   )r    Úitem©Ú	item_namer   r"   r¼   ï  s    z find_by_name.<locals>.<listcomp>r   N)rw   )rE  Ú	item_listÚitemsr   rD  r"   Úfind_by_nameè  s   rH  c                 C  s*   d}t t|ƒƒD ]
}|| | kr|}q|S )zC
    Helper function to return index of an item in a node list
    ri   )rÄ   rw   )Ú	elem_nameÚ	elem_listÚelem_idxr{   r   r   r"   Úget_elem_indexó  s   €rL  c                 C  s   t j d| |g|¡S )zÝ
    Helper function to create a Mul node.
        parameter inputs: list of input names.
        parameter output: output name.
        parameter name: name of the node.
        return: Mul node in NodeProto format.
    ÚMul)r’   r“   r   )ÚinputsÚoutputr*   r   r   r"   Úget_mul_nodeþ  s   rP  Úfilenamer   Ú
identifierr÷   c                 C  s   | j  | j| | j ¡S )zp
    Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix.
    )ÚparentÚjoinpathÚstemÚsuffix)rQ  rR  r   r   r"   Úgenerate_identified_filename	  s   rW  c                 C  s„   dd l }dd lm} dd l}|j|jd� tdƒ t| ƒ tdƒ t|ƒ |j| |dd� | d¡ | 	d¡ | 
d	¡ | ¡  d S )
Nr   )Ú	thresholdz
Histogram:zHistogram Edges:T)ÚfillzTensor valueÚCountszTensor value V.S. Counts)ÚsysÚmatplotlib.pyplotÚpyplotrm   Úset_printoptionsÚmaxsizeÚprintÚstairsÚxlabelÚylabelÚtitleÚshow)ÚhistÚ
hist_edgesr[  Úpltrm   r   r   r"   Ú
apply_plot  s   


ri  rG   c                   s<  ddl ‰ddl}ddl‰ddlm  m  m} ddlm  m  m} ddl	m
‰ m‰m‰ t d| › �¡ G ‡ ‡‡‡‡fdd„dˆjƒ}ˆj| |d�}ttj |d¡d	ƒ�}| |¡ W d  ƒ n1 sfw   Y  ˆ d¡}| d
¡}	g }
t|  ¡ ƒD ]I}| | }| ¡ }t| d|¡ ¡ ƒt| d|¡ ¡ ƒg}tt|ƒƒ}|	  |¡}|	  |¡}| !|	¡ | "|	|¡ | #|	|¡ | $|	¡}|
 %|¡ q}| &|	t'|
ƒ¡ |
D ]}|	 (|¡ qÑ|	 )¡ }| *|	¡ | +|	|¡ | ,|	¡}|	 -|¡ |	 .¡ }ttj |d¡dƒ�}| |¡ W d  ƒ n	1 �sw   Y  tj/ dd¡dv �rF|j 0|d¡}| 1¡ }t2|ƒD ]}| 3|¡}t | 4¡ ¡ t | 5¡ ¡ �q/ttj |d¡d	ƒ�C}t|  ¡ ƒD ]3}| | }| ¡ }t| d|¡ ¡ ƒt| d|¡ ¡ ƒg}|d tt|ƒƒ }| |¡ | d¡ �qWW d  ƒ dS 1 �s—w   Y  dS )z>
    Helper function to write calibration table to files.
    r   N)ÚCalibrationMethodÚ
TensorDataÚTensorsDatazcalibration cache: c                      s    e Zd Z‡ ‡‡‡‡fdd„ZdS )z*write_calibration_table.<locals>.MyEncoderc                   sb   t |ˆˆfƒr| ¡ S t |ˆjƒr| ¡ t|jƒddœS t |ˆ ƒr*|jjt|ƒdœS ˆj 	| |¡S )Nznumpy.array)rÍ   ra   ÚCLS)rm  r‚   )
r   Úto_dictrs   Útolistr÷   ra   Ú	__class__r5   ÚJSONEncoderÚdefault)r,   Úobj©rj  rk  rl  ÚjsonÚnpr   r"   rr  3  s   
z2write_calibration_table.<locals>.MyEncoder.defaultN)r5   r6   r7   rr  r   rt  r   r"   Ú	MyEncoder2  s    rw  )Úclszcalibration.jsonÚwi   ÚhighestÚlowestzcalibration.flatbuffersÚwbÚQUANTIZATION_DEBUGÚ0)r&   Ú1zcalibration.cacheú Ú
)6ru  Úflatbuffersrm   Ú5onnxruntime.quantization.CalTableFlatBuffers.KeyValueÚquantizationÚCalTableFlatBuffersÚKeyValueÚ5onnxruntime.quantization.CalTableFlatBuffers.TrtTableÚTrtTableÚ"onnxruntime.quantization.calibraterj  rk  rl  ÚloggingÚinforq  ÚdumpsÚopenÚosÚpathÚjoinÚwriterr   ÚBuilderÚsortedÚkeysrn  Úfloatr  rC  r÷   r—   ÚCreateStringÚKeyValueStartÚKeyValueAddKeyÚKeyValueAddValueÚKeyValueEndrq   ÚTrtTableStartDictVectorrw   ÚPrependUOffsetTRelativeÚ	EndVectorÚTrtTableStartÚTrtTableAddDictÚTrtTableEndÚFinishÚOutputÚenvironÚGetRootAsTrtTableÚ
DictLengthrÄ   ÚDictÚKeyÚValue)Úcalibration_cacheÚdirr‚  r†  rˆ  rw  Ú	json_dataÚfilerË   ÚbuilderÚkey_value_listÚkeyrÉ   Úd_valuesr<  r‚   Úflat_keyÚ
flat_valueÚ	key_valueÚ	main_dictÚ	cal_tableÚbufÚdict_lenr{   r   rt  r"   Úwrite_calibration_table"  sx   
ÿ

þ






ÿ
þ
÷$ÿr¸  ç-Cëâ6?c                 C  sª   | dk  tj¡}| dk  tj¡}| ¡ }| j| }|sdS |t|ƒ t|ƒ }|dk s8J d|› d|› d|› �ƒ‚|   tj¡}||| | |  7 }|dk ¡ dksSJ ‚|S )a~  Given a discrete distribution (may have not been normalized to 1),
    smooth it by replacing zeros with eps multiplied by a scaling factor
    and taking the corresponding amount off the non-zero values.
    Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf
         https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py
    r   Nr¬   zn_zeros=z, n_nonzeros=z, eps1=)rš   rm   ru   ÚsumÚsizer•  )ÚpÚepsÚis_zerosÚis_nonzerosÚn_zerosÚ
n_nonzerosÚeps1rf  r   r   r"   Úsmooth_distribution~  s   
"rÃ  Ú
model_pathc                 C  s(   t j|  ¡ dd�}tdd„ |jjD ƒƒS )NF)Úload_external_datac                 s  s   � | ]}t  |¡V  qd S r(   )r   Úuses_external_data)r    Ú
intializerr   r   r"   Ú	<genexpr>™  s   € z*model_has_external_data.<locals>.<genexpr>)r’   ÚloadÚas_posixrÛ   Úgraphr   )rÄ  Úmodelr   r   r"   Úmodel_has_external_data—  s   rÍ  Úopt_model_pathc                 C  sF   t ƒ }| ¡ |_tj|_i }dg|d< t|  ¡ |fddgi|¤Ž}dS )zø
        Generate model that applies graph optimization (constant folding, etc.)
        parameter model_path: path to the original onnx model
        parameter opt_model_path: path to the optimized onnx model
    :return: optimized onnx model
    ÚConstantSharingÚdisabled_optimizersÚ	providersÚCPUExecutionProviderN)r   rÊ  Úoptimized_model_filepathr   ÚORT_ENABLE_BASICÚgraph_optimization_levelr   )rÄ  rÎ  Úsess_optionÚkwargsÚ_r   r   r"   Úoptimize_modelœ  s   

 rÙ  rÌ  r   c                 C  s>   ddi}| j r| j D ]}| |j|ji¡ q
tj | |¡ dS )z>Tag the model that it went through quantization pre-processingúonnx.quant.pre_processúonnxruntime.quantN©Úmetadata_propsÚupdater¯  r‚   r’   r“   Úset_model_props)rÌ  rÝ  Úpropr   r   r"   Úadd_pre_process_metadata¬  s
   
rá  c                 C  ó0   | j r| j D ]}|jdkr|jdkr dS qdS )zCCheck the model whether it went through quantization pre-processingrÚ  rÛ  TF©rÝ  r¯  r‚   )rÌ  rà  r   r   r"   Úmodel_has_pre_process_metadataµ  s   
€rä  c                 C  s>   ddi}| j r| j D ]}| |j|ji¡ q
tj | |¡ d S )Nú
onnx.inferrÛ  rÜ  )rÌ  rÝ  r¼  r   r   r"   Úadd_infer_metadata¾  s
   
ræ  c                 C  râ  )Nrå  rÛ  TFrã  )rÌ  r¼  r   r   r"   Úmodel_has_infer_metadataÆ  s   
€rç  c                 C  s2   dd„ | j D ƒ}t|ƒdkrtdƒ‚|d j}|S )Nc                 S  s    g | ]}|j r|j d kr|‘qS )r   )Údomain)r    Úopsetr   r   r"   r¼   Ï  s     z%get_opset_version.<locals>.<listcomp>r&   z$Failed to find proper ai.onnx domainr   )Úopset_importrw   r1   Úversion)rÌ  Úai_onnx_domainÚopset_versionr   r   r"   Úget_opset_versionÎ  s
   
rî  Úweight_typec                 C  s    t | ƒ}|}t|d|ƒ}|dk r"|tjjkr"t d|› d�¡ d}n|dkr0t d|› d�¡ n|dk r?t d|› d�¡ d}||krNtj | |¡} t	| ƒ} | S )	NrV   é   z$The original model opset version is z¨, which does not support quantization to float 8. Please update the model to opset >= 19. Automatically update the model to opset 19. Please verify the quantized model.ræ   ze, which does not support node fusions. Please update the model to opset >= 11 for better performance.z�, which does not support quantization. Please update the model to opset >= 11. Automatically update the model to opset 11. Please verify the quantized model.é   )
rî  r   r’   r   rQ   rŠ  ÚwarningÚversion_converterÚconvert_versionÚ&save_and_reload_model_with_shape_infer)rÌ  rï  rí  Útarget_opset_versionÚweight_quant_typer   r   r"   Úupdate_opset_version×  s*   
ÿ
ÿ
ÿrø  c                 C  sB   t | dƒ}tj t| ƒt|ƒ¡ t | ¡ ¡}t|ƒ | ¡  |S )Nz	-inferred)	rW  r’   Úshape_inferenceÚinfer_shapes_pathr÷   rÉ  rÊ  ræ  Úunlink)rÄ  Úinferred_model_pathrÌ  r   r   r"   Úload_model_with_shape_inferû  s   
rý  c                 C  sd   t jdd��"}t | ¡}t|ƒ d¡}tj|| ¡ dd� t	|ƒW  d   ƒ S 1 s+w   Y  d S )Nz
ort.quant.)Úprefixz
model.onnxT)Úsave_as_external_data)
ÚtempfileÚTemporaryDirectoryrô   Údeepcopyr   rT  r’   Ú
save_modelrÊ  rý  )rÌ  Úquant_tmp_dirÚ
model_copyrÄ  r   r   r"   rõ    s   
$ürõ  r   r   c                 C  s>   | j tjjtjjfv rtj | ¡S td| j	› dt
| j  › �ƒ‚)Nz&Only float type is supported. Weights z is )rð   r‹   r   r�   r‘   r’   rû   Úto_arrayr1   r*   Útype_to_name)r   r   r   r"   ré     s
   ÿré   Útensor_namec                 C  ó   | d S )NÚ_QuantizeLinearr   ©r  r   r   r"   Úadd_quant_suffix  ó   r  c                 C  ó   | t  S r(   )ÚQUANT_INPUT_SUFFIXr  r   r   r"   Úadd_quant_input_suffix  r  r  c                 C  r	  )NÚ_QuantizeLinear_Outputr   r  r   r   r"   Úadd_quant_output_suffix  r  r  c                 C  r	  )NÚ_DequantizeLinearr   r  r   r   r"   Úadd_dequant_suffix!  r  r  c                 C  r	  )NÚ_DequantizeLinear_Inputr   r  r   r   r"   Úadd_dequant_input_suffix%  r  r  c                 C  r  r(   )ÚDEQUANT_OUTPUT_SUFFIXr  r   r   r"   Úadd_dequant_output_suffix)  r  r  )NN)FN)FNNN)rÍ   rÎ   rÏ   rÐ   rˆ   rÑ   r‡   rÑ   rµ   rÒ   rÓ   rÒ   rÔ   rÒ   rÕ   rÖ   )rÕ   rÚ   )rà   rá   rÏ   rÐ   r�   rÎ   r„   rÎ   râ   rã   rä   rå   rÕ   rá   )FF)râ   r   r  r   rÕ   r  )r  rø   rÕ   r  )rQ  r   rR  r÷   rÕ   r   )rG   )r¹  )rÄ  r   )rÄ  r   rÎ  r   )rÌ  r   )rÌ  r   rÕ   rÑ   )rÌ  r   rÕ   r   )rÌ  r   rï  r@   rÕ   r   )rÄ  r   rÕ   r   )rÌ  r   rÕ   r   )r   r   rÕ   rÎ   )r  r÷   rÕ   r÷   )rÕ   r÷   )lÚ
__future__r   rô   rŠ  rŽ  r   Úenumr   Úpathlibr   rm   r’   r   r   r   r   r‹   Úonnx.helperr	   r
   r   r   Úonnx.referencer   Úonnxruntimer   r   r   rÃ   r   ÚImportErrorr   r   Úonnx.reference.op_runr   Ú__producer__Ú__version__Úonnx_domainÚ	ms_domainÚQUANT_OP_NAMEr  ÚDEQUANT_OP_NAMEr  rï   ÚMODEL_SIZE_THRESHOLDrÁ   rª  r  r%   r<   r@   rX   rI   ra   rK   rO   rM   rQ   rU   rS   rŠ   rr   r]   r\   r_   r^   r  r  r  r}   r©   r¹   rÌ   rÙ   rß   r  r–   r  r  rù   r  r(  r4  rB  rH  rL  rP  rW  ri  r¸  rÃ  rÍ  rÙ  rá  rä  ræ  rç  rî  rø  rý  rõ  ré   r  r  r  r  r  r  r   r   r   r"   Ú<module>   sî   ÿþþ&ù$$$$  ú$
$$$ü$$$$  ú


4?)ù?ÿBú
O
 


%


\



	
	


	
$
	

	



