o
    °ƒðj Î  ã                   @   sÎ  d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlZd dlZd dlmZmZmZmZ d dlZddlmZmZmZ dejd	ejd
ejfdd„Z		 d0dejd	ejdedB ded
ejf
dd„ZG dd„ dƒZG dd„ dƒZG dd„ deƒZG dd„ de jd�Z G dd„ dƒZ!G dd„ de!ƒZ"G dd„ de!ƒZ#G d d!„ d!e#ƒZ$G d"d#„ d#e#ƒZ%G d$d%„ d%e#ƒZ&G d&d'„ d'e jd�Z'G d(d)„ d)e'ƒZ(dd*ej)d+di fd,e*e
B d-ee* dB fd.d/„Z+dS )1é    N)ÚSequence)ÚEnum)ÚPath)Ú
ModelProtoÚTensorProtoÚhelperÚnumpy_helperé   )Ú
apply_plotÚload_model_with_shape_inferÚsmooth_distributionÚpkÚqkÚreturnc                 C   s|   t j| j| jd�}| dd… t  | dd… |dd…  ¡ |dd…< | dk|dk@ }d||< | dk|dk@ }t j|| < |S )z‘
    See https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.rel_entr.html#scipy.special.rel_entr.
    Python implementation.
    ©ÚdtypeNr   )ÚnpÚemptyÚshaper   ÚlogÚinf)r   r   ÚresÚc2Úc1© r   ú¤/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/onnxruntime/quantization/calibrate.pyÚrel_entr   s   2r   ÚbaseÚaxisc                 C   sÆ   |du s|dksJ dƒ‚|dusJ dƒ‚t  | ¡ t j¡} d|  t j| |dd� } t  |¡ t j¡}t  | |¡\} }d| t j||dd� }t| |ƒ}t j||d�}|dur]|t  |¡ }| | j¡S )	zÉ
    Simplifeied version of entropy.
    Source: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.entropy.html.
    This avoids taking a dependency on scipy just for this function.
    Nr   z0base={base} must be a positive number or `None`.z
qk is Noneç      ð?T)r   Úkeepdims©r   )	r   ÚasarrayÚastypeÚfloat32ÚsumÚbroadcast_arraysr   r   r   )r   r   r   r   ÚvecÚsr   r   r   Úentropy'   s   
r)   c                   @   sL   e Zd Zeg d¢ƒZeg d¢ƒZdd„ Zedd„ ƒZedd„ ƒZ	d	d
„ Z
dS )Ú
TensorData)ÚavgÚstdÚlowestÚhighestÚhistÚ
hist_edgesÚbins)r+   r,   r-   r.   r0   c                 K   s¦   t | ¡ ƒ| _| ¡ D ]E\}}|tjvr td|›dtj› d�ƒ‚|tjv rJt|dƒs6tdt	|ƒ› d|›�ƒ‚|j
tjtjfvrJtd|j
› d|›�ƒ‚t| ||ƒ qd S )NzUnexpected value z not in Ú.r   úUnexpected type z for k=zUnexpected dtype )ÚlistÚkeysÚ_attrsÚitemsr*   Ú_allowedÚ
ValueErrorÚ_floatsÚhasattrÚtyper   r   Úfloat16r$   Úsetattr)ÚselfÚkwargsÚkÚvr   r   r   Ú__init__G   s   


øzTensorData.__init__c                 C   ó4   t | dƒr
t | dƒstdt| ƒ› d�ƒ‚| j| jfS )Nr-   r.   z0Attributes 'lowest' and/or 'highest' missing in r2   )r;   ÚAttributeErrorÚdirr-   r.   ©r?   r   r   r   Úrange_valueS   ó   zTensorData.range_valuec                 C   rD   )Nr+   r,   z)Attributes 'avg' and/or 'std' missing in r2   )r;   rE   rF   r+   r,   rG   r   r   r   Úavg_stdY   rI   zTensorData.avg_stdc                    s$   ‡ fdd„ˆ j D ƒ}ˆ jj|d< |S )Nc                    s   i | ]}|t ˆ |ƒ“qS r   )Úgetattr)Ú.0rA   rG   r   r   Ú
<dictcomp>a   s    z&TensorData.to_dict.<locals>.<dictcomp>ÚCLS)r6   Ú	__class__Ú__name__©r?   Údatar   rG   r   Úto_dict_   s   zTensorData.to_dictN)rP   Ú
__module__Ú__qualname__Ú	frozensetr8   r:   rC   ÚpropertyrH   rJ   rS   r   r   r   r   r*   C   s    

r*   c                   @   sf   e Zd ZdeeeeB f fdd„Zdd„ Zdd„ Z	dd	„ Z
d
d„ Zdd„ Zdd„ Zdd„ Zdd„ ZdS )ÚTensorsDatarR   c              	   C   sú   || _ i | _| ¡ D ]p\}}t|tƒstdt|ƒ› d�ƒ‚t|tƒrf|tj	kr;t
|ƒdkr;t|d |d d�| j|< q
t
|ƒdkrUt|d |d |d |d d	�| j|< q
td
|d›dt
|ƒ› d|› d�ƒ‚t|tƒsutdt|ƒ› d�ƒ‚|| j|< q
d S )NzKeys must be strings not r2   é   r   r	   ©r-   r.   é   é   )r-   r.   r/   r1   zUnexpected tuple for Úrz	, it has z elements: zValues must be TensorData not )Úcalibration_methodrR   r7   Ú
isinstanceÚstrÚ	TypeErrorr<   ÚtupleÚCalibrationMethodÚMinMaxÚlenr*   )r?   r^   rR   rA   rB   r   r   r   rC   g   s"   

&"
ózTensorsData.__init__c                 c   s   � | j E d H  d S ©N©rR   rG   r   r   r   Ú__iter__y   s   €zTensorsData.__iter__c                 C   s
   || j v S rf   rg   ©r?   Úkeyr   r   r   Ú__contains__|   ó   
zTensorsData.__contains__c                 C   s
   | j | S rf   rg   ri   r   r   r   Ú__getitem__   rl   zTensorsData.__getitem__c                 C   s(   || j vrtd|›d�ƒ‚|| j |< d S )Nz)Only an existing tensor can be modified, z is not.)rR   ÚRuntimeError)r?   rj   Úvaluer   r   r   Ú__setitem__‚   s   
zTensorsData.__setitem__c                 C   ó
   | j  ¡ S rf   )rR   r5   rG   r   r   r   r5   ‡   rl   zTensorsData.keysc                 C   rq   rf   )rR   ÚvaluesrG   r   r   r   rr   Š   rl   zTensorsData.valuesc                 C   rq   rf   )rR   r7   rG   r   r   r   r7   �   rl   zTensorsData.itemsc                 C   s   | j j| j| jdœ}|S )N)rN   rR   r^   )rO   rP   rR   r^   rQ   r   r   r   rS   �   s
   ýzTensorsData.to_dictN)rP   rT   rU   Údictr`   r*   rb   rC   rh   rk   rm   rp   r5   rr   r7   rS   r   r   r   r   rX   f   s    rX   c                   @   s   e Zd ZdZdZdZdZdS )rc   r   r	   rY   r\   N)rP   rT   rU   rd   ÚEntropyÚ
PercentileÚDistributionr   r   r   r   rc   š   s
    rc   c                   @   sV   e Zd Zedd„ ƒZejdefdd„ƒZdd„ Z	dd	„ Z
d
d„ Zdedefdd„ZdS )ÚCalibrationDataReaderc                 C   s   t |dƒr
t|jƒptS )NÚget_next)r;   Úcallablerx   ÚNotImplemented)ÚclsÚsubclassr   r   r   Ú__subclasshook__¢   s   z&CalibrationDataReader.__subclasshook__r   c                 C   ó   t ‚)z9generate the input data dict for ONNXinferenceSession run©ÚNotImplementedErrorrG   r   r   r   rx   ¦   s   zCalibrationDataReader.get_nextc                 C   s   | S rf   r   rG   r   r   r   rh   «   ó   zCalibrationDataReader.__iter__c                 C   s   |   ¡ }|d u r
t‚|S rf   )rx   ÚStopIteration)r?   Úresultr   r   r   Ú__next__®   s   zCalibrationDataReader.__next__c                 C   r~   rf   r   rG   r   r   r   Ú__len__´   r�   zCalibrationDataReader.__len__Ústart_indexÚ	end_indexc                 C   r~   rf   r   )r?   r†   r‡   r   r   r   Ú	set_range·   r�   zCalibrationDataReader.set_rangeN)rP   rT   rU   Úclassmethodr}   ÚabcÚabstractmethodrs   rx   rh   r„   r…   Úintrˆ   r   r   r   r   rw   ¡   s    
rw   )Ú	metaclassc                   @   s†   e Zd Z					ddeeB dee dB fdd„Zdgfd	d
„Zdd„ Zde	fdd„Z
dd„ Zdd„ Zdefdd„Zdefdd„ZdS )ÚCalibraterBaseNúaugmented_model.onnxFÚ
model_pathÚop_types_to_calibratec                 C   sn   t |tƒrtt|ƒƒ| _nt |tƒrt|ƒ| _ntdƒ‚|| _|| _|| _|| _	|| _
d| _d| _dg| _dS )a  
        :param model_path: ONNX model to calibrate. It should be a model file path
        :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors.
        :param augmented_model_path: save augmented model to this path.
        :param symmetric: make range of tensor symmetric (central point is 0).
        :param use_external_data_format: use external data format to store model which size is >= 2Gb.
        :param per_channel: whether to compute ranges per each channel.
        z model_path should be model path.NÚCPUExecutionProvider)r_   r`   r   r   Úmodelr9   r‘   Úaugmented_model_pathÚ	symmetricÚuse_external_data_formatÚper_channelÚaugment_modelÚinfer_sessionÚexecution_providers)r?   r�   r‘   r”   r•   r–   r—   r   r   r   rC   ¼   s   

zCalibraterBase.__init__r’   c                 C   s   || _ |  ¡  dS )zz
        reset the execution providers to execute the collect_data. It triggers to re-creating inference session.
        N)rš   Úcreate_inference_session)r?   rš   r   r   r   Úset_execution_providersÞ   s   z&CalibraterBase.set_execution_providersc                 C   s,   t  ¡ }t jj|_t j| j|| jd�| _dS )z9
        create an OnnxRuntime InferenceSession.
        )Úsess_optionsÚ	providersN)	ÚonnxruntimeÚSessionOptionsÚGraphOptimizationLevelÚORT_DISABLE_ALLÚgraph_optimization_levelÚInferenceSessionr”   rš   r™   )r?   r�   r   r   r   r›   å   s   
ýz'CalibraterBase.create_inference_sessionr“   c           	      C   sà   dd„ |j jD ƒ}| dd„ |j jD ƒ¡ | dd„ |j jD ƒ¡ dd„ |j jD ƒ}tƒ }tjtj	h}|j j
D ]4}| jrB|j| jv rkt |j|j¡D ] }||v rj|| }|j d¡rj|jjj|v rj||vrj| |¡ qJq7||fS )zÉ
        select input/output tensors of candidate nodes to calibrate.
        returns:
            tensors (set): set of tensor name.
            value_infos (dict): tensor name to value info.
        c                 S   ó   i | ]}|j |“qS r   ©Úname©rL   Úvir   r   r   rM   ø   ó    z>CalibraterBase.select_tensors_to_calibrate.<locals>.<dictcomp>c                 S   r¥   r   r¦   )rL   Úotr   r   r   rM   ù   rª   c                 S   r¥   r   r¦   )rL   Úitr   r   r   rM   ú   rª   c                 S   ó   h | ]}|j ’qS r   r¦   )rL   Úinitr   r   r   Ú	<setcomp>û   ó    z=CalibraterBase.select_tensors_to_calibrate.<locals>.<setcomp>Útensor_type)ÚgraphÚ
value_infoÚupdateÚoutputÚinputÚinitializerÚsetr   ÚFLOATÚFLOAT16Únoder‘   Úop_typeÚ	itertoolsÚchainr<   ÚHasFieldr±   Ú	elem_typeÚadd)	r?   r“   Úvalue_infosr·   Útensors_to_calibrateÚtensor_type_to_calibrater»   Útensor_namer©   r   r   r   Úselect_tensors_to_calibrateñ   s$   
ÿ
€z*CalibraterBase.select_tensors_to_calibratec                 C   ó   | j S )zP
        return: augmented onnx model. Call after calling augment_graph
        )r“   rG   r   r   r   Úget_augment_model  s   z CalibraterBase.get_augment_modelc                 C   r~   )zï
        abstract method: augment the input model to prepare for collecting data. It will:
            1. augment the model to be able to collect desired statistics data
            2. save augmented model to augmented_model_paths
        r   rG   r   r   r   Úaugment_graph  s   zCalibraterBase.augment_graphÚdata_readerc                 C   r~   )z€
        abstract method: collect the tensors that will be used for range computation. It can be called multiple times.
        r   )r?   rÊ   r   r   r   Úcollect_data  ó   zCalibraterBase.collect_datar   c                 C   r~   )ze
        abstract method: compute data based on the calibration method stored in TensorsData
        r   rG   r   r   r   Úcompute_data"  rÌ   zCalibraterBase.compute_data)Nr�   FFF)rP   rT   rU   r`   r   r   rC   rœ   r›   r   rÆ   rÈ   rÉ   rw   rË   rX   rÍ   r   r   r   r   rŽ   »   s$    ùþ

ý"rŽ   c                       sx   e Zd Z								ddeeB dee dB f‡ fdd„Zd	d
„ Zdd„ Zde	fdd„Z
dd„ Zdefdd„Z‡  ZS )ÚMinMaxCalibraterNr�   Fç{®Gáz„?r�   r‘   c
           
         s|   t ƒ j||||||	d� g | _d| _t| jjjƒ| _dd„ | jjjD ƒ| _	|| _
|r6|dk s2|dkr6tdƒ‚|| _|| _dS )aw  
        :param model_path: ONNX model to calibrate. It is a model path
        :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors.
        :param augmented_model_path: save augmented model to this path.
        :param symmetric: make range of tensor symmetric (central point is 0).
        :param use_external_data_format: use external data format to store model which size is >= 2Gb
        :param moving_average: compute the moving average of the minimum and maximum values instead of the global minimum and maximum.
        :param averaging_constant: constant smoothing factor to use when computing the moving average.
        :param max_intermediate_outputs: maximum number of intermediate outputs before an intermediate range is computed.
        :param per_channel: whether to compute ranges per each channel.
        )r‘   r”   r•   r–   r—   Nc                 S   r­   r   r¦   ©rL   rµ   r   r   r   r¯   L  r°   z,MinMaxCalibrater.__init__.<locals>.<setcomp>r   r	   z;Invalid averaging constant, which should not be < 0 or > 1.)ÚsuperrC   Úintermediate_outputsÚcalibrate_tensors_rangere   r“   r²   rµ   Únum_model_outputsÚmodel_original_outputsÚmoving_averager9   Úaveraging_constantÚmax_intermediate_outputs)
r?   r�   r‘   r”   r•   r–   rÖ   r×   rØ   r—   ©rO   r   r   rC   *  s"   ú
zMinMaxCalibrater.__init__c                    s–   ˆ  ˆj¡\}}tt ¡ ƒ‰t tjdgtj	d�ˆ¡}ˆjj
j |¡ dd„ ‰ ‡ ‡‡fdd„}|D ]}||dƒ ||dƒ q1tjˆjˆjˆjd	� d
S )zÙ
        Adds ReduceMin and ReduceMax nodes to all quantization_candidates op type nodes in
        model and ensures their outputs are stored as part of the graph output
        :return: augmented ONNX model
        éÿÿÿÿr   c                 S   s6   |j D ]}tj | |j¡r|j  S qtd| › d�ƒ‚)Nz&Model does not contain a version for 'z'.)Úopset_importÚonnxÚdefsÚhasÚdomainÚversionrn   )r¼   r“   rÛ   r   r   r   Úget_op_version^  s
   

ÿz6MinMaxCalibrater.augment_graph.<locals>.get_op_versionc                    s€  d}| d | }|d }t jj|| g|g||d�}t jjd|ˆg|g|d�}dd„ ˆjjjD ƒ}| d	d„ ˆjjjD ƒ¡ | d
d„ ˆjjjD ƒ¡ | |v rV||  j	j
j}ntd| ›d�ƒ‚ˆjr¦t||  j	j
jjƒ}	dgtd|	ƒ¢}
ˆ |ˆjƒdk r†|j t d|
¡¡ n tt ¡ ƒ}t tj|
tjd�|¡}|j |¡ ˆjjj |¡ ˆjjj ||g¡ ˆjjj t  ||d g¡¡ d S )Nr	   Ú_Ú_Reshape)r    r§   ÚReshape)ÚinputsÚoutputsr§   c                 S   r¥   r   r¦   r¨   r   r   r   rM   w  rª   zNMinMaxCalibrater.augment_graph.<locals>.add_reduce_min_max.<locals>.<dictcomp>c                 S   r¥   r   r¦   )rL   Úor   r   r   rM   x  rª   c                 S   r¥   r   r¦   ©rL   Úir   r   r   rM   y  rª   z'Unable to guess tensor type for tensor zE, running shape inference before quantization may resolve this issue.r   rY   é   Úaxesr   )!rÜ   r   Ú	make_noder“   r²   r³   r´   rµ   r¶   r<   r±   rÀ   r9   r—   re   r   ÚdimÚrangeÚ	attributeÚappendÚmake_attributer`   ÚuuidÚuuid4r   Ú
from_arrayr   ÚarrayÚint64r·   r»   ÚextendÚmake_tensor_value_info)rÅ   Úreduce_op_namer    Úreduce_outputÚintermediate_outputÚreduce_nodeÚreshape_noderÂ   Ú	onnx_typeÚtensor_rankÚreduced_axesÚreduce_axes_nameÚreduce_axes©rá   Úreshape_shape_namer?   r   r   Úadd_reduce_min_maxd  s>   ÿü
ÿ z:MinMaxCalibrater.augment_graph.<locals>.add_reduce_min_maxÚ	ReduceMinÚ	ReduceMax©Úsave_as_external_dataN)rÆ   r“   r`   rò   ró   r   rô   r   rõ   rö   r²   r·   rð   rÜ   Úsaver”   r–   )r?   Útensorsrâ   Úreshape_shaper  Útensorr   r  r   rÉ   S  s   .

ýzMinMaxCalibrater.augment_graphc                 C   ó
   g | _ d S rf   ©rÒ   rG   r   r   r   Úclear_collected_dataœ  rl   z%MinMaxCalibrater.clear_collected_datarÊ   c                 C   sœ   	 |  ¡ }|sn| j | j d |¡¡ | jd ur$t| jƒ| jkr$|  ¡  qt| jƒdkr5| jd u r5t	dƒ‚|  
¡ }t|tƒsHtdt|ƒ› d�ƒ‚|  ¡  d S )NTr   úNo data is collected.z+compute_data must return a TensorsData not r2   )rx   rÒ   rð   r™   ÚrunrØ   re   r  rÓ   r9   rÍ   r_   rX   ra   r<   )r?   rÊ   rå   Útr   r   r   rË   Ÿ  s   
÷
zMinMaxCalibrater.collect_datac                 C   sô   |s|S |  ¡ D ]o\}}t|tƒr|jd }|jd }n|\}}t|| tƒr6|| jd }|| jd }n|| \}}| jrR|| j||   }	|| j||   }
n
t||ƒ}	t||ƒ}
t|tƒsht|| tƒrqt|	|
d�||< q|	|
f||< q|S )Nr   r	   rZ   )r7   r_   r*   rH   rÖ   r×   ÚminÚmax)r?   Ú	old_rangeÚ	new_rangerj   ro   Úold_minÚold_maxÚnew_minÚnew_maxÚ	min_valueÚ	max_valuer   r   r   Úmerge_range³  s(   



zMinMaxCalibrater.merge_ranger   c                    sÂ  t ˆjƒdkr
ˆjS ‡fdd„tt ˆjd ƒƒD ƒ‰‡fdd„ˆjD ƒ}i ‰|D ]}| ¡ D ]\}}ˆ |g ¡ |¡ q.q(ˆˆjd… ‰ ‡ fdd„tdt ˆ ƒdƒD ƒ}‡‡fdd	„ˆD ƒ}g }tdt ˆ ƒdƒD ]X}ˆjr…t	j
|ˆ |  dd
�}	t	j
|ˆ |d   dd
�}
nt	j|ˆ |  dd
�}	t	j|ˆ |d   dd
�}
ˆjr¸t	jt	 |	¡t	 |
¡gdd
�}| | |f¡ qg| |	|
f¡ qgttjtt||dd�ƒƒ}ˆjrÛˆ ˆj|¡ˆ_ˆjS |ˆ_ˆjS )zŒ
        Compute the min-max range of tensor
        :return: dictionary mapping: {added node names: (ReduceMin, ReduceMax) pairs }
        r   c                    s   g | ]
}ˆ j  ¡ | j‘qS r   )r™   Úget_outputsr§   rè   rG   r   r   Ú
<listcomp>Ý  s    z1MinMaxCalibrater.compute_data.<locals>.<listcomp>c                    ó   g | ]}t tˆ |d d�ƒ‘qS ©F©Ústrict©rs   Úzip©rL   rû   ©Úoutput_namesr   r   r   Þ  ó    ÿÿNc                    s   g | ]}ˆ |   d ¡d ‘qS )râ   r   )Ú
rpartitionrè   )Úadded_output_namesr   r   r   è  s    ÿrY   c                    s    i | ]}|ˆj vr|ˆ | “qS r   )rÕ   rè   )Úmerged_output_dictr?   r   r   rM   ì  s    z1MinMaxCalibrater.compute_data.<locals>.<dictcomp>r!   r	   Fr#  )re   rÒ   rÓ   rî   r7   Ú
setdefaultrð   rÔ   rÖ   r   ÚnanmeanÚnanminÚnanmaxr•   ÚabsrX   rc   rd   rs   r&  r  )r?   Úoutput_dicts_listÚdrA   rB   Úcalibrate_tensor_namesÚmerged_added_output_dictÚpairsré   Úmin_value_arrayÚmax_value_arrayÚmax_absolute_valueÚnew_calibrate_tensors_ranger   )r,  r-  r)  r?   r   rÍ   Ô  sJ    
þÿ
ÿÿÿþzMinMaxCalibrater.compute_data)Nr�   FFFrÏ   NF)rP   rT   rU   r`   r   r   rC   rÉ   r  rw   rË   r  rX   rÍ   Ú__classcell__r   r   rÙ   r   rÎ   )  s&    öþ
ý)I!rÎ   c                       sr   e Zd Z									dd	eeB d
ee dB f‡ fdd„Zdd„ Zdd„ Zde	fdd„Z
defdd„Z‡  ZS )ÚHistogramCalibraterNr�   FÚ
percentileé€   é   ç-²�ïÿX@Úsamer�   r‘   c                    sv   t ƒ j|||||d� g | _d| _t| jjjƒ| _dd„ | jjjD ƒ| _	d| _
|| _|| _|| _|	| _d| _|
| _dS )a=  
        :param model_path: ONNX model to calibrate. It is a model path.
        :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors.
        :param augmented_model_path: save augmented model to this path.
        :param use_external_data_format: use external data format to store model which size is >= 2Gb
        :param method: A string. One of ['entropy', 'percentile'].
        :param symmetric: make range of tensor symmetric (central point is 0).
        :param num_bins: number of bins to create a new histogram for collecting tensor values.
        :param num_quantized_bins: number of quantized bins. Default 128.
        :param percentile: A float number between [0, 100]. Default 99.99.
        :param scenario: see :class:`DistributionCalibrater`
        )r‘   r”   r•   r–   Nc                 S   r­   r   r¦   rÐ   r   r   r   r¯   .  r°   z/HistogramCalibrater.__init__.<locals>.<setcomp>)rÑ   rC   rÒ   rÓ   re   r“   r²   rµ   rÔ   rÕ   Ú	collectorÚmethodÚnum_binsÚnum_quantized_binsr>  rÃ   Úscenario)r?   r�   r‘   r”   r–   rD  r•   rE  rF  r>  rG  rÙ   r   r   rC     s$   û
zHistogramCalibrater.__init__c                 C   sV   |   | j¡\| _}| jD ]}|| jvr| jjj || ¡ qtj| j| j	| j
d� dS )zƒ
        make all quantization_candidates op type nodes as part of the graph output.
        :return: augmented ONNX model
        r  N)rÆ   r“   rÃ   rÕ   r²   rµ   rð   rÜ   r
  r”   r–   )r?   rÂ   r  r   r   r   rÉ   7  s   

€
ýz!HistogramCalibrater.augment_graphc                 C   r  rf   r  rG   r   r   r   r  G  rl   z(HistogramCalibrater.clear_collected_datarÊ   c                    sB  dd„ ˆj  ¡ D ƒ}dd„ ˆj  ¡ D ƒ‰	 | ¡ }|sn-ˆj  d|¡}g }t|ƒD ]\}}ˆ| |v r<| t |¡¡ q)| |¡ q)ˆj |¡ qt	ˆjƒdkrTt
dƒ‚‡fd	d„ˆjD ƒ}i ‰ |D ]}	|	 ¡ D ]\}
}ˆ  |
g ¡ |¡ qhqb‡ ‡fd
d„ˆ D ƒ}ˆjs•tˆjˆjˆjˆjˆjˆjd�ˆ_ˆj |¡ ˆ ¡  dS )zy
        Entropy Calibrator collects operators' tensors as well as generates tensor histogram for each operator.
        c                 S   r­   r   r¦   ©rL   Únode_argr   r   r   r¯   N  r°   z3HistogramCalibrater.collect_data.<locals>.<setcomp>c                 S   s   g | ]}|j ‘qS r   r¦   rH  r   r   r   r   O  r°   z4HistogramCalibrater.collect_data.<locals>.<listcomp>TNr   r  c                    r!  r"  r%  r'  r(  r   r   r   e  r*  c                    s    i | ]}|ˆj v r|ˆ | “qS r   )rÃ   rè   )Úmerged_dictr?   r   r   rM   o  s     z4HistogramCalibrater.collect_data.<locals>.<dictcomp>)rD  r•   rE  rF  r>  rG  )r™   Ú
get_inputsr  rx   r  Ú	enumeraterð   ÚcopyrÒ   re   r9   r7   r.  rC  ÚHistogramCollectorrD  r•   rE  rF  r>  rG  Úcollectr  )r?   rÊ   Úinput_names_setrå   ræ   Úfixed_outputsÚoutput_indexrµ   r3  r4  rA   rB   Úclean_merged_dictr   )rJ  r)  r?   r   rË   J  sH   ñ
þÿúz HistogramCalibrater.collect_datar   c                 C   sh   | j stdƒ‚t| tƒrtj}nt| tƒrtj}nt| tƒr"tj	}n
t
dt| ƒ› d�ƒ‚t|| j  ¡ ƒS )z€
        Compute the min-max range of tensor
        :return: dictionary mapping: {tensor name: (min value, max value)}
        z9No collector created and can't generate calibration data.zUnknown calibrater z". This method must be overwritten.)rC  r9   r_   ÚEntropyCalibraterrc   rt   ÚPercentileCalibraterru   ÚDistributionCalibraterrv   ra   r<   rX   Úcompute_collection_result)r?   Úcalr   r   r   rÍ   ~  s   


z HistogramCalibrater.compute_data)	Nr�   Fr>  Fr?  r@  rA  rB  )rP   rT   rU   r`   r   r   rC   rÉ   r  rw   rË   rX   rÍ   r<  r   r   rÙ   r   r=  
  s&    õþ
ý,4r=  c                       sB   e Zd Z							d
deeB dee dB f‡ fdd	„Z‡  ZS )rT  Nr�   Fr)   r?  r�   r‘   c	           	   
      ó    t ƒ j||||||||d� dS )aÄ  
        :param model_path: ONNX model to calibrate. It is a model path
        :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors.
        :param augmented_model_path: save augmented model to this path.
        :param use_external_data_format: use external data format to store model which size is >= 2Gb
        :param method: A string. One of ['entropy', 'percentile', 'distribution'].
        :param symmetric: make range of tensor symmetric (central point is 0).
        :param num_bins: number of bins to create a new histogram for collecting tensor values.
        :param num_quantized_bins: number of quantized bins. Default 128.
        )rD  r•   rE  rF  N©rÑ   rC   )	r?   r�   r‘   r”   r–   rD  r•   rE  rF  rÙ   r   r   rC   ’  ó   
øzEntropyCalibrater.__init__)Nr�   Fr)   Fr?  r?  ©rP   rT   rU   r`   r   r   rC   r<  r   r   rÙ   r   rT  ‘  ó    ÷þ
ýrT  c                       sB   e Zd Z							ddeeB dee dB f‡ fd	d
„Z‡  ZS )rU  Nr�   Fr>  r@  rA  r�   r‘   c	           	   
      rY  )a¯  
        :param model_path: ONNX model to calibrate. It is a model path
        :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors.
        :param augmented_model_path: save augmented model to this path.
        :param use_external_data_format: use external data format to store model which size is >= 2Gb
        :param method: A string. One of ['entropy', 'percentile', 'distribution'].
        :param symmetric: make range of tensor symmetric (central point is 0).
        :param num_quantized_bins: number of quantized bins. Default 128.
        :param percentile: A float number between [0, 100]. Default 99.99.
        )rD  r•   rE  r>  NrZ  )	r?   r�   r‘   r”   r–   rD  r•   rE  r>  rÙ   r   r   rC   ´  r[  zPercentileCalibrater.__init__)Nr�   Fr>  Fr@  rA  r\  r   r   rÙ   r   rU  ³  r]  rU  c                       s@   e Zd Z						ddeeB dee dB f‡ fd	d
„Z‡  ZS )rV  Nr�   FÚdistributionr?  rB  r�   r‘   c              	      s   t ƒ j|||||||d� dS )aŒ  
        :param model_path: ONNX model to calibrate. It is a model path
        :param op_types_to_calibrate: operator types to calibrate. By default, calibrate all the float32/float16 tensors.
        :param augmented_model_path: save augmented model to this path.
        :param use_external_data_format: use external data format to store model which size is >= 2Gb
        :param method: A string. One of ['entropy', 'percentile', 'distribution'].
        :param symmetric: make range of tensor symmetric (central point is 0).
        :param num_bins: number of bins to create a new histogram for collecting tensor values.
        :param scenario: for float 8 only, if `scenario="same"`,
            the algorithm weights and float 8 follow the same distribution,
            if `scenario="p3"`, it assumes the weights follow
            a gaussian law and float 8 ~ X^3 where X is a gaussian law
        )rD  rE  rG  NrZ  )r?   r�   r‘   r”   r–   rD  rE  rG  rÙ   r   r   rC   Ö  s   
ùzDistributionCalibrater.__init__)Nr�   Fr^  r?  rB  r\  r   r   rÙ   r   rV  Õ  s    øþ
ýrV  c                   @   s,   e Zd ZdZejdd„ ƒZejdd„ ƒZdS )ÚCalibrationDataCollectorzL
    Base class for collecting data for calibration-based quantization.
    c                 C   r~   )z‹
        Generate informative data based on given data.
            name_to_arr : dict
                tensor name to NDArray data
        r   ©r?   Úname_to_arrr   r   r   rO  ý  s   z CalibrationDataCollector.collectc                 C   r~   )z?
        Get the optimal result among collection data.
        r   rG   r   r   r   rW    s   z2CalibrationDataCollector.compute_collection_resultN)rP   rT   rU   Ú__doc__rŠ   r‹   rO  rW  r   r   r   r   r_  ø  s    
r_  c                   @   sv   e Zd ZdZdd„ Zdd„ Zdd„ Zdd	„ Zd
d„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zeddd„ƒZdd„ Zdd„ ZdS )rN  a`  
    Collecting histogram for each tensor. Percentile and Entropy method are supported.

    ref: https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py
    ref: https://docs.nvidia.com/deeplearning/tensorrt/pytorch-quantization-toolkit/docs/_modules/
                 pytorch_quantization/calib/histogram.html
    c                 C   s.   i | _ || _|| _|| _|| _|| _|| _d S rf   )Úhistogram_dictrD  r•   rE  rF  r>  rG  )r?   rD  r•   rE  rF  r>  rG  r   r   r   rC     s   
zHistogramCollector.__init__c                 C   rÇ   rf   )rc  rG   r   r   r   Úget_histogram_dict   s   z%HistogramCollector.get_histogram_dictc                 C   sH   t dƒ | jdv r|  |¡S | jdkr | jr|  |¡S |  |¡S tdƒ‚)Nz/Collecting tensor data and making histogram ...>   r)   r^  r>  úDOnly 'entropy', 'percentile' or 'distribution' methods are supported)ÚprintrD  Úcollect_valuer•   Úcollect_absolute_valuer9   r`  r   r   r   rO  #  s   




zHistogramCollector.collectc                 C   sr  |  ¡ D �]1\}}t|tƒrB|D ]}t|tjƒs$J dt|ƒ› d|›�ƒ‚qdd„ |D ƒ}t|ƒdks<J d|› d|›�ƒ‚t |¡}nt|tjƒsTtdt|ƒ› d|›�ƒ‚|}| 	¡ }|j
dkrjt |¡}t |¡}ntjd|jd�}tjd|jd�}t |¡}|| jvr¨tj|| jd	�\}	}
|
 |j¡}
|jtjksžJ d
ƒ‚|	|
||f| j|< q| j| }|d }|d }t|dƒsÃJ dt|ƒ› �ƒ‚t|dƒsÑJ dt|ƒ› �ƒ‚|d }|d }t |¡}||d k�r|d |d  }t |d | || |¡}t ||f¡}tj||d	�\}	}
|
 |j¡}
|	dt|ƒ…  |7  < |jtjk�s'J d
ƒ‚|	|
t||ƒt||ƒf| j|< qdS )z5
        Collect histogram on absolute value
        r3   z for tensor=c                 S   r­   r   r   )rL   Úar   r   r   r¯   :  r°   z<HistogramCollector.collect_absolute_value.<locals>.<setcomp>r	   z6The calibration expects only one element type but got r   r   )r1   zMonly float32 or float16 is supported, every constant must be explicitly typedrY   r\   r   z'old_min should be a numpy array but is rÚ   N)r7   r_   r4   r   Úndarrayr<   re   r"   r9   ÚflattenÚsizer0  r1  rõ   r   Úabsoluterc  Ú	histogramrE  r#   Úfloat64r;   ÚarangeÚhstackr  r  )r?   ra  r  Údata_arrÚarrÚdtypesÚdata_arr_npr  r  r/   r0   Úold_histogramr  r  Úold_histÚold_hist_edgesÚ	temp_amaxÚwidthÚnew_bin_edgesr   r   r   rh  2  s\   
&ÿ



ÿ

ÿ Ìz)HistogramCollector.collect_absolute_valuec           
      C   sä   |  ¡ D ]k\}}t |¡}| ¡ }|jdkr!t |¡}t |¡}ntjd|jd�}tjd|jd�}tjt	t
|ƒt
|ƒƒ|jd�}|| jv rW| j| }|  |||||¡| j|< qtj|| j| |fd�\}}	||	|||f| j|< qdS )z1
        Collect histogram on real value
        r   r   ©rî   N)r7   r   r"   rk  rl  r0  r1  rõ   r   r  r2  rc  Úmerge_histogramrn  rE  )
r?   ra  r  rr  r  r  Ú	thresholdrv  r/   r0   r   r   r   rg  l  s,   





ÿûìz HistogramCollector.collect_valuec                 C   s  |\}}}}	}
||
kr)t j|t|ƒ|
 |
fd�\}}|| |t||ƒt|	|ƒ|
fS |
dkrAt j|t|ƒ| |fd�\}}||7 }n9t|ƒ}d|
 | }t||
 | d ƒ}|d|  }|| |
 }t j||| |fd�\}}|||| …  |7  < ||t||ƒt|	|ƒ|fS )Nr|  r   rY   r	   )r   rn  re   r  r  rŒ   )r?   rv  rr  r  r  Únew_thresholdrw  rx  r  r  Úold_thresholdÚnew_histrâ   r/   r0   Úold_num_binsÚ
old_strideÚhalf_increased_binsÚnew_num_binsr   r   r   r}  Œ  s2   û
ûz"HistogramCollector.merge_histogramc                 C   sl   | j r
t| j ƒdkrtdƒ‚td| j›d�ƒ | jdkr |  ¡ S | jdkr)|  ¡ S | jdkr2|  ¡ S tdƒ‚)	Nr   z=Histogram has not been collected. Please run collect() first.z0Finding optimal threshold for each tensor using z algorithm ...r)   r>  r^  re  )rc  re   r9   rf  rD  Úcompute_entropyÚcompute_percentileÚcompute_distributionrG   r   r   r   rW  ¬  s   


z,HistogramCollector.compute_collection_resultc                 C   sÐ  | j dk s
| j dkrtdƒ‚| j}| j }i }tdt|ƒ› �ƒ td| j› �ƒ tdd| › d|› d	�ƒ | ¡ D ]­\}}|d }|d
 }| ¡ }t 	|| ¡}	| j
rrt |	|d ¡}
tj||
 |jd� tj||
 |jd�f||< n*d| d }t |	d| ¡}
t |	|¡}tj|| |jd�tj||
 |jd�f||< |d }|d }|| d |k r¶||| d
 f||< || d
 |krÈ|| d |f||< g || ¢|d d… ¢R ||< tj dd¡dv råt||ƒ q8|S )Nr   éd   z<Invalid percentile. Must be in range 0 <= percentile <= 100.úNumber of tensors : úNumber of histogram bins : zPercentile : (g      Y@ú,ú)r	   r   g      i@r   rY   r\   ÚQUANTIZATION_DEBUGÚ0©r	   Ú1)r>  r9   rc  rf  re   rE  r7   r%   r   Úcumsumr•   Úsearchsortedrõ   r   ÚosÚenvironÚgetr
   )r?   rc  r>  Úthresholds_dictr  rn  r/   r0   ÚtotalÚcdfÚ	idx_rightÚpercent_to_cut_one_sideÚidx_leftr  r  r   r   r   r‡  º  sF   
þþ
€z%HistogramCollector.compute_percentilec                 C   s¬   | j }| j}i }tdt|ƒ› �ƒ td| j› d�ƒ td| j› �ƒ | ¡ D ]-\}}|  ||¡}|||< g |¢|d d… ¢R ||< tj 	dd¡dv rSt
|d	 |d
 ƒ q&|S )NrŠ  r‹  z: (The number may increase depends on the data it collects)zNumber of quantized bins : rY   rŽ  r�  r�  r   r	   )rc  rF  rf  re   rE  r7   Úget_entropy_thresholdr”  r•  r–  r
   )r?   rc  rF  r—  r  rn  Úoptimal_thresholdr   r   r   r†  è  s   €z"HistogramCollector.compute_entropyr	   c                 C   s   |dkrt d|› d�ƒ‚|d d… |dd …  d }|dkrJ| |  ¡ |  ¡  }| |d   ¡ |  ¡  |d  d }tj||jd�tj||jd�fS t|ƒ|kr†t|ƒd dkr†| ||   ¡ |  ¡  }| || | d   ¡ |  ¡  d }tj||jd�tj||jd�fS t |¡| }d|t |¡< d|t |¡< t |¡| | }| |  ¡ |  ¡  }| |d   ¡ |  ¡  |d  d }tj||jd�tj||jd�fS )	Nr   zpower=z <= 0 is invalid.rÚ   r	   g      à?rY   r   )	r9   r%   r   rõ   r   rŒ   r2  ÚisnanÚisinf)r/   r0   Úpowerrr   r+   r,   Úfactr   r   r   Ú_avg_stdý  s$   $ $ $ zHistogramCollector._avg_stdc           	   	   C   s:  | j dk r	tdƒ‚| j}i }tdt|ƒ› �ƒ td| j › �ƒ td| j›d�ƒ | ¡ D ]n\}}|d }|d }|jtj	ks@J ‚| jd	krP| j
||dd
�\}}n| jdkr`| j
||dd
�\}}ntdƒ‚|jtj	kslJ ‚|jtj	kstJ ‚|jtj	ks|J ‚t||||| ¡ | ¡ d�||< tj dd¡dv ršt||ƒ q,|S )Ni   z3Invalid num_bins. Must be in range 512 <= num_bins.rŠ  r‹  zScenario : r�  r   r	   rB  )r¡  Úp3gUUUUUUÕ?z,Invalid scenario. Must be in {'same', 'p3'}.)r+   r,   r/   r0   r-   r.   rŽ  r�  r�  )rE  r9   rc  rf  re   rG  r7   r   r   ro  r£  r*   r  r  r”  r•  r–  r
   )	r?   rc  r—  r  rn  r/   r0   Úavg_coefÚstd_coefr   r   r   rˆ    s>   



ú

€z'HistogramCollector.compute_distributionc                    sª  |d }|d }|j }|d }|d }|d j‰ t || d ¡}‡ fdd„t|j ƒD ƒ}	t||d dƒD ]Þ}
||
 }t||
 d |ƒ}|| || f|	|
| < t |||… ¡}| ¡ }t|d|… ƒ}t||d… ƒ}|d  |7  < |d  |7  < |dk 	tj
¡}tj|tj
d�}|j | }t|ƒD ]}|| }|| }t|||… ƒ||< q–|d  t||| d… ƒ7  < tj|j tj
d�}t|ƒD ] }|| }|| }t|||… ƒ}|dkrè|| | |||…< qÈt|ƒ}t|ƒ}|du sú|du �rtjtjˆ d�}n
tjt||ƒˆ d�}|||
| < q5t |¡}|	| }|d }|d	 }|d |k �r2||d f}|d |k�r?|d |f}t|d d
ƒ�sIJ ‚t|d d
ƒ�sSJ ‚|S )aF  Given a dataset, find the optimal threshold for quantizing it.
        The reference distribution is `q`, and the candidate distribution is `p`.
        `q` is a truncated version of the original distribution.
        Ref: http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf
        r   r	   rY   c                    s(   g | ]}t jd ˆ d�t jd ˆ d�f‘qS )r   r   )r   rõ   rè   r   r   r   r   I  s   ( z<HistogramCollector.get_entropy_threshold.<locals>.<listcomp>NrÚ   r   r\   r   )rl  r   r   Úzerosrî   r  rM  Údeepcopyr%   r#   rö   r   rõ   r   r)   Úargminr;   )r?   rn  rF  r/   r0   rE  Úzero_bin_indexÚnum_half_quantized_binÚkl_divergenceÚ
thresholdsré   r†   r‡   Úsliced_distributionÚpÚleft_outliers_countÚright_outliers_countÚnonzerosÚquantized_binsÚnum_merged_binsÚindexÚstartÚendÚqÚnormÚdivÚmin_kl_divergence_idxrž  r  r  r   r   r   r�  ;  sf   

 €
z(HistogramCollector.get_entropy_thresholdN)r	   )rP   rT   rU   rb  rC   rd  rO  rh  rg  r}  rW  r‡  r†  Ústaticmethodr£  rˆ  r�  r   r   r   r   rN    s    	:  .(rN  r�   Fr“   r‘   c                 C   sX  d }|t jkr3| dd¡}| dd¡}	| dd¡}
| dd ¡}| dd¡}t| |||||	|
||d�	}na|t jkrV| d	d
¡}| dd
¡}| dd¡}t| ||||||d�}n>|t jkry| d	d¡}| dd¡}| dd¡}t| ||||||d�}n|t jkr”| d	d¡}| dd¡}t	| |||||d�}|r¥| 
¡  |rŸ||_| ¡  |S td|› �ƒ‚)Nr•   FrÖ   r×   rÏ   rØ   r—   )r–   r•   rÖ   r×   rØ   r—   rE  r?  rF  )r–   r•   rE  rF  r@  r>  rA  T)r–   r•   rE  r>  rG  rB  )r–   rE  rG  zUnsupported calibration method )rc   rd   r–  rÎ   rt   rT  ru   rU  rv   rV  rÉ   rš   r›   r9   )r“   r‘   r”   Úcalibrate_methodr–   rž   Úextra_optionsÚ
calibratorr•   rÖ   r×   rØ   r—   rE  rF  r>  rG  r   r   r   Úcreate_calibrator–  s|   	
÷
ù
	ù

ú	rÀ  )Nr   ),rŠ   rM  r½   r”  rò   Úcollections.abcr   Úenumr   Úpathlibr   Únumpyr   rÜ   r   r   r   r   rŸ   Úquant_utilsr
   r   r   rj  r   ÚfloatrŒ   r)   r*   rX   rc   ÚABCMetarw   rŽ   rÎ   r=  rT  rU  rV  r_  rN  rd   r`   rÀ  r   r   r   r   Ú<module>   sn   üÿþýü
û#4n b ""#   ùÿ
þ