o
    §ôj]c  ã                   @   s¦  d Z ddlmZ ddlZddlmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ e e¡Zddddddddddddddœddddddddddddddœddddddddddddddœi dd“dd“dd“dd“dd“d d!“d"d“d#d$“d%d“d&d'“d(d)“d*d“d+d“d,d“d-d“d.d“d/d“d0di¥ddd1dd1ddd2dddd3œd4d5d6d7d8ddd9d:dd;d<œd=d>d7d8dd9d:d?d@dAœ	d=d>d7d8dBdCd9d:d?d@dDœ
ddddddddddddddœdEd6dFd;dGdHdIdJdKdLdMdNœdddLdMdddddddddOœdPœZdQdRdSœdTdUdVdWdXdYdZd[d\d]d^œ
dTdUdVdWdXdYdZd[d\d]d^œ
dTdUdVdWddYdZd[d\d]d^œ
dTdUdVdWddYdZd[d\d]d_d`daœdTdUdVdWddYdZd[d\d]d^œ
dbdcdddedfœdTdUdVdWddYdZd[d\d]d^œ
dgdWdhd]didjœdTdUdVdWddZd[dkd]dlœ	dgdmdndodhdidpœdUdTdWdVdZd[dqdrœdsœZdtdudvdwdxdbdcdddedydzœ
d{dQdbdcddded|œd}œZd~d„ ZG d€d�„ d�ƒZG d‚dƒ„ dƒeƒZG d„d…„ d…eƒZG d†d‡„ d‡eƒZG dˆd‰„ d‰eƒZ eeeee e e e e dŠœ	Z!d‹efdŒd�„Z"dS )Žz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizers)ÚBPEé   ©Ú
AddedToken)ÚGPT2ConverterÚLlamaConverterÚQwen2Converter)Úlogging)Útqdmzmodel.embed_tokenszmodel.layerszmlp.up_projzmlp.down_projzmlp.gate_projZpost_attention_layernormZinput_layernormzself_attn.q_projzself_attn.v_projzself_attn.k_projzself_attn.o_projzlm_head.weightz
model.norm)Ú
token_embdÚblkÚffn_upÚffn_downÚffn_gateÚffn_normÚ	attn_normÚattn_qÚattn_vÚattn_kÚattn_outputúoutput.weightÚoutput_normr   r   Zffn_up_expszmlp.expertsZffn_up_shexpzmlp.shared_expert.up_projZffn_down_expsZffn_down_shexpzmlp.shared_expert.down_projr   zffn_gate_inp.weightzmlp.gate.weightZffn_gate_expsZffn_gate_shexpzmlp.shared_expert.gate_projZffn_gate_inp_shexpzmlp.shared_expert_gater   r   r   r   r   r   r   zmlp.gate_up_projzself_attn.qkv_proj)r   r   r   r   r   r   r   Úattn_qkvr   r   r   z"transformer.word_embeddings.weightz%transformer.word_embeddings_layernormztransformer.hzmlp.dense_h_to_4hzmlp.dense_4h_to_hzself_attention.query_key_valuezself_attention.denseztransformer.ln_f)ztoken_embd.weightZtoken_embd_normr   r   r   r   r   r   r   r   r   Zword_embeddingsÚhz	.lm_head.Zln_f)	r   r   r   r   r   r   r   ú.output.r   z.ln_mlp.Zln_attn)
r   r   r   r   z.attn_norm.Zattn_norm_2r   r   r   r   ztransformer.wteztransformer.wpeZln_1zattn.c_attnzattn.c_proj.weightzattn.c_proj.biasZln_2zmlp.c_fcz
mlp.c_proj)r   r   Zposition_embdr   r   r   zattn_output.weightzattn_output.biasr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )ÚllamaÚmistralÚqwen2Úqwen2moeÚphi3ÚbloomZfalcon7bZ	falcon40bÚstablelmÚgpt2Ú
starcoder2Ú
model_typeZ_model_name_or_path)ÚarchitectureÚnameZmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimZ
rope_thetaÚnum_attention_headsZnum_key_value_headsZrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr1   Znum_expertsZnum_experts_per_tok)r2   r3   r4   r5   r6   r7   r8   r9   r:   r1   Zexpert_countZexpert_used_countÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚn_layerÚn_headÚlayer_norm_epsilon)r3   r5   r8   r1   úattention.layer_norm_epsilonÚlayer_norm_eps)	r2   r3   r4   r5   r6   r8   r9   rF   r1   Zn_ctxZn_embdr4   )r3   r2   r5   r4   r8   rF   Znorm_epsilon)r3   r2   r5   r4   r8   r9   rF   )Úgeneralr    r!   r"   r#   ÚfalconÚ	tokenizerr$   r%   r&   r'   r(   Útokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr?   r@   rA   rB   zggml.add_space_prefixÚchat_template)rR   rQ   r?   r@   rA   rB   )rJ   Útokenizer_configc                 C   sÆ   t |tƒs|g}t|ƒdkr|d }d }n|d dkrtdƒ‚|\}}|dv r/t| d ƒ} | S |dv r;t| d ƒ} | S |dv rGt| d ƒ} | S |dv rXtd	t| ƒƒ ¡  	¡ } | S |d
v rat
| |ƒ} | S )Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rT   r   é   é   é   é
   é   )é   é   )é   )é   ÚB)rU   )Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeZarray_data_type© rl   úo/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/transformers/integrations/ggml.pyri   X  s.   
	øúüþ
ri   c                   @   s   e Zd Zdd„ ZdS )ÚGGUFTokenizerSkeletonc                    s¢  |  ¡ D ]
\}}t| ||ƒ qt| dƒs’t| dƒrt| dƒs"tdƒ‚| j}| j‰ ‡ fdd„t|ƒD ƒ‰t d¡ g }t	ˆ  ¡ ƒD ]=\}}g }t
dt|ƒƒD ]}	|d |	… ||	d … }
}|
|v rl||v rl| |
||f¡ qMt|‡fd	d
„dd�}| |¡ q@t|dd
„ dd�}dd„ |D ƒ}|| _ndd„ | jD ƒ| _t| dƒs­dd„ t
t| jƒƒD ƒ| _t| dƒsµg | _t| dƒs½d | _t| dƒrÍ| jd u rÏ| j| _d S d S d S )NrO   rL   rM   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                    s   i | ]	\}}|ˆ | “qS rl   rl   )Ú.0ÚiÚt)rM   rl   rm   Ú
<dictcomp>|  ó    z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>z:Merges were not in checkpoint, building merges on the fly.rT   c                    s   ˆ | d  ˆ | d  fS )Nr   rT   rl   )Úx)Úvocabrl   rm   Ú<lambda>†  s    z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>T)ÚkeyÚreversec                 S   s   | d S )Nr   rl   )Úvalrl   rl   rm   rv   ˆ  s    c                 S   s   g | ]
}|d  |d f‘qS )r   rT   rl   )ro   ry   rl   rl   rm   Ú
<listcomp>‰  s    z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>c                 S   s   g | ]	}t | d ¡ƒ‘qS )ú )ÚtupleÚsplit)ro   Úmergerl   rl   rm   rz   Œ  rs   c                 S   s   g | ]}d ‘qS ©Nrl   )ro   Ú_rl   rl   rm   rz   Ž  s    Úadded_tokensr=   Úunknown_token_id)ÚitemsÚsetattrÚhasattrrc   rL   rM   Ú	enumerateÚloggerÚwarningr   Úrangerb   ÚappendÚsortedÚextendrO   r�   r=   r‚   )ÚselfÚdict_ÚkÚvrL   rO   r~   Úpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrl   )rM   ru   rm   Ú__init__q  sD   
ÿ
€


ÿzGGUFTokenizerSkeleton.__init__N)Ú__name__Ú
__module__Ú__qualname__r–   rl   rl   rl   rm   rn   p  s    rn   c                   @   s<   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )ÚGGUFLlamaConverterc                 C   s0   t |ƒ| _| j| _i | _t| jddƒdk| _d S )NrK   r    )rn   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©r�   Útokenizer_dictrl   rl   rm   r–   œ  s   
zGGUFLlamaConverter.__init__c                 C   ó   t t|j|jƒƒS r   ©ra   ÚziprL   rM   ©r�   r›   rl   rl   rm   ru   ¢  ó   zGGUFLlamaConverter.vocabc                 C   ó   |j S r   ©rO   r¥   rl   rl   rm   rO   ¥  ó   zGGUFLlamaConverter.mergesc                 C   sÎ  |   | j¡}|  | j¡}dd„ t|ƒD ƒ}|jd ur |j|j nd }t|dd ƒd ur0|j|j nd }t|dd ƒd ur@|j|j nd }tt	|||ddd�ƒ}g }	t
| jdƒs€|d urc|	 t|ddd	�¡ |d urq|	 t|ddd	�¡ |d ur|	 t|ddd	�¡ n!t t | jj¡d
k¡d }
|
D ]}|	 t| jj| ddd	�¡ q�t|	ƒdkr¬| |	¡ t| jjƒdkrÀ| dd„ | jjD ƒ¡ || jd< || jd< || jd< | jråd | jd< d| jd< d| jd< d| j_|S )Nc                 S   ó   i | ]	\}\}}||“qS rl   rl   ©ro   rp   ÚwordZ_scorerl   rl   rm   rr   «  rs   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>r;   r<   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackrN   F©Ú
normalizedÚspecialrV   r   c                 S   s   g | ]	}t |d d d�‘qS )Fr°   r	   )ro   Úadded_tokenrl   rl   rm   rz   Ò  rs   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>r­   Ú	eos_tokenÚ	bos_tokenrP   Úclean_up_tokenization_spacesÚlegacy)ru   r›   rO   r†   r=   rL   rž   r;   r   r   r…   rŠ   r
   ÚnpÚwherer   rN   rb   Úadd_special_tokensr�   Ú
add_tokensr�   rŸ   rœ   r·   )r�   r›   Úvocab_scoresrO   Ú	bpe_vocabr­   rµ   r´   rJ   Úspecial_tokensZspecial_tokens_idxÚidxrl   rl   rm   rJ   ¨  sT     ûÿ
€
ÿ





zGGUFLlamaConverter.tokenizerc                 C   sX   t  ¡ t  ¡ t  dd¡g}| jr|t jdddd�g7 }|r'|t jddd�g7 }t  |¡S )Nõ   â–�r{   FT©rP   Útrim_offsetsÚ	use_regexrT   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacerŸ   Ú	ByteLevelÚStripÚSequence©r�   ÚreplacementrP   Úsequencerl   rl   rm   Údecoderâ  s   
ý
zGGUFLlamaConverter.decoderc                 C   s¤   |   | j¡}|  | j¡}|d ur||_d}d}t| jdƒr!| jj}|  ||¡}|d ur.||_|  ||¡|_|  ¡ }|r>||_| j	rPt
jdddd�|_t g ¡|_|S )NrÀ   TrP   FrÁ   )rJ   r›   Ú
normalizerr…   rœ   rP   Úpre_tokenizerrÐ   Úpost_processorrŸ   r   rÊ   r   rÌ   )r�   rJ   rÑ   rÎ   rP   rÒ   rÓ   rl   rl   rm   Ú	convertedð  s*   ÿzGGUFLlamaConverter.convertedN)	r—   r˜   r™   r–   ru   rO   rJ   rÐ   rÔ   rl   rl   rl   rm   rš   ›  s    :rš   c                       ó*   e Zd Zdd„ Zdef‡ fdd„Z‡  ZS )ÚGGUFQwen2Converterc                 C   ó   t |ƒ| _i | _d S r   ©rn   rœ   r�   r    rl   rl   rm   r–     ó   

zGGUFQwen2Converter.__init__Úreturnc              	      s^   dd„ t | jjƒD ƒ}| jj}tƒ  ||¡}| tdddd�tdddd�tdddd�g¡ |S )	Nc                 S   ó   i | ]\}}||“qS rl   rl   ©ro   rp   r¬   rl   rl   rm   rr     ó    z0GGUFQwen2Converter.converted.<locals>.<dictcomp>ú<|endoftext|>FTr°   z<|im_start|>z
<|im_end|>)r†   rœ   rL   rO   ÚsuperrÔ   rº   r
   ©r�   ru   rO   rJ   ©Ú	__class__rl   rm   rÔ     s   ýÿzGGUFQwen2Converter.converted©r—   r˜   r™   r–   r   rÔ   Ú__classcell__rl   rl   rá   rm   rÖ     ó    rÖ   c                   @   sB   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdefdd„Z	dS )ÚGGUFPhi3Converterc                 C   s   t |ƒ| _| j| _i | _d S r   )rn   r›   rœ   r�   r    rl   rl   rm   r–   )  s   

zGGUFPhi3Converter.__init__c                 C   r¢   r   r£   r¥   rl   rl   rm   ru   .  r¦   zGGUFPhi3Converter.vocabc                 C   r§   r   r¨   r¥   rl   rl   rm   rO   1  r©   zGGUFPhi3Converter.mergesc                 C   sn  |   | j¡}|  | j¡}dd„ t|ƒD ƒ}tt||ƒƒ}| tdddddd�tdddd�td	dddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�g¡ |jd ur€|j	|j nd | j
d< |jd ur�|j	|j nd | j
d< |jd ur |j	|j nd | j
d< |jd ur°|j	|j nd | j
d< |S )Nc                 S   rª   rl   rl   r«   rl   rl   rm   rr   7  rs   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>z</s>TF)ÚrstripÚlstripr±   r²   rÞ   r°   z<|assistant|>)rç   r±   r²   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r­   r´   rµ   Ú	pad_token)ru   r›   rO   r†   r   r   rº   r
   r=   rL   r�   r<   r;   r>   )r�   r›   r¼   rO   r½   rJ   rl   rl   rm   rJ   4  s8   ôÿÿÿÿÿzGGUFPhi3Converter.tokenizerc                 C   s<   t  ¡ t  ¡ t  |d¡g}|r|t jddd�g7 }t  |¡S )Nr{   rT   rÄ   )r   rÇ   rÈ   rÉ   rË   rÌ   rÍ   rl   rl   rm   rÐ   [  s   
ý
zGGUFPhi3Converter.decoderrÚ   c                 C   s:   |   | j¡}d}d}t| jdƒr| jj}|  ||¡|_|S )NrÀ   TrP   )rJ   r›   r…   rœ   rP   rÐ   )r�   rJ   rÎ   rP   rl   rl   rm   rÔ   f  s   zGGUFPhi3Converter.convertedN)
r—   r˜   r™   r–   ru   rO   rJ   rÐ   r   rÔ   rl   rl   rl   rm   ræ   (  s    'ræ   c                       rÕ   )ÚGGUFGPTConverterc                 C   r×   r   rØ   r    rl   rl   rm   r–   t  rÙ   zGGUFGPTConverter.__init__rÚ   c                    s0   dd„ t | jjƒD ƒ}| jj}tƒ  ||¡}|S )Nc                 S   rÛ   rl   rl   rÜ   rl   rl   rm   rr   y  rÝ   z.GGUFGPTConverter.converted.<locals>.<dictcomp>)r†   rœ   rL   rO   rß   rÔ   rà   rá   rl   rm   rÔ   x  s   zGGUFGPTConverter.convertedrã   rl   rl   rá   rm   rê   s  rå   rê   )	r    r"   Ú	qwen2_moer$   r%   rI   r&   r'   r(   rÚ   c                 C   s"   | }t | |ƒ}| ¡ }||jfS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSrÔ   r�   )r*   r¡   Útokenizer_class_nameÚ	converterÚfast_tokenizerrl   rl   rm   Úconvert_gguf_tokenizerŒ  s   
rð   )#Ú__doc__r   Únumpyr¸   Ú
tokenizersr   r   r   r   Útokenizers.modelsr   Ú r
   Úconvert_slow_tokenizerr   r   r   Úutilsr   Zutils.loggingr   Ú
get_loggerr—   r‡   ÚGGUF_TENSOR_MAPPINGÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGri   rn   rš   rÖ   ræ   rê   rì   rð   rl   rl   rl   rm   Ú<module>   s€  
óóóÿþýüûúùø	÷
öõôóòñðïîõõ÷öóõô �ð #þõõöôöüöû÷ú	ù�~öúó+yK÷