o
    Fôj¯g  ã                   @   sø  d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ e e¡Zi ddddœ“ddddddddddddœ
“ddddddddddddœ
“ddddddddddddœ
“dddddddddddddd œ“d!dddddddddddœ
“d"d#d$d%d&d'œ“d(dddddddddddœ
“d)d*dd+dd,d-œ“d.d/d0d1d2d3d4dd,d5d6dd7œ“d8dddddddd9dd:œ	“d;d*d<d=d>d+d,d?œ“d@dddddddAdBœ“dCdddd,ddDdEdFddGœ	“dHdddddddddIddœ
“dJddddddddddddKœ“ZdLdMdNdOdPd#d$d%d&dQdRœ
dSdd#d$d%d&dTœdUœZdVdW„ ZG dXdY„ dYƒZG dZd[„ d[eƒZ G d\d]„ d]eƒZ!G d^d_„ d_eƒZ"G d`da„ daeƒZ#G dbdc„ dceƒZ$G ddde„ deeƒZ%e e!e!e"e#e#e#e#e#e$e#e#e%dfœZ&dgefdhdi„Z'dS )jz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   ©Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeZ_model_name_or_path)ÚarchitectureÚnameÚllamaZmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimZ
rope_thetaÚnum_attention_headsÚnum_key_value_headsZrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr   ÚmistralÚqwen2Úqwen2moeÚnum_expertsZnum_experts_per_tok)r    r!   r"   r#   r$   r%   r&   r'   r(   r   Zexpert_countZexpert_used_countÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomZn_layerÚn_headÚlayer_norm_epsilon)r!   r#   r&   r   úattention.layer_norm_epsilonÚt5Zn_positionsÚ
num_layersZd_ffÚd_modelZd_kvÚ	num_headsZrelative_attention_num_bucketsÚdecoder_start_token_id)r    r!   r"   r#   úattention.key_lengthr&   r'   r;   z attention.relative_buckets_countr@   r   ÚstablelmÚlayer_norm_eps)	r    r!   r"   r#   r$   r&   r'   r;   r   Úgpt2Zn_ctxZn_embdr"   )r!   r    r#   r"   r&   r;   Ú
starcoder2Znorm_epsilon)r!   r    r#   r"   r&   r'   r;   ÚmambaÚconv_kernelZ
state_sizeZtime_step_rank)	r   r    r#   r(   r!   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnemotronZnorm_epsÚgemma2)r    r!   r"   r#   r$   r%   rA   r&   r'   r(   r   Útokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr3   r4   r5   r6   zggml.add_space_prefixÚchat_template)rQ   rP   r3   r4   r5   r6   )r.   Útokenizer_configc                 C   sÆ   t |tƒs|g}t|ƒdkr|d }d }n|d dkrtdƒ‚|\}}|dv r/t| d ƒ} | S |dv r;t| d ƒ} | S |dv rGt| d ƒ} | S |dv rXtd	t| ƒƒ ¡  	¡ } | S |d
v rat
| |ƒ} | S )Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rS   r
   é   é   é   é
   é   )é   é   )é   )é   ÚB)rT   )Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeZarray_data_type© rk   úq/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/transformers/integrations/ggml.pyrh   ë   s.   
	øúüþ
rh   c                   @   s   e Zd Zdd„ ZdS )ÚGGUFTokenizerSkeletonc                    s¢  |  ¡ D ]
\}}t| ||ƒ qt| dƒs’t| dƒrt| dƒs"tdƒ‚| j}| j‰ ‡ fdd„t|ƒD ƒ‰t d¡ g }t	ˆ  ¡ ƒD ]=\}}g }t
dt|ƒƒD ]}	|d |	… ||	d … }
}|
|v rl||v rl| |
||f¡ qMt|‡fd	d
„dd�}| |¡ q@t|dd
„ dd�}dd„ |D ƒ}|| _ndd„ | jD ƒ| _t| dƒs­dd„ t
t| jƒƒD ƒ| _t| dƒsµg | _t| dƒs½d | _t| dƒrÍ| jd u rÏ| j| _d S d S d S )NrN   rK   rL   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.c                    s   i | ]	\}}|ˆ | “qS rk   rk   )Ú.0ÚiÚt)rL   rk   rl   Ú
<dictcomp>  ó    z2GGUFTokenizerSkeleton.__init__.<locals>.<dictcomp>z:Merges were not in checkpoint, building merges on the fly.rS   c                    s   ˆ | d  ˆ | d  fS )Nr   rS   rk   )Úx)Úvocabrk   rl   Ú<lambda>  s    z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>T)ÚkeyÚreversec                 S   s   | d S )Nr
   rk   )Úvalrk   rk   rl   ru     s    c                 S   s   g | ]
}|d  |d f‘qS )r   rS   rk   )rn   rx   rk   rk   rl   Ú
<listcomp>  s    z2GGUFTokenizerSkeleton.__init__.<locals>.<listcomp>c                 S   s   g | ]	}t | d ¡ƒ‘qS )ú )ÚtupleÚsplit)rn   Úmergerk   rk   rl   ry     rr   c                 S   s   g | ]}d ‘qS ©Nrk   )rn   Ú_rk   rk   rl   ry   !  s    Úadded_tokensr1   Úunknown_token_id)ÚitemsÚsetattrÚhasattrrb   rK   rL   Ú	enumerateÚloggerÚwarningr   Úrangera   ÚappendÚsortedÚextendrN   r€   r1   r�   )ÚselfÚdict_ÚkÚvrK   rN   r}   Zpiece_scoreÚlocalÚindexZpiece_lZpiece_rrk   )rL   rt   rl   Ú__init__  sD   
ÿ
€


ÿzGGUFTokenizerSkeleton.__init__N)Ú__name__Ú
__module__Ú__qualname__r’   rk   rk   rk   rl   rm     s    rm   c                   @   s<   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ ZdS )ÚGGUFLlamaConverterc                 C   s0   t |ƒ| _| j| _i | _t| jddƒdk| _d S )NrJ   r   )rm   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©rŒ   Útokenizer_dictrk   rk   rl   r’   /  s   
zGGUFLlamaConverter.__init__c                 C   ó   t t|j|jƒƒS r~   ©r`   ÚziprK   rL   ©rŒ   r—   rk   rk   rl   rt   5  ó   zGGUFLlamaConverter.vocabc                 C   ó   |j S r~   ©rN   r¡   rk   rk   rl   rN   8  ó   zGGUFLlamaConverter.mergesc                 C   sÎ  |   | j¡}|  | j¡}dd„ t|ƒD ƒ}|jd ur |j|j nd }t|dd ƒd ur0|j|j nd }t|dd ƒd ur@|j|j nd }tt	|||ddd�ƒ}g }	t
| jdƒs€|d urc|	 t|ddd	�¡ |d urq|	 t|ddd	�¡ |d ur|	 t|ddd	�¡ n!t t | jj¡d
k¡d }
|
D ]}|	 t| jj| ddd	�¡ q�t|	ƒdkr¬| |	¡ t| jjƒdkrÀ| dd„ | jjD ƒ¡ || jd< || jd< || jd< | jråd | jd< d| jd< d| jd< d| j_|S )Nc                 S   ó   i | ]	\}\}}||“qS rk   rk   ©rn   ro   ÚwordZ_scorerk   rk   rl   rq   >  rr   z0GGUFLlamaConverter.tokenizer.<locals>.<dictcomp>r/   r0   T)Ú	unk_tokenZfuse_unkÚbyte_fallbackrM   F©Ú
normalizedÚspecialrU   r   c                 S   s   g | ]	}t |d d d�‘qS )Fr«   r   )rn   Zadded_tokenrk   rk   rl   ry   e  rr   z0GGUFLlamaConverter.tokenizer.<locals>.<listcomp>r©   Ú	eos_tokenÚ	bos_tokenrO   Zclean_up_tokenization_spacesÚlegacy)rt   r—   rN   r…   r1   rK   rš   r/   r   r   r„   r‰   r   ÚnpÚwherer   rM   ra   Úadd_special_tokensr€   Z
add_tokensr™   r›   r˜   r°   )rŒ   r—   Úvocab_scoresrN   Ú	bpe_vocabr©   r¯   r®   r.   Úspecial_tokensZspecial_tokens_idxÚidxrk   rk   rl   r.   ;  sT     ûÿ
€
ÿ





zGGUFLlamaConverter.tokenizerc                 C   sX   t  ¡ t  ¡ t  dd¡g}| jr|t jdddd�g7 }|r'|t jddd�g7 }t  |¡S )Nõ   â–�rz   FT©rO   Ztrim_offsetsZ	use_regexrS   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacer›   Ú	ByteLevelÚStripÚSequence©rŒ   ÚreplacementrO   Úsequencerk   rk   rl   Údecoderu  s   
ý
zGGUFLlamaConverter.decoderc                 C   s¤   |   | j¡}|  | j¡}|d ur||_d}d}t| jdƒr!| jj}|  ||¡}|d ur.||_|  ||¡|_|  ¡ }|r>||_| j	rPt
jdddd�|_t g ¡|_|S )Nr¸   TrO   Fr¹   )r.   r—   Ú
normalizerr„   r˜   rO   Úpre_tokenizerrÆ   Úpost_processorr›   r   rÀ   r   rÂ   )rŒ   r.   rÇ   rÄ   rO   rÈ   rÉ   rk   rk   rl   Ú	convertedƒ  s*   ÿzGGUFLlamaConverter.convertedN)	r“   r”   r•   r’   rt   rN   r.   rÆ   rÊ   rk   rk   rk   rl   r–   .  s    :r–   c                       ó*   e Zd Zdd„ Zdef‡ fdd„Z‡  ZS )ÚGGUFQwen2Converterc                 C   ó   t |ƒ| _i | _d S r~   ©rm   r˜   r™   rœ   rk   rk   rl   r’   ¨  ó   

zGGUFQwen2Converter.__init__Úreturnc              	      s^   dd„ t | jjƒD ƒ}| jj}tƒ  ||¡}| tdddd�tdddd�tdddd�g¡ |S )	Nc                 S   ó   i | ]\}}||“qS rk   rk   ©rn   ro   r¨   rk   rk   rl   rq   ­  ó    z0GGUFQwen2Converter.converted.<locals>.<dictcomp>ú<|endoftext|>FTr«   z<|im_start|>z
<|im_end|>)r…   r˜   rK   rN   ÚsuperrÊ   r³   r   ©rŒ   rt   rN   r.   ©Ú	__class__rk   rl   rÊ   ¬  s   ýÿzGGUFQwen2Converter.converted©r“   r”   r•   r’   r   rÊ   Ú__classcell__rk   rk   r×   rl   rÌ   §  ó    rÌ   c                   @   sB   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdefdd„Z	dS )ÚGGUFPhi3Converterc                 C   s   t |ƒ| _| j| _i | _d S r~   ©rm   r—   r˜   r™   rœ   rk   rk   rl   r’   ¼  s   

zGGUFPhi3Converter.__init__c                 C   rž   r~   rŸ   r¡   rk   rk   rl   rt   Á  r¢   zGGUFPhi3Converter.vocabc                 C   r£   r~   r¤   r¡   rk   rk   rl   rN   Ä  r¥   zGGUFPhi3Converter.mergesc                 C   sn  |   | j¡}|  | j¡}dd„ t|ƒD ƒ}tt||ƒƒ}| tdddddd�tdddd�td	dddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�tddddd
�g¡ |jd ur€|j	|j nd | j
d< |jd ur�|j	|j nd | j
d< |jd ur |j	|j nd | j
d< |jd ur°|j	|j nd | j
d< |S )Nc                 S   r¦   rk   rk   r§   rk   rk   rl   rq   Ê  rr   z/GGUFPhi3Converter.tokenizer.<locals>.<dictcomp>ú</s>TF)ÚrstripÚlstripr¬   r­   rÔ   r«   z<|assistant|>)rß   r¬   r­   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r©   r®   r¯   Z	pad_token)rt   r—   rN   r…   r   r   r³   r   r1   rK   r™   r0   r/   r2   )rŒ   r—   r´   rN   rµ   r.   rk   rk   rl   r.   Ç  s8   ôÿÿÿÿÿzGGUFPhi3Converter.tokenizerc                 C   s<   t  ¡ t  ¡ t  |d¡g}|r|t jddd�g7 }t  |¡S )Nrz   rS   rº   )r   r½   r¾   r¿   rÁ   rÂ   rÃ   rk   rk   rl   rÆ   î  s   
ý
zGGUFPhi3Converter.decoderrÐ   c                 C   s:   |   | j¡}d}d}t| jdƒr| jj}|  ||¡|_|S )Nr¸   TrO   )r.   r—   r„   r˜   rO   rÆ   )rŒ   r.   rÄ   rO   rk   rk   rl   rÊ   ù  s   zGGUFPhi3Converter.convertedN)
r“   r”   r•   r’   rt   rN   r.   rÆ   r   rÊ   rk   rk   rk   rl   rÜ   »  s    'rÜ   c                       rË   )ÚGGUFGPTConverterc                 C   rÍ   r~   rÎ   rœ   rk   rk   rl   r’     rÏ   zGGUFGPTConverter.__init__rÐ   c                    s0   dd„ t | jjƒD ƒ}| jj}tƒ  ||¡}|S )Nc                 S   rÑ   rk   rk   rÒ   rk   rk   rl   rq     rÓ   z.GGUFGPTConverter.converted.<locals>.<dictcomp>)r…   r˜   rK   rN   rÕ   rÊ   rÖ   r×   rk   rl   rÊ     s   zGGUFGPTConverter.convertedrÙ   rk   rk   r×   rl   rá     rÛ   rá   c                   @   ó:   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	efd
d„ZdS )ÚGGUFT5Converterc                 C   s>   dg|d< t |ƒ| _dd„ t| jjƒD ƒ| _| j| _i | _d S )Nú
dummy textrN   c                 S   rÑ   rk   rk   )rn   r�   rŽ   rk   rk   rl   rq     rÓ   z,GGUFT5Converter.__init__.<locals>.<dictcomp>)rm   r—   r…   rK   Útoken2idr˜   r™   rœ   rk   rk   rl   r’     s
   


zGGUFT5Converter.__init__c                 C   rž   r~   rŸ   r¡   rk   rk   rl   rt     r¢   zGGUFT5Converter.vocabc                 C   sT   t | jddƒr(g }t | jddƒr|tjdd�g7 }|tjddd�g7 }t |¡S d S )Nr°   TrO   r¸   )Úprependrz   )Úpatternr»   )rš   r˜   r   ZPrependr¿   rÂ   )rŒ   r—   rÅ   rk   rk   rl   rÇ     s   
zGGUFT5Converter.normalizerc                 C   s$   t jddgg d¢d| jd fgd�S )Nú$ArÞ   )rè   rÞ   z$BrÞ   )ÚsingleÚpairr¶   )r   ZTemplateProcessingrå   )rŒ   rk   rk   rl   rÉ   (  s   ÿýzGGUFT5Converter.post_processorrÐ   c                 C   s–   |   | j¡}tt|| jjdd�ƒ}|  | j¡}|d ur||_d}d}t| jdƒr,| jj}|  	||¡}|d ur9||_	|  
||¡|_
|  ¡ }|rI||_|S )NF©Úunk_idrª   r¸   TrO   )rt   r—   r   r	   r1   rÇ   r„   r˜   rO   rÈ   rÆ   rÉ   )rŒ   r´   r.   rÇ   rÄ   rO   rÈ   rÉ   rk   rk   rl   rÊ   1  s.   ýÿ	zGGUFT5Converter.convertedN)	r“   r”   r•   r’   rt   rÇ   rÉ   r   rÊ   rk   rk   rk   rl   rã     s    			rã   c                   @   râ   )ÚGGUFGemmaConverterc                 C   s&   dg|d< t |ƒ| _| j| _i | _d S )Nrä   rN   rÝ   rœ   rk   rk   rl   r’   R  s   


zGGUFGemmaConverter.__init__c                 C   s‚   t t|j|jƒƒ}g }|D ]1\}}|dkr| d|f¡ qd|v r7t| ¡ ƒdkr7dt|ƒ }| ||f¡ q| ||f¡ q|S )Nz<0x09>ú	rz   r   r¸   )r`   r    rK   rL   r‰   ra   Ústrip)rŒ   r—   Zoriginal_vocabZupdated_vocabÚtokenÚscoreÚunderscoresrk   rk   rl   rt   Z  s   zGGUFGemmaConverter.vocabc                 C   s   t  dd¡S )Nrz   r¸   )r   r¿   r¡   rk   rk   rl   rÇ   i  s   zGGUFGemmaConverter.normalizerc                 C   s<   t  dd¡t  ¡ t  ¡ g}|r|t jddd�g7 }t  |¡S )Nr¸   rz   rS   rº   )r   r¿   r½   r¾   rÁ   rÂ   rÃ   rk   rk   rl   rÆ   l  s   
ý
zGGUFGemmaConverter.decoderrÐ   c                 C   s†   |   | j¡}tt|| jj| jd�ƒ}|  | j¡}|d ur||_d}d}t| jdƒr-| jj	}|  
||¡|_
|  ||¡}|d urA||_|S )Nrë   r¸   TrO   )rt   r—   r   r	   r1   Zhandle_byte_fallbackrÇ   r„   r˜   rO   rÆ   rÈ   )rŒ   r´   r.   rÇ   rÄ   rO   rÈ   rk   rk   rl   rÊ   w  s(   ýÿzGGUFGemmaConverter.convertedN)	r“   r”   r•   r’   rt   rÇ   rÆ   r   rÊ   rk   rk   rk   rl   rí   Q  s    rí   )r   r*   Ú	qwen2_moer7   r8   r-   rB   rD   rE   r<   rF   rH   rI   rÐ   c                 C   s"   | }t | |ƒ}| ¡ }||jfS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSrÊ   r™   )r   r�   Ztokenizer_class_nameÚ	converterZfast_tokenizerrk   rk   rl   Úconvert_gguf_tokenizer£  s   
rö   )(Ú__doc__r   Únumpyr±   Ú
tokenizersr   r   r   r   r   Ztokenizers.modelsr   r	   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerr“   r†   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGrh   rm   r–   rÌ   rÜ   rá   rã   rí   rô   rö   rk   rk   rk   rl   Ú<module>   sè  
þÿõûõîöá+ôÕ9öÇEü»KöµWû©^õ¢k÷•vúŠ~ù‚ ÷ �ø ö �í ó �á 2öúó+yK?Bó