o
    Íôj‰  ã                   @   st   d dl mZmZmZmZmZmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlmZ ddlmZ G dd„ deƒZd	S )
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚNFKCé   )ÚBaseTokenizerc                       sV  e Zd ZdZ							d$deeeeeef f  deeee	e
eef  f  d	eeef d
ededee dee f‡ fdd„Zededefdd„ƒZdddgdg dfdeee	e f dedede	eeef  dede	e defdd„Zdddgdg ddfd eee eee  f dedede	eeef  dede	e ded!ee fd"d#„Z‡  ZS )%ÚSentencePieceBPETokenizerzrSentencePiece BPE Tokenizer

    Represents the BPE algorithm, with the pretokenization used by SentencePiece
    Nú<unk>õ   â–�TFÚvocabÚmergesÚ	unk_tokenÚreplacementÚadd_prefix_spaceÚdropoutÚfuse_unkc                    s²   |d ur|d urt t|||||d�ƒ}n	t t|||d�ƒ}| t|ƒ¡d ur.| t|ƒg¡ tƒ |_|r6dnd}	tj||	d�|_	t
j||	d�|_d||||dœ}
tƒ  ||
¡ d S )N)r   r   r   ÚalwaysÚnever)r   Úprepend_schemeZSentencePieceBPE)Úmodelr   r   r   r   )r	   r   Útoken_to_idÚstrÚadd_special_tokensr   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr
   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   Ú	tokenizerr   Ú
parameters©Ú	__class__© ú/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/tokenizers/implementations/sentencepiece_bpe.pyr'      s    
ûz"SentencePieceBPETokenizer.__init__Úvocab_filenameÚmerges_filenamec                 K   s"   t  | |¡\}}t||fi |¤ŽS )N)r   Ú	read_filer   )r/   r0   Úkwargsr   r   r-   r-   r.   Ú	from_file1   s   z#SentencePieceBPETokenizer.from_filei0u  é   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc           	      C   s:   t j||||||d�}t|tƒr|g}| jj||d� dS )z%Train the model using the given files©r6   r7   r8   r9   r:   r;   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer    Ú
_tokenizerÚtrain)	r(   r5   r6   r7   r8   r9   r:   r;   r=   r-   r-   r.   rA   6   s   ú
zSentencePieceBPETokenizer.trainÚiteratorÚlengthc	           
      C   s,   t j||||||d�}	| jj||	|d� dS )z(Train the model using the given iteratorr<   )r=   rC   N)r   r>   r@   Útrain_from_iterator)
r(   rB   r6   r7   r8   r9   r:   r;   rC   r=   r-   r-   r.   rD   N   s   ú
ýz-SentencePieceBPETokenizer.train_from_iterator)NNr   r   TNF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r    r   Úintr   r   r   ÚboolÚfloatr'   Ústaticmethodr3   rA   r   rD   Ú__classcell__r-   r-   r+   r.   r   
   sŽ    øþý
üûúùø!øþýüûúù
ø÷þýüûúùø	÷r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr   r	   r
   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   Úbase_tokenizerr   r   r-   r-   r-   r.   Ú<module>   s     