o
    LôjÒŠ  ã                   @   sô  d dl Z d dlZd dlZd dlZd dlZddlmZ dd„ Zda	da
dd„ Zdadad	d
„ Zdd„ Zdd„ ZedededddddddddiedededddddddddidœZededdd dd!ied"ed#dd$dd%ied&ed'dd(dd!ied)ed*dd+dd%id,œZejZejZejZejZejZeZeZeZeZ							d<d-d.„ZeZ G d/d0„ d0ƒZ!G d1d2„ d2ƒZ"G d3d4„ d4ƒZ#G d5d6„ d6ƒZ$d7d8„ Z%e!j&e!_'e!j(e!_)e%e!ƒ e%e#ƒ e%e$ƒ dd9l*m+Z+ zee j, -eej. /d:¡ƒd;¡ƒ W dS  e0yù   Y dS w )=é    Né   )Ú_sentencepiecec                 C   s   t | dƒot| ttfƒ S )NÚ__len__)ÚhasattrÚ
isinstanceÚstrÚbytes)Úx© r
   úg/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/sentencepiece/__init__.pyÚ_is_sequence   s   r   Fc               
   C   sH   t s"z
ddlm}  | aW n ty } ztdƒ|‚d }~ww da d S d S )Nr   )Úsentencepiece_pb2z_protobuf is required when using out_type='proto'. Please install protobuf: pip install protobufT)Ú_protobuf_loadedÚsentencepiecer   Ú_sentencepiece_pb2ÚImportError)r   Úer
   r
   r   Ú_load_protobuf   s   ÿý€ÿ÷r   c               
   C   s@   t szdd l} | aW n ty } ztdƒ|‚d }~ww da tS )Nr   zSnumpy is required when return_type='numpy'. Please install numpy: pip install numpyT)Ú_numpy_loadedÚnumpyÚ_numpyr   )Únpr   r
   r
   r   Ú_load_numpy   s   ÿý€ÿr   c                 C   ó   t ƒ  t ¡ }| | ¡ |S ©N)r   r   ZSentencePieceTextÚParseFromString)Ú
serializedZsptr
   r
   r   Ú_to_spt,   ó   
r   c                 C   r   r   )r   r   ZNBestSentencePieceTextr   )r   Z	nbest_sptr
   r
   r   Ú_to_nbest_spt2   r   r   Z_EncodeAsIdsZ_EncodeAsPiecesÚserialized_protoZ_EncodeAsSerializedProtoÚprotoÚoffset_mappingZ_EncodeAsOffsetMappingr   Z_EncodeAsBufferZ_EncodeAsIdsBatchZ_EncodeAsPiecesBatchZ_EncodeAsSerializedProtoBatchZ_EncodeAsOffsetMappingBatchZ_EncodeAsBufferBatch©FTZ
_DecodeIdsZ_DecodeIdsAsBytesZ_DecodeIdsAsSerializedProtoZ_DecodeAsOffsetMappingZ_DecodeIdsBatchZ_DecodeIdsAsBytesBatchZ _DecodeIdsAsSerializedProtoBatchZ_DecodeAsOffsetMappingBatchZ_DecodePiecesZ_DecodePiecesAsBytesZ_DecodePiecesAsSerializedProtoZ_DecodePiecesBatchZ_DecodePiecesAsBytesBatchZ#_DecodePiecesAsSerializedProtoBatch))FFr#   )TF)TTc                 C   sú   t dd„ | ||fD ƒƒ}|dkrtdƒ‚d}|dur(t t |¡¡}|s(tdƒ‚| durIt t | ¡¡}	|	s:tdƒ‚t |	|t|ƒt|ƒt|ƒ¡S |durct|d	ƒrW| 	¡ }
nt
|tƒr_|}
ntd
ƒ‚t
|tƒsltdƒ‚|}
t |
|t|ƒt|ƒt|ƒ¡S )aO  Converts a SentencePiece ModelProto into a FlatBuffers (.spm.fb) model.

    Note: WORD and CHAR models are not natively supported in FlatBuffers
    (.spm.fb) and raise a ValueError by default. Even when
    `allow_unsupported_model_type=True` is specified, they fall back to
    UNIGRAM at runtime in SentencePieceLiteProcessor.
    c                 s   s   � | ]}|d uV  qd S r   r
   ©Ú.0r	   r
   r
   r   Ú	<genexpr>š   s   € 
ÿz&ConvertToFlatbuffer.<locals>.<genexpr>r   zTExactly one of model_file, model_proto, or serialized_model_proto must be specified.Ú Nzoutput_file must not be empty.zmodel_file must not be empty.ÚSerializeToStringz1model_proto must be a ModelProto object or bytes.z%serialized_model_proto must be bytes.)ÚsumÚ
ValueErrorÚosÚfsdecodeÚfspathr   Z_ConvertFileToFlatbufferÚboolr   r(   r   r   Ú	TypeErrorZ#_ConvertSerializedProtoToFlatbuffer)Ú
model_fileÚmodel_protoÚserialized_model_protoÚoutput_fileZskip_char_bigramsZtreat_null_byte_as_unusedZallow_unsupported_model_typeÚ
num_inputsZout_pathÚ
model_pathr   r
   r
   r   ÚConvertToFlatbufferŠ   sN   ÿÿû



ûr6   c                   @   s*  e Zd Z												dkdd„Zdldd„Zed	d
„ ƒZedd„ ƒZdmdd„Zdd„ Z	dd„ Z
												dndd„Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdmdd „Zdmd!d"„Zdmd#d$„Zdmd%d&„Z							dod'd(„Zdld)d*„Zdld+d,„Zdld-d.„Zdld/d0„Z									dpd1d2„Zd3d4„ Zd5d6„ Zd7d8„ Zd9d:„ Zdqd;d<„Zdrd=d>„Z dld?d@„Z!dldAdB„Z"dCdD„ Z#dEdF„ Z$dldGdH„Z%dIdJ„ Z&dKdL„ Z'dMdN„ Z(dOdP„ Z)dQdR„ Z*dSdT„ Z+dUdV„ Z,dWdX„ Z-dYdZ„ Z.d[d\„ Z/d]d^„ Z0d_d`„ Z1dadb„ Z2e&Z3e&Z4dcdd„ Z5dedf„ Z6dgdh„ Z7didj„ Z8dS )sÚSentencePieceProcessorNFéÿÿÿÿçš™™™™™¹?c                 C   sj   t  ¡ | _| j||td�| _|| _|| _|| _|| _	|| _
|	| _|
| _|| _|s*|r3| j||d� d S d S )N©Údefault)r0   r1   )r   r7   Ú
_processorÚ_resolve_return_typeÚintÚ_return_typeÚ_add_bosÚ_add_eosÚ_reverseÚ_emit_unk_pieceÚ_enable_samplingÚ_nbest_sizeÚ_alphaÚ_num_threadsÚLoad)Úselfr0   r1   Úout_typeÚadd_bosÚadd_eosÚreverseÚemit_unk_pieceÚenable_samplingÚ
nbest_sizeÚalphaÚnum_threadsÚreturn_typer
   r
   r   Ú__init__Ì   s   
ÿzSentencePieceProcessor.__init__c                 C   st   |d ur|d urt dƒ‚|}|d u r|d ur|n| j}|dkr#t dƒ‚tttddddf}||vr8t d ||¡ƒ‚|S )	NzFCannot specify both out_type and return_type. Use return_type instead.Zimmutable_protozWimmutable_proto is deprecated and no longer supported. Use return_type='proto' instead.r!   r    r   r"   z*Invalid return_type: {}. Must be one of {})r*   r?   r>   r   r   Úformat)rI   rS   rJ   r;   Úvalid_typesr
   r
   r   r=   è   s   z+SentencePieceProcessor._resolve_return_typec                 K   ó4   d|v sd|v rt dƒ‚| di |¤Ž}| |¡ |S )zCCreates a SentencePieceProcessor by loading a model file from disk.r0   r1   zXmodel_file and model_proto cannot be passed via kwargs when using from_file classmethod.Nr
   )r*   ÚLoadFromFile)Úclsr0   ÚkwargsÚ	processorr
   r
   r   Ú	from_fileö   ó
   
z SentencePieceProcessor.from_filec                 K   rW   )zQCreates a SentencePieceProcessor by loading a serialized model proto from memory.r0   r1   zYmodel_file and model_proto cannot be passed via kwargs when using from_proto classmethod.Nr
   )r*   ÚLoadFromSerializedProto)rY   r1   rZ   r[   r
   r
   r   Ú
from_protoÿ   r]   z!SentencePieceProcessor.from_protoc                 C   s4   |r|rt dƒ‚|r|  |¡S |r|  |¡S t dƒ‚)Nz-model_file and model_proto must be exclusive.z3Either model_file or model_proto must be specified.)r*   r^   rX   )rI   r0   r1   r
   r
   r   rH     s   

zSentencePieceProcessor.Loadc                 C   ó   | j  |¡S r   )r<   rX   )rI   Úfilenamer
   r
   r   rX     ó   z#SentencePieceProcessor.LoadFromFilec                 C   s   t |dƒr	| ¡ }| j |¡S )Nr(   )r   r(   r<   r^   )rI   r   r
   r
   r   r^     s   
z.SentencePieceProcessor.LoadFromSerializedProtoc              	      s@  |   ||¡}|d ur|dkrtdƒ‚|d u r| j}|d u r | j}|d u r'| j}|d u r.| j}|d u r5| j}|d u r<| j}|d u rC| j}|	d u rJ| j	}	|du rb|d u s^|dks^|dks^|d u rbtdƒ‚|	d u skt
|	tƒsotdƒ‚t
|tƒ}t |¡}|r||vr†td |¡ƒ‚|| }t| j|ƒ}|ttfv }|p›|d	v }|rª||	|
||||||g	}n	|||||||g}|rº| |¡ |rÆ|tu }| |¡ n%|dkrë|d u rät
|tƒpâtt
|tƒoá|oát
|d tƒƒ}n|}| |¡ ||Ž }|d
k�rtƒ ‰ |�r‡ fdd„|D ƒS ˆ j|ˆ jd�S |dk�r|�rdd„ |D ƒS t|ƒS |S )Nr"   ú@return_bytes is only supported when return_type='offset_mapping'Tr   r   a  When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. when "nbest_size = -1" , this method samples from all candidates on the lattice instead of nbest segmentations.únum_threads must be intúunknown return_type={})r    r!   r"   r   c                    ó   g | ]
}ˆ j |ˆ jd �‘qS ©©Údtype©Ú
frombufferÚint32©r%   Úbuf©r   r
   r   Ú
<listcomp>s  ó    z1SentencePieceProcessor.Encode.<locals>.<listcomp>rh   r!   c                 S   ó   g | ]}t |ƒ‘qS r
   ©r   ©r%   Úsr
   r
   r   rp   x  ó    )r=   r*   r@   rA   rB   rC   rD   rE   rF   rG   r   r>   r/   ÚlistÚ_ENCODE_DISPATCHÚgetrU   Úgetattrr<   r   r   Úappendr.   r   rk   rl   r   )rI   ÚinputrK   rL   rM   rN   rO   rP   rQ   rR   Úthread_poolrS   rJ   Úreturn_bytesÚis_batchÚmethodsÚmethod_nameÚmethodZis_pieces_methodZhas_unkÚargsÚreturn_bytes_valZraw_resr
   ro   r   ÚEncode  sx   ÿ


ÿ


zSentencePieceProcessor.Encodec                 K   ó   | j d|tdœ|¤ŽS ©N©r|   rS   r
   ©r…   r   ©rI   r|   rZ   r
   r
   r   ÚEncodeAsPieces}  ó   z%SentencePieceProcessor.EncodeAsPiecesc                 K   r†   r‡   ©r…   r>   rŠ   r
   r
   r   ÚEncodeAsIds€  rŒ   z"SentencePieceProcessor.EncodeAsIdsc                 K   ó   | j d|ddœ|¤ŽS ©Nr   rˆ   r
   ©r…   rŠ   r
   r
   r   ÚEncodeAsNumpyƒ  rŒ   z$SentencePieceProcessor.EncodeAsNumpyc                 K   r�   ©Nr!   rˆ   r
   r‘   rŠ   r
   r
   r   ÚEncodeAsProto†  rŒ   z$SentencePieceProcessor.EncodeAsProtoc                 K   r�   )Nr"   rˆ   r
   r‘   rŠ   r
   r
   r   ÚEncodeAsOffsetMapping‰  rŒ   z,SentencePieceProcessor.EncodeAsOffsetMappingc                 K   ó   | j d|||tddœ|¤ŽS ©NT©r|   rP   rQ   rS   rO   r
   r‰   ©rI   r|   rP   rQ   rZ   r
   r
   r   ÚSampleEncodeAsPiecesŒ  ó
   ÿÿz+SentencePieceProcessor.SampleEncodeAsPiecesc                 K   r–   r—   r�   r™   r
   r
   r   ÚSampleEncodeAsIds�  r›   z(SentencePieceProcessor.SampleEncodeAsIdsc                 K   ó   | j d|||dddœ|¤ŽS )Nr   Tr˜   r
   r‘   r™   r
   r
   r   ÚSampleEncodeAsNumpy”  r›   z*SentencePieceProcessor.SampleEncodeAsNumpyc                 K   r�   )Nr!   Tr˜   r
   r‘   r™   r
   r
   r   ÚSampleEncodeAsProto˜  r›   z*SentencePieceProcessor.SampleEncodeAsProtoc	           	         sâ   ˆ	  ˆ|¡‰ˆd u rˆ	j‰ˆd u rˆ	j‰ˆd u rˆ	j‰ˆd u r"ˆ	j‰ˆd u r)ˆ	j‰ˆdkr/d‰ˆdkrStƒ ‰‡‡‡‡‡‡	fdd„‰t|tƒrO‡fdd„|D ƒS ˆ|ƒS ‡‡‡‡‡‡‡	fdd	„‰ t|tƒrm‡ fd
d„|D ƒS ˆ |ƒS )Nr   r   r   c                    s&   ˆj  | ˆˆ ˆˆ¡}‡fdd„|D ƒS )Nc                    rf   rg   rj   rm   ro   r
   r   rp   º  rq   zJSentencePieceProcessor.NBestEncode.<locals>._encode_np.<locals>.<listcomp>)r<   Z_NBestEncodeAsBuffer)ÚtextZraw_buffers)rK   rL   rP   r   rM   rI   r
   r   Ú
_encode_np·  s   
ÿz6SentencePieceProcessor.NBestEncode.<locals>._encode_npc                    ó   g | ]}ˆ |ƒ‘qS r
   r
   ©r%   Ún©r¡   r
   r   rp   ½  rv   z6SentencePieceProcessor.NBestEncode.<locals>.<listcomp>c              	      sž   ˆt u rˆj | ˆˆ ˆˆ¡S ˆtu sˆtu r&ˆtu }ˆj | ˆˆ ˆˆˆ|¡S ˆdkr5ˆj | ˆˆ ˆˆˆ¡S ˆdkrHˆj | ˆˆ ˆˆˆ¡}t|ƒS td 	ˆ¡ƒ‚©Nr    r!   re   )
r>   r<   Z_NBestEncodeAsIdsr   r   Z_NBestEncodeAsPiecesZ_NBestEncodeAsSerializedProtor   r*   rU   ©r    r~   r   )rK   rL   rN   rP   rS   rM   rI   r
   r   Ú_encodeÀ  s&   
ÿÿÿÿz3SentencePieceProcessor.NBestEncode.<locals>._encodec                    r¢   r
   r
   r£   ©r¨   r
   r   rp   Ò  rv   )	r=   r@   rA   rB   rC   rE   r   r   rw   )	rI   r|   rK   rL   rM   rN   rP   rS   rJ   r
   )
r¨   r¡   rK   rL   rN   rP   r   rS   rM   rI   r   ÚNBestEncodeœ  s.   	

z"SentencePieceProcessor.NBestEncodec                 K   ó   | j d||tdœ|¤ŽS ©N©r|   rP   rS   r
   )rª   r   ©rI   r|   rP   rZ   r
   r
   r   ÚNBestEncodeAsPiecesÕ  ó   z*SentencePieceProcessor.NBestEncodeAsPiecesc                 K   r«   r¬   )rª   r>   r®   r
   r
   r   ÚNBestEncodeAsIdsØ  r°   z'SentencePieceProcessor.NBestEncodeAsIdsc                 K   ó   | j d||ddœ|¤ŽS )Nr   r­   r
   ©rª   r®   r
   r
   r   ÚNBestEncodeAsNumpyÛ  r°   z)SentencePieceProcessor.NBestEncodeAsNumpyc                 K   r²   )Nr!   r­   r
   r³   r®   r
   r
   r   ÚNBestEncodeAsProtoÞ  r°   z)SentencePieceProcessor.NBestEncodeAsProtoc              	      s  ˆ
  ˆ|
¡‰ˆd u rˆ
j‰ˆd u rˆ
j‰ˆ	d u rˆ
j‰	ˆd u r"ˆ
j‰ˆd u r)ˆ
j‰ˆd u s2tˆtƒs6tdƒ‚ˆd u sCtˆtƒrCˆdkrGt	dƒ‚ˆdkrmt
ƒ ‰‡‡‡‡‡‡	‡
‡fdd„‰t|tƒri‡fdd„|D ƒS ˆ|ƒS ‡‡‡‡‡‡‡	‡
‡f	d	d
„‰ t|tƒr‰‡ fdd„|D ƒS ˆ |ƒS )Nrd   r   zchunk_len must be positive.r   c              	      s(   ˆj  | ˆˆˆˆ ˆˆ¡}ˆj|ˆjd�S )Nrh   )r<   Z_ParallelEncodeAsBufferrk   rl   )r    rn   )rK   rL   Ú	chunk_lenr   rR   rM   rI   r}   r
   r   r¡      s   ÿz9SentencePieceProcessor.ParallelEncode.<locals>._encode_npc                    r¢   r
   r
   r£   r¥   r
   r   rp     rv   z9SentencePieceProcessor.ParallelEncode.<locals>.<listcomp>c                    s®   ˆt u rˆj | ˆˆˆˆ ˆˆ¡S ˆtu sˆtu r*ˆtu }ˆj | ˆˆˆˆ ˆˆˆ|¡	S ˆdkr;ˆj | ˆˆˆˆ ˆˆˆ¡S ˆdkrPˆj | ˆˆˆˆ ˆˆˆ¡}t|ƒS td 	ˆ¡ƒ‚r¦   )
r>   r<   Z_ParallelEncodeAsIdsr   r   Z_ParallelEncodeAsPiecesZ _ParallelEncodeAsSerializedProtor   r*   rU   r§   )	rK   rL   r¶   rN   rR   rS   rM   rI   r}   r
   r   r¨   	  s&   ÿÿÿÿz6SentencePieceProcessor.ParallelEncode.<locals>._encodec                    r¢   r
   r
   r£   r©   r
   r   rp     rv   )r=   r@   rA   rB   rC   rG   r   r>   r/   r*   r   rw   )rI   r|   rK   rL   rM   rN   r¶   rR   r}   rS   rJ   r
   )r¨   r¡   rK   rL   r¶   rN   r   rR   rS   rM   rI   r}   r   ÚParallelEncodeá  s2   

z%SentencePieceProcessor.ParallelEncodec                 K   r†   r‡   )r·   r   rŠ   r
   r
   r   ÚParallelEncodeAsPieces  rŒ   z-SentencePieceProcessor.ParallelEncodeAsPiecesc                 K   r†   r‡   )r·   r>   rŠ   r
   r
   r   ÚParallelEncodeAsIds!  rŒ   z*SentencePieceProcessor.ParallelEncodeAsIdsc                 K   r�   r�   ©r·   rŠ   r
   r
   r   ÚParallelEncodeAsNumpy$  rŒ   z,SentencePieceProcessor.ParallelEncodeAsNumpyc                 K   r�   r“   rº   rŠ   r
   r
   r   ÚParallelEncodeAsProto'  rŒ   z,SentencePieceProcessor.ParallelEncodeAsProtoc           
      C   s´   | j ||td�}|d ur|dkrtdƒ‚|d u r| j}|d u s$t|tƒs(tdƒ‚|dkr0t|ƒnd}|dkr8dn|}| j|||||d�}	|dkrXt|	t	ƒrTd	d
„ |	D ƒS t
|	ƒS |	S )Nr:   r"   rc   rd   Fr!   r    )r~   c                 S   rr   r
   rs   rt   r
   r
   r   rp   9  rv   z1SentencePieceProcessor.Decode.<locals>.<listcomp>)r=   r   r*   rG   r   r>   r/   r.   Ú_decode_rawrw   r   )
rI   r|   rS   rJ   rR   r}   r~   r„   Zcpp_return_typeZraw_valr
   r
   r   ÚDecode*  s   
zSentencePieceProcessor.Decodec                 C   s*  |d u rg }nt |ttfƒrt|ƒdkrg }dd„ }t|ƒ }|r$|g}t|dƒr/|jdk}nt|ƒdko:t|d ƒ}|rUd}	|D ]}
t|
ƒrSt|
ƒdkrS||
ƒ}	 nqAn||ƒ}	t |	|f¡}|rf||vrmt	d 
|¡ƒ‚|| }t| j|ƒ}|dkr‰|r„|||||ƒS |||ƒS |r‘||||ƒS ||ƒS )	Nr   c                 S   s0   t | ƒdkrdS t| dƒrdS t| d ttfƒS )Nr   Fri   )Úlenr   r   r   r   )Úseqr
   r
   r   Ú
_is_piecesH  s
   
z6SentencePieceProcessor._decode_raw.<locals>._is_piecesÚndimr   Fz$unknown return_type={} or input typer"   )r   r   r   r¿   r   r   rÂ   Ú_DECODE_DISPATCHry   r*   rU   rz   r<   )rI   r|   rS   rR   r}   r~   rÁ   Ú	is_singler   Zinput_is_piecesÚitemr€   r�   r‚   r
   r
   r   r½   =  s>   

€
z"SentencePieceProcessor._decode_rawc                 K   ó   | j d||dœ|¤ŽS r‡   ©r¾   ©rI   r|   rS   rZ   r
   r
   r   ÚDecodePieces|  rŒ   z#SentencePieceProcessor.DecodePiecesc                 K   rÆ   r‡   rÇ   rÈ   r
   r
   r   Ú	DecodeIds  rŒ   z SentencePieceProcessor.DecodeIdsc                 K   r�   r“   rÇ   rŠ   r
   r
   r   ÚDecodePiecesAsProto‚  rŒ   z*SentencePieceProcessor.DecodePiecesAsProtoc                 K   r�   r“   rÇ   rŠ   r
   r
   r   ÚDecodeIdsAsProto…  rŒ   z'SentencePieceProcessor.DecodeIdsAsProtoc                    óN   t |tƒr|r‡ fdd„|D ƒS ‡ fdd„|D ƒS |r!ˆ j |¡S ˆ j |¡S )Nc                    ó   g | ]}ˆ j  |¡‘qS r
   )r<   Ú_NormalizeWithOffsetsr$   ©rI   r
   r   rp   ‹  ó    z4SentencePieceProcessor.Normalize.<locals>.<listcomp>c                    rÎ   r
   )r<   Ú
_Normalizer$   rÐ   r
   r   rp   Œ  rÑ   )r   rw   r<   rÏ   rÒ   ©rI   r|   Zwith_offsetsr
   rÐ   r   Ú	Normalizeˆ  ó   
z SentencePieceProcessor.Normalizec                 C   ó
   | j  ¡ S r   )r<   ÚGetPieceSizerÐ   r
   r
   r   r×   ‘  ó   
z#SentencePieceProcessor.GetPieceSizec                 C   r`   r   )r<   Ú	PieceToId©rI   Úpiecer
   r
   r   rÙ   ”  rb   z SentencePieceProcessor.PieceToIdc                 C   r`   r   )r<   Ú	IdToPiece©rI   Úidr
   r
   r   rÜ   —  rb   z SentencePieceProcessor.IdToPiecec                 C   r`   r   )r<   ÚGetScorerÝ   r
   r
   r   rß   š  rb   zSentencePieceProcessor.GetScorec                 C   r`   r   )r<   Ú	IsUnknownrÝ   r
   r
   r   rà   �  rb   z SentencePieceProcessor.IsUnknownc                 C   r`   r   )r<   Ú	IsControlrÝ   r
   r
   r   rá      rb   z SentencePieceProcessor.IsControlc                 C   r`   r   )r<   ÚIsUnusedrÝ   r
   r
   r   râ   £  rb   zSentencePieceProcessor.IsUnusedc                 C   r`   r   )r<   ÚIsByterÝ   r
   r
   r   rã   ¦  rb   zSentencePieceProcessor.IsBytec                 C   rÖ   r   )r<   Úunk_idrÐ   r
   r
   r   rä   ©  rØ   zSentencePieceProcessor.unk_idc                 C   rÖ   r   )r<   Úbos_idrÐ   r
   r
   r   rå   ¬  rØ   zSentencePieceProcessor.bos_idc                 C   rÖ   r   )r<   Úeos_idrÐ   r
   r
   r   ræ   ¯  rØ   zSentencePieceProcessor.eos_idc                 C   rÖ   r   )r<   Úpad_idrÐ   r
   r
   r   rç   ²  rØ   zSentencePieceProcessor.pad_idc                 C   rÖ   r   )r<   r2   rÐ   r
   r
   r   r2   µ  rØ   z-SentencePieceProcessor.serialized_model_protoc                 C   s&   | j  ¡ }| dd ¡ |  ¡ |d< |S )Nr<   r1   )Ú__dict__ÚcopyÚpopr2   ©rI   Ústater
   r
   r   Ú__getstate__»  s   
z#SentencePieceProcessor.__getstate__c                 C   sF   t |tƒs
d|i}n| ¡ }t | ¡ |  | d¡¡ | j |¡ d S )Nr1   )	r   Údictré   r7   rT   r^   rê   rè   Úupdaterë   r
   r
   r   Ú__setstate__Á  s   


z#SentencePieceProcessor.__setstate__c                 C   s   |   ¡ S r   )r×   rÐ   r
   r
   r   r   Ë  s   zSentencePieceProcessor.__len__c                 C   s
   |   |¡S r   )rÙ   rÚ   r
   r
   r   Ú__getitem__Î  rØ   z"SentencePieceProcessor.__getitem__)NNNFFFFFr8   r9   r8   Nr   ©NN)NNNNNNNNNNNN)NNNNNNN©	NNNNNNNNN)NNNNN)F)9Ú__name__Ú
__module__Ú__qualname__rT   r=   Úclassmethodr\   r_   rH   rX   r^   r…   r‹   rŽ   r’   r”   r•   rš   rœ   rž   rŸ   rª   r¯   r±   r´   rµ   r·   r¸   r¹   r»   r¼   r¾   r½   rÉ   rÊ   rË   rÌ   rÔ   r×   rÙ   rÜ   rß   rà   rá   râ   rã   rä   rå   ræ   rç   r2   Z
piece_sizeÚ
vocab_sizerí   rð   r   rñ   r
   r
   r
   r   r7   Ë   sÀ    
ô



	
ód




ø
9



ö=


?

	
r7   c                   @   s&   e Zd Zddd„Zdd„ Zdd„ ZdS )	Ú
_LogStreamNc                 C   s$   || _ | j d urtj ¡ | _d S d S r   )ÚostreamÚsysÚstderrÚfilenoÚorig_stream_fileno)rI   rú   r
   r
   r   rT   Ó  s   
ÿz_LogStream.__init__c                 C   sX   | j d ur*| j  ¡ }t | j¡| _z
t || j¡ W d S  ty)   t | j¡ ‚ w d S r   )	rú   rý   r+   Úduprþ   Úorig_stream_dupÚdup2Ú	ExceptionÚclose)rI   Zostream_filenor
   r
   r   Ú	__enter__Ø  s   

þûz_LogStream.__enter__c                 C   s8   | j d urt | j| j¡ t | j¡ | j  ¡  d S d S r   )rú   r+   r  r   rþ   r  Úflush)rI   ÚtypeÚvalueÚ	tracebackr
   r
   r   Ú__exit__â  s
   
ýz_LogStream.__exit__r   )rô   rõ   rö   rT   r  r	  r
   r
   r
   r   rù   Ò  s    

rù   c                   @   s(   e Zd Zeddd„ƒZeddd„ƒZdS )ÚSentencePieceTrainerNc                 K   sî   | d urt | tƒrtj | ¡S dd„ }d }d }d}d }d }i }| ¡ D ]/\}	}
|	dv r.|
}q#|	dkr5|
}q#|	dkr>t|
ƒ}q#|	dv rE|
}q#|	dv rL|
}q#||
ƒ||	< q#|r[| ¡ |d	< tjj|||||d ud
�}|ru|d uru| 	|¡ d S d S )Nc                 S   sD   t | tƒrt ¡ }tj|dd�}| dd„ | D ƒ¡ | ¡ S t| ƒS )Nr'   )Úlineterminatorc                 S   rr   r
   )r   )r%   Úvr
   r
   r   rp   ó  rv   z@SentencePieceTrainer._Train.<locals>._encode.<locals>.<listcomp>)	r   rw   ÚioÚStringIOÚcsvÚwriterÚwriterowÚgetvaluer   )r  Úfr  r
   r
   r   r¨   ï  s   
z,SentencePieceTrainer._Train.<locals>._encodeF)Úsentence_iteratorZsentence_readerÚpretokenizerÚ"allow_inconsistent_pretokenization)Úmodel_writer)Ú
normalizerZ_serialized_normalizer_spec)r  r  r  Zreturn_model_proto)
r   r   r   r
  Z_TrainFromStringÚitemsr.   Úserialized_normalizer_specZ_TrainFromMapÚwrite)ÚargrZ   r¨   r  r  r  r  r  Ú
new_kwargsÚkeyr  Úresr
   r
   r   Ú_Trainê  sD   	
û
zSentencePieceTrainer._Trainc                 K   sB   t |d�� tjdd| i|¤ŽW  d   ƒ S 1 sw   Y  d S )N)rú   r  r
   )rù   r
  r   )r  Z	logstreamrZ   r
   r
   r   ÚTrain  s   $ÿzSentencePieceTrainer.Trainr   rò   )rô   rõ   rö   Ústaticmethodr   r!  r
   r
   r
   r   r
  é  s
    0r
  c                   @   sZ   e Zd Z									ddd„Zdd„ Zdd„ Zdd	„ Zdd
d„Zdd„ Zdd„ Z	dS )ÚSentencePieceNormalizerNc
           
      C   sì   t  ¡ | _|r| j |¡ nC|r t|dƒr| ¡ }| j |¡ n1|r2t|dƒr+| ¡ }| j |¡ n|r;| j |¡ n|rD| j 	|¡ n|rM| j 
|¡ ntdƒ‚|d ur\| j d|¡ |d urg| j d|¡ |	d urt| j d|	¡ d S d S )Nr(   zno model is specifiedÚadd_dummy_prefixÚescape_whitespacesÚremove_extra_whitespaces)r   r#  Ú_normalizerrX   r   r(   r^   Z LoadFromSerializedNormalizerSpecZLoadFromRuleTSVZLoadFromRuleNameZLoadFromMapr*   Z_SetProtoField)
rI   r0   r1   Únormalizer_specZrule_tsvZ	rule_nameZnorm_mapr$  r%  r&  r
   r
   r   rT   "  s6   



ÿÿz SentencePieceNormalizer.__init__c                 C   rÖ   r   )r'  Ú	DecompilerÐ   r
   r
   r   r)  K  rØ   z!SentencePieceNormalizer.Decompilec                 C   rÖ   r   ©r'  r2   rÐ   r
   r
   r   r2   N  rØ   z.SentencePieceNormalizer.serialized_model_protoc                 C   rÖ   r   )r'  r  rÐ   r
   r
   r   r  Q  rØ   z2SentencePieceNormalizer.serialized_normalizer_specc                    rÍ   )Nc                    rÎ   r
   )r'  rÏ   r$   rÐ   r
   r   rp   W  rÑ   z5SentencePieceNormalizer.Normalize.<locals>.<listcomp>c                    rÎ   r
   )r'  rÒ   r$   rÐ   r
   r   rp   X  rÑ   )r   rw   r'  rÏ   rÒ   rÓ   r
   rÐ   r   rÔ   T  rÕ   z!SentencePieceNormalizer.Normalizec                 C   rÖ   r   r*  rÐ   r
   r
   r   rí   ]  rØ   z$SentencePieceNormalizer.__getstate__c                 C   s   t  ¡ | _| j |¡ d S r   )r   r#  r'  r^   )rI   r2   r
   r
   r   rð   `  s   
z$SentencePieceNormalizer.__setstate__ró   r   )
rô   rõ   rö   rT   r)  r2   r  rÔ   rí   rð   r
   r
   r
   r   r#  !  s"    
÷)
	r#  c                 C   sp   dd l }i }| j ¡ D ]\}}| d|¡r&| dd|¡ ¡  dd¡}|||< q| ¡ D ]
\}}t| ||ƒ q+d S )Nr   z^[A-Z]+z(?<!^)(?=[A-Z])Ú_Zn_bestZnbest)Úrerè   r  ÚmatchÚsubÚlowerÚreplaceÚsetattr)Ú	classnamer,  Z	snake_mapÚkr  Úsnaker
   r
   r   Ú_add_snake_casef  s   €ÿr5  )Ú__version__r   Úpackage_data)NNNNFFF)1r+   rû   r  r  Úimportlib.resourcesÚ	importlibr'   r   r   r   r   r   r   r   r   r   r   r>   r   r   rx   rÃ   Ú
ThreadPoolZSetRandomGeneratorSeedZSetMinLogLevelZSetNBestTimeoutZ
SetDataDirZset_random_generator_seedZset_min_log_levelZset_nbest_timeoutZset_data_dirr6   Zconvert_to_flatbufferr7   rù   r
  r#  r5  r…   ZTokenizer¾   Z
DetokenizeÚ_versionr6  ÚpathÚjoinÚ	resourcesÚfilesr  r
   r
   r
   r   Ú<module>   s¶    ùùô!üüüüé 
ù?    8E$ÿ