o
    5ôj20  ã                   @   s0  d dl Z d dlZd dlZd dlmZmZ d dlmZmZ d dl	m
Z
mZmZmZ d dlZi dd“dd“d	d
“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd “d!d"“d#d$“d%d&“i d'd(“d)d*“d+d,“d-d.“d/d0“d1d2“d3d4“d5d6“d7d8“d9d:“d;d<“d=d>“d?d@“dAdB“dCdD“dEdF“dGdH“¥i dIdJ“dKdL“dMdN“dOdP“dQdR“dSdT“dUdV“dWdX“dYdZ“d[d\“d]d^“d_d`“dadb“dcdd“dedf“dgdh“didj“¥i dkdl“dmdn“dodp“dqdr“dsdt“dudv“dwdx“dydz“d{d|“d}d~“dd€“d�d‚“dƒd„“d…d†“d‡dˆ“d‰dŠ“d‹dŒ“¥i d�dŽ“d�d�“d‘d’“d“d”“d•d–“d—d˜“d™dš“d›dœ“d�dž“dŸd “d¡d¢“d£d¤“d¥d¦“d§d¨“d©dª“d«d¬“d­d®“¥d¯d°d±d²d³d´dµd¶d·d¸d¹dºd»d¼d½d¾œ¥Zi d¿dÀ„ e ¡ D ƒ¥dÁddd¥dÂd§d�d7d7dƒdddÃœ¥ZeG dÄdÅ„ dÅƒƒZeddÆ�dÔdÉedÊefdËdÌ„ƒZeddÆ�dÈdddÍœdÎedÊedÏee dÐee dÑef
dÒdÓ„ƒZdS )Õé    N)Ú	dataclassÚfield)Úcached_propertyÚ	lru_cache)ÚDictÚListÚOptionalÚTupleÚenÚenglishÚzhÚchineseÚdeÚgermanÚesÚspanishÚruÚrussianÚkoÚkoreanÚfrÚfrenchÚjaÚjapaneseÚptÚ
portugueseÚtrÚturkishÚplÚpolishÚcaÚcatalanÚnlÚdutchÚarÚarabicÚsvÚswedishÚitÚitalianÚidZ
indonesianÚhiZhindiÚfiÚfinnishÚviZ
vietnameseÚheÚhebrewÚukZ	ukrainianÚelÚgreekÚmsZmalayÚcsÚczechÚroÚromanianÚdaÚdanishÚhuÚ	hungarianÚtaZtamilÚnoÚ	norwegianÚthÚthaiÚurZurduÚhrÚcroatianÚbgÚ	bulgarianÚltÚ
lithuanianÚlaÚlatinÚmiZmaoriÚmlZ	malayalamÚcyZwelshÚskÚslovakÚteZteluguÚfaZpersianÚlvZlatvianÚbnZbengaliÚsrZserbianÚazZazerbaijaniÚslÚ	slovenianÚknZkannadaÚetÚestonianÚmkZ
macedonianÚbrZbretonÚeuZbasqueÚisÚ	icelandicÚhyZarmenianÚneZnepaliÚmnZ	mongolianÚbsZbosnianÚkkZkazakhÚsqZalbanianÚswZswahiliÚglÚgalicianÚmrZmarathiÚpaZpunjabiÚsiÚsinhalaÚkmZkhmerZsnZshonaZyoZyorubaÚsoZsomaliÚafZ	afrikaansÚocZoccitanÚkaZgeorgianÚbeZ
belarusianÚtgZtajikÚsdZsindhiZguZgujaratiÚamZamharicÚyiZyiddishÚloZlaoÚuzZuzbekÚfoZfaroeseZhtzhaitian creoleÚpsZpashtoÚtkZturkmenÚnnÚnynorskÚmtZmalteseZsanskritZluxembourgishZmyanmarZtibetanZtagalogZmalagasyZassameseZtatarZhawaiianZlingalaZhausaZbashkirZjavaneseZ	sundaneseZ	cantonese)ÚsaÚlbÚmyZboÚtlZmgÚasÚttZhawÚlnZhaÚbaZjwZsuÚyuec                 C   s   i | ]\}}||“qS © rˆ   )Ú.0ÚcodeÚlanguagerˆ   rˆ   úd/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/whisper/tokenizer.pyÚ
<dictcomp>s   s    r�   r�   r€   )ZburmeseZ	valencianZflemishZhaitianZletzeburgeschZpushtoZpanjabiZ	moldavianZmoldovanZ	sinhaleseZ	castilianZmandarinc                   @   sô  e Zd ZU dZejed< eed< dZe	e
 ed< dZe	e
 ed< dZee ed< eed	�Zee
ef ed
< dd„ Zdd„ Zdee de
fdd„Zdee de
fdd„Zedefdd„ƒZedefdd„ƒZedefdd„ƒZedefdd„ƒZedefdd„ƒZedefdd „ƒZedefd!d"„ƒZedefd#d$„ƒZedefd%d&„ƒZ edefd'd(„ƒZ!d)d*„ Z"edee fd+d,„ƒZ#edee
 fd-d.„ƒZ$edee fd/d0„ƒZ%edee fd1d2„ƒZ&d3ee fd4d5„Z'd3ee fd6d7„Z(d3ee fd8d9„Z)dS ):Ú	TokenizerzIA thin wrapper around `tiktoken` providing quick access to special tokensÚencodingÚnum_languagesNr‹   Útaskrˆ   Úsot_sequence)Údefault_factoryÚspecial_tokensc           	      C   s¶   | j jD ]}| j  |¡}|| j|< q| jd }| jd }| jd }tt ¡ ƒd | j… }|g}| jd urA| 	|d | 
| j¡ ¡ | jd urT| jdkrM|n|}| 	|¡ t|ƒ| _d S )Nú<|startoftranscript|>ú<|translate|>ú<|transcribe|>é   Ú
transcribe)r�   Zspecial_tokens_setZencode_single_tokenr”   ÚtupleÚ	LANGUAGESÚkeysr�   r‹   ÚappendÚindexr‘   r’   )	ÚselfÚspecialZspecial_tokenÚsotÚ	translater™   Zlangsr’   Z
task_tokenrˆ   rˆ   rŒ   Ú__post_init__Ž   s   





zTokenizer.__post_init__c                 K   ó   | j j|fi |¤ŽS ©N)r�   Úencode)rŸ   ÚtextÚkwargsrˆ   rˆ   rŒ   r¦   ¡   s   zTokenizer.encodeÚ	token_idsÚreturnc                    s&   ‡ fdd„|D ƒ}ˆ j j|fi |¤ŽS )Nc                    s   g | ]	}|ˆ j k r|‘qS rˆ   )Útimestamp_begin)r‰   Út©rŸ   rˆ   rŒ   Ú
<listcomp>¥   s    z$Tokenizer.decode.<locals>.<listcomp>©r�   Údecode©rŸ   r©   r¨   rˆ   r­   rŒ   r°   ¤   s   zTokenizer.decodec                 K   r¤   )zÆ
        Timestamp tokens are above other special tokens' id range and are ignored by `decode()`.
        This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>".
        r¯   r±   rˆ   rˆ   rŒ   Údecode_with_timestamps¨   s   z Tokenizer.decode_with_timestampsc                 C   s   | j jS r¥   )r�   Z	eot_tokenr­   rˆ   rˆ   rŒ   Úeot¯   s   zTokenizer.eotc                 C   ó
   | j d S )Nr—   ©r”   r­   rˆ   rˆ   rŒ   r™   ³   ó   
zTokenizer.transcribec                 C   r´   )Nr–   rµ   r­   rˆ   rˆ   rŒ   r¢   ·   r¶   zTokenizer.translatec                 C   r´   )Nr•   rµ   r­   rˆ   rˆ   rŒ   r¡   »   r¶   zTokenizer.sotc                 C   r´   )Nú<|startoflm|>rµ   r­   rˆ   rˆ   rŒ   Úsot_lm¿   r¶   zTokenizer.sot_lmc                 C   r´   )Nú<|startofprev|>rµ   r­   rˆ   rˆ   rŒ   Úsot_prevÃ   r¶   zTokenizer.sot_prevc                 C   r´   )Nú<|nospeech|>rµ   r­   rˆ   rˆ   rŒ   Ú	no_speechÇ   r¶   zTokenizer.no_speechc                 C   r´   )Nú<|notimestamps|>rµ   r­   rˆ   rˆ   rŒ   Úno_timestampsË   r¶   zTokenizer.no_timestampsc                 C   r´   )Nz<|0.00|>rµ   r­   rˆ   rˆ   rŒ   r«   Ï   r¶   zTokenizer.timestamp_beginc                 C   s   | j du r	tdƒ‚|  | j ¡S )zGReturns the token id corresponding to the value of the `language` fieldNz6This tokenizer does not have language token configured)r‹   Ú
ValueErrorÚto_language_tokenr­   rˆ   rˆ   rŒ   Úlanguage_tokenÓ   s   
zTokenizer.language_tokenc                 C   s.   | j  d|› d�d ¡ }r|S td|› d�ƒ‚)Nú<|ú|>z	Language z not found in tokenizer.)r”   ÚgetÚKeyError)rŸ   r‹   Útokenrˆ   rˆ   rŒ   rÀ   Û   s   zTokenizer.to_language_tokenc                 C   sB   g }| j  ¡ D ]\}}| d¡tv r| |¡ qt|ƒd | j… S )Nú<|>)r”   ÚitemsÚstripr›   r�   rš   r�   )rŸ   ÚresultrÆ   Ztoken_idrˆ   rˆ   rŒ   Úall_language_tokensá   s   
€zTokenizer.all_language_tokensc                    s   t ‡ fdd„ˆ jD ƒƒS )Nc                 3   s"   � | ]}ˆ   |g¡ d ¡V  qdS )rÇ   N)r°   rÉ   )r‰   Ú_lr­   rˆ   rŒ   Ú	<genexpr>ë   s   €  z/Tokenizer.all_language_codes.<locals>.<genexpr>)rš   rË   r­   rˆ   r­   rŒ   Úall_language_codesé   s   zTokenizer.all_language_codesc                 C   s   t t| jƒ| jg ƒS r¥   )rš   Úlistr’   r¾   r­   rˆ   rˆ   rŒ   Ú#sot_sequence_including_notimestampsí   s   z-Tokenizer.sot_sequence_including_notimestampsc                 C   s´   t dƒ}|d ¡ 7 }tdƒ}tdd„ |D ƒƒsJ ‚| j d¡d | j d¡d h}|t |ƒ D ]$}| j |¡| j d	| ¡fD ]}t|ƒd
ksK||v rR| |d ¡ q?q/tt	|ƒƒS )u…  
        Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech
        annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.

        - â™ªâ™ªâ™ª
        - ( SPEAKING FOREIGN LANGUAGE )
        - [DAVID] Hey there,

        keeping basic punctuations like commas, periods, question marks, exclamation points, etc.
        u#   "#()*+/:;<=>@[\]^_`{|}~ã€Œã€�ã€Žã€�uK   << >> <<< >>> -- --- -( -[ (' (" (( )) ((( ))) [[ ]] {{ }} â™ªâ™ª â™ªâ™ªâ™ªu   â™©â™ªâ™«â™¬â™­â™®â™¯c                 s   s,   � | ]}d t |ƒ  kodkn  V  qdS )i@&  i&  N)Úord)r‰   Úcrˆ   rˆ   rŒ   rÍ     s   €* z.Tokenizer.non_speech_tokens.<locals>.<genexpr>z -r   z 'ú r˜   )
rÏ   ÚsplitÚsetÚallr�   r¦   ÚlenÚaddrš   Úsorted)rŸ   ÚsymbolsZmiscellaneousrÊ   ÚsymbolÚtokensrˆ   rˆ   rŒ   Únon_speech_tokensñ   s    ÿ 
þ€ûzTokenizer.non_speech_tokensrÜ   c                 C   s   | j dv r
|  |¡S |  |¡S )N>   r‡   r@   r�   r   r   rw   )r‹   Úsplit_tokens_on_unicodeÚsplit_tokens_on_spaces)rŸ   rÜ   rˆ   rˆ   rŒ   Úsplit_to_word_tokens  s   


zTokenizer.split_to_word_tokensc           
      C   s†   |   |¡}d}g }g }g }d}|D ]-}| |¡ |   |¡}	||	vs,|||	 |¡  |kr>| |	¡ | |¡ g }|t|	ƒ7 }q||fS )Nu   ï¿½r   )r²   r�   rž   r×   )
rŸ   rÜ   Zdecoded_fullZreplacement_charÚwordsÚword_tokensZcurrent_tokensZunicode_offsetrÆ   Údecodedrˆ   rˆ   rŒ   rÞ     s&   


ÿ

€z!Tokenizer.split_tokens_on_unicodec                 C   s¤   |   |¡\}}g }g }t||ƒD ]=\}}|d | jk}| d¡}	| ¡ tjv }
|s3|	s3|
s3t|ƒdkr>| |¡ | |¡ q|d | |d< |d  	|¡ q||fS )Nr   rÓ   éÿÿÿÿ)
rÞ   Úzipr³   Ú
startswithrÉ   ÚstringÚpunctuationr×   r�   Úextend)rŸ   rÜ   ZsubwordsZsubword_tokens_listrá   râ   ZsubwordZsubword_tokensr    Z
with_spacerè   rˆ   rˆ   rŒ   rß   7  s   

z Tokenizer.split_tokens_on_spaces)*Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚtiktokenÚEncodingÚ__annotations__Úintr‹   r   Ústrr‘   r’   r	   r   Údictr”   r   r£   r¦   r   r°   r²   r   r³   r™   r¢   r¡   r¸   rº   r¼   r¾   r«   rÁ   rÀ   rË   rÎ   rÐ   rÝ   rà   rÞ   rß   rˆ   rˆ   rˆ   rŒ   rŽ   ƒ   sX   
 
#	rŽ   )ÚmaxsizeÚgpt2éc   Únamer�   c                 C   sÌ   t j t j t¡d| › d�¡}dd„ dd„ t|ƒD ƒD ƒ}t|ƒ}i }ddgd	d
„ tt 	¡ ƒd |… D ƒ¢d‘d‘d‘d‘d‘d‘dd
„ t
dƒD ƒ¢}|D ]
}|||< |d7 }qMtjt j |¡|d||d�S )NÚassetsz	.tiktokenc                 S   s    i | ]\}}t  |¡t|ƒ“qS rˆ   )Úbase64Ú	b64decoderñ   )r‰   rÆ   Úrankrˆ   rˆ   rŒ   r�   M  s    ÿÿz get_encoding.<locals>.<dictcomp>c                 s   s   � | ]	}|r|  ¡ V  qd S r¥   )rÔ   )r‰   Úlinerˆ   rˆ   rŒ   rÍ   O  s   € zget_encoding.<locals>.<genexpr>z<|endoftext|>r•   c                 S   s   g | ]}d |› d�‘qS )rÂ   rÃ   rˆ   )r‰   Úlangrˆ   rˆ   rŒ   r®   W  s    z get_encoding.<locals>.<listcomp>r–   r—   r·   r¹   r»   r½   c                 S   s   g | ]}d |d d›d�‘qS )rÂ   g{®Gáz”?z.2frÃ   rˆ   )r‰   Úirˆ   rˆ   rŒ   r®   ^  s    iÝ  r˜   zJ's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+)r÷   Zexplicit_n_vocabÚpat_strZmergeable_ranksr”   )ÚosÚpathÚjoinÚdirnameÚ__file__Úopenr×   rÏ   r›   rœ   Úrangerî   rï   Úbasename)r÷   r�   Z
vocab_pathÚranksZn_vocabr”   ÚspecialsrÆ   rˆ   rˆ   rŒ   Úget_encodingJ  sF   þþýüûúùø	÷
ö

ûr
  )r�   r‹   r‘   Úmultilingualr‹   r‘   rª   c                C   sz   |d ur|  ¡ }|tvr|tv rt| }ntd|› �ƒ‚| r)d}|p#d}|p'd}nd}d }d }t||d�}t||||d�S )NzUnsupported language: r  r
   r™   rõ   )r÷   r�   )r�   r�   r‹   r‘   )Úlowerr›   ÚTO_LANGUAGE_CODEr¿   r
  rŽ   )r  r�   r‹   r‘   Zencoding_namer�   rˆ   rˆ   rŒ   Úget_tokenizern  s"   

ÿr  )rõ   rö   )rù   r   rç   Údataclassesr   r   Ú	functoolsr   r   Útypingr   r   r   r	   rî   r›   rÈ   r  rŽ   rò   rñ   r
  Úboolr  rˆ   rˆ   rˆ   rŒ   Ú<module>   sÌ   ÿþýüûúùø	÷
öõôóòñðïîíìëêéèçæåäãâá à!ß"Þ#Ý$Ü%Û&Ú'Ù(Ø)×*Ö+Õ,Ô-Ó.Ò/Ñ0Ð1Ï2Î3Í4Ì5Ë6Ê7É8È9Ç:Æ;Å<Ä=Ã>Â?Á@ÀA¿B¾C½D¼E»FºG¹H¸I·J¶KµL´M³N²O±P°Q¯R®S­T¬U«Vœhÿó G#ûÿýüûú