o
    ±ƒðjÑQ  ã                   @   sR  d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZ ddlZddlZe e¡Zg d¢ZdZddd	d
ddœZdedefdd„Zdefdd„Zdefdd„Zde
e
e  fdd„Zde
e
e  fdd„ZG dd„ dƒZdeee  dejfdd„Zd ee eef  de eejf fd!d"„Z!G d#d$„ d$ƒZ"d%edee fd&d'„Z#dS )(a…  Torch-free ONNX g2pW converter.

A replacement for ``g2pw.G2PWConverter`` that runs the same ``g2pw.onnx`` graph
under onnxruntime without importing torch.

Upstream g2pW already runs its model in onnxruntime; torch is used only to build
padded integer tensors and to iterate batches (``TextDataset`` is a
``torch.utils.data.Dataset``, batched by a ``DataLoader`` whose ``collate_fn``
calls ``torch.tensor``/``pad_sequence``, and its ``predict()`` immediately calls
``.numpy()`` on every tensor before feeding the session). That plumbing is numpy
here, which keeps torch -- around 750 MB installed -- out of the Chinese
phonemization path. Dropping the ``DataLoader`` also stops it forking worker
processes on every call, which makes this faster than upstream as a side effect.

The text processing (``wordize_and_map``, ``tokenize_and_map``, the label
builders, the windowing and truncation rules) is ported from g2pW unchanged so
tokenization and index mapping stay identical.

Ported from g2pW, Copyright (c) 2022 Yi-Chang Chen, Apache License 2.0.
https://github.com/GitYCC/g2pW
A copy of the license is distributed as licenses/LICENSE.g2pW-Apache-2.0.
é    N)ÚPath)ÚAnyÚOptionalÚSequenceÚUnion)ÚUNKÚAÚCÚDÚIÚNÚPÚTÚVÚDEÚSHI)ú$bopomofo_to_pinyin_wo_tune_dict.jsonúchar_bopomofo_dict.jsonúbert-base-chinese_s2t_dict.txtzbert-base-chineseé    é   TF)Úmodel_sourceÚwindow_sizeÚ
batch_sizeÚuse_maskÚuse_char_phonemeÚ	model_dirÚreturnc                    s�   t ‡ fdd„tD ƒƒrˆ S ddl}|j d¡}tt|ddƒpg ƒ}|s+tdˆ › d�ƒ‚t|d ƒ‰‡fd	d
„tD ƒ}|rFtdˆ› d|› �ƒ‚ˆS )a  Locate the directory holding the pinyin/bopomofo lookup tables.

    Prefers the model directory, so the tables can travel in the model archive
    alongside ``g2pw.onnx``. Falls back to the installed ``g2pw`` package, which
    is where they ship today.
    c                 3   s   � | ]	}ˆ |   ¡ V  qd S ©N©Úis_file)Ú.0Ú	file_name)r   © ú‘/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/piper/g2pw_onnx.pyÚ	<genexpr>A   s   € z!_find_data_dir.<locals>.<genexpr>r   NÚg2pwÚsubmodule_search_locationsz#Chinese lookup tables not found in z? and the g2pW package is not installed. Install the [zh] extra.c                    s   g | ]
}ˆ |   ¡ s|‘qS r#   r   )r!   Úf)Úpackage_dirr#   r$   Ú
<listcomp>R   s    z"_find_data_dir.<locals>.<listcomp>z#Chinese lookup tables missing from z: )	ÚallÚ
DATA_FILESÚimportlib.utilÚutilÚ	find_specÚlistÚgetattrÚRuntimeErrorr   )r   Ú	importlibÚspecÚ	locationsÚmissingr#   )r   r)   r$   Ú_find_data_dir:   s    
ÿÿr7   Útextc           
      C   s   g }g }g }t | ƒdkr‹t d| ¡}|r+| d¡}|dgt |ƒ 7 }| t |ƒd… } qt d| ¡}|rb| d¡}t |ƒ}|t |ƒ }	| ||	f¡ |t |ƒgt |ƒ 7 }| |¡ | t |ƒd… } n#t |ƒ}|d }	| ||	f¡ |t |ƒg7 }| | d ¡ | dd… } t | ƒdks|||fS )z°Split text into words, mapping indices both ways.

    Runs of ASCII alphanumerics become one word, every other character becomes
    its own word, and spaces map to None.
    r   z^ +Nz^[a-zA-Z0-9]+é   )ÚlenÚreÚmatchÚgroupÚappend)
r8   ÚwordsÚindex_map_from_text_to_wordÚindex_map_from_word_to_textÚmatch_spaceÚ	space_strÚmatch_enÚen_wordÚword_start_posÚword_end_posr#   r#   r$   Úwordize_and_map`   s6   


ä
rH   c                 C   sè   t |ƒ\}}}g }g }t||ƒD ]E\}\}}	|  |¡}
t|
ƒdks&|
dgkr3| ||	f¡ | d¡ q|}|
D ]}tt dd|¡ƒ}| ||| f¡ || }| |¡ q7q|}t|ƒD ]\}\}}t||ƒD ]}|||< qgq\|||fS )zDTokenize text, returning tokens and index maps to and from the text.r   z[UNK]z^##Ú )	rH   ÚzipÚtokenizer:   r>   r;   ÚsubÚ	enumerateÚrange)Ú	tokenizerr8   r?   Ú	text2wordÚ	word2textÚtokensÚindex_map_from_token_to_textÚwordÚ
word_startÚword_endÚword_tokensÚcurrent_word_startÚ
word_tokenÚword_token_lenÚindex_map_from_text_to_tokenÚiÚtoken_startÚ	token_endÚ	token_posr#   r#   r$   Útokenize_and_map‰   s.   

ÿú
ÿ
r`   Úpolyphonic_charsc                 C   sD   t dd„ | D ƒƒ}i }| D ]\}}| |g ¡ | |¡¡ q||fS )z:Get bare phoneme labels, plus char to candidate label ids.c                 S   s   h | ]\}}|’qS r#   r#   ©r!   ÚcharÚphonemer#   r#   r$   Ú	<setcomp>©   ó    z%get_phoneme_labels.<locals>.<setcomp>©ÚsortedÚ
setdefaultr>   Úindex©ra   ÚlabelsÚchar2phonemesrc   rd   r#   r#   r$   Úget_phoneme_labels§   s
   rn   c                 C   sN   t dd„ | D ƒƒ}i }| D ]\}}| |g ¡ | |› d|› �¡¡ q||fS )z@Get "<char> <phoneme>" labels, plus char to candidate label ids.c                 S   s   h | ]\}}|› d |› �’qS )ú r#   rb   r#   r#   r$   re   ²   s    z*get_char_phoneme_labels.<locals>.<setcomp>ro   rg   rk   r#   r#   r$   Úget_char_phoneme_labels°   s
   $rp   c                   @   s¨   e Zd ZdZ			ddee deeee f dee dee d	ee d
ede	e deddfdd„Z
edefdd„ƒZdd„ Zdefdd„Zdedeeef fdd„ZdS )Ú_FeatureBuilderzÄBuilds one set of model inputs per (text, query char).

    Kept as a class with ``__len__``/``__getitem__`` so upstream's recursive
    skip-on-unusable-text behaviour ports across directly.
    FNé   rl   rm   ÚcharsÚtextsÚ	query_idsr   r   Úmax_lenr   c
           
      C   s\   || _ || _|| _|| _|| _|| _|| _|| _|	| _|d ur,|  	|||¡\| _
| _d S d S r   )rO   rl   rm   rs   rt   ru   r   r   rv   Ú_truncate_textsÚtruncated_textsÚtruncated_query_ids)
ÚselfrO   rl   rm   rs   rt   ru   r   r   rv   r#   r#   r$   Ú__init__Å   s   ÿÿz_FeatureBuilder.__init__c           	      C   sl   g }g }t ||ƒD ](\}}td|| d  ƒ}tt|ƒ|| d  ƒ}| |||… ¡ | || ¡ q	||fS )z=Keep a window of characters centered on each query character.r   é   )rJ   ÚmaxÚminr:   r>   )	r   rt   ru   rx   ry   r8   Úquery_idÚstartÚendr#   r#   r$   rw   à   s   z_FeatureBuilder._truncate_textsc                    sì   |d }t |ƒ|kr|||||fS || }||d  ‰ˆ| }	ˆ }
|	t |ƒ }|
dkr5ˆ|
7 ‰|	|
7 }	n|dkrAˆ|8 ‰|	|8 }	|ˆ d ‰ ||	d  d }|ˆ |… |ˆ  |ˆ|	… ‡fdd„|ˆ |… D ƒ‡ fdd„|ˆ|	… D ƒfS )zCCenter the token window on the query when the text exceeds max_len.r|   r   r9   c                    s    g | ]}|d ur|ˆ  nd ‘qS r   r#   ©r!   r\   )r]   r#   r$   r*     ó     z-_FeatureBuilder._truncate.<locals>.<listcomp>c                    s    g | ]\}}|ˆ  |ˆ  f‘qS r#   r#   )r!   ÚsÚe)r€   r#   r$   r*     rƒ   ©r:   )rz   rv   r8   r   rR   Ú
text2tokenÚ
token2textÚtruncate_lenÚtoken_positionr^   Úfont_exceed_distÚback_exceed_distr�   r#   )r€   r]   r$   Ú	_truncateì   s,   


ûz_FeatureBuilder._truncatec                 C   s
   t | jƒS r   )r:   rt   ©rz   r#   r#   r$   Ú__len__
  s   
z_FeatureBuilder.__len__Úidxc           	         s$  ˆj rˆjnˆj|  ¡ }ˆj rˆjnˆj| }ztˆj|ƒ\}}}W n ty<   t	 
d|¡ ˆ|d tˆƒ   Y S w ˆ ˆj|||||¡\}}}}}dg| dg }|| ‰ ˆjrk‡ ‡fdd„ttˆjƒƒD ƒndgtˆjƒ }ˆj |¡dgt|ƒ dgt|ƒ |ˆj ˆ ¡|| d dœS )	NzSkipping unusable text: %sr9   z[CLS]z[SEP]c                    s"   g | ]}|ˆj ˆ  v rd nd‘qS )r9   r   )rm   r‚   ©Ú
query_charrz   r#   r$   r*   #  s    ÿÿz/_FeatureBuilder.__getitem__.<locals>.<listcomp>r   )Ú	input_idsÚtoken_type_idsÚattention_maskÚphoneme_maskÚchar_idÚposition_id)r   rx   rt   Úlowerry   ru   r`   rO   Ú	ExceptionÚ_LOGGERÚwarningr:   r�   rv   r   rN   rl   Úconvert_tokens_to_idsrs   rj   )	rz   r�   r8   r   rR   r‡   rˆ   Úprocessed_tokensr–   r#   r‘   r$   Ú__getitem__  s8   ÿüÿüþú



ùz_FeatureBuilder.__getitem__)FNrr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r0   ÚstrÚdictÚintÚboolr   r{   Ústaticmethodrw   r�   r�   r   rŸ   r#   r#   r#   r$   rq   ¾   s:    öýüûúùø	÷
ö
õrq   Ú	sequencesc                 C   sT   t dd„ | D ƒƒ}tjt| ƒ|ftjd�}t| ƒD ]\}}|||dt|ƒ…f< q|S )zÏRight-pad variable-length int sequences with 0 into one int64 array.

    Equivalent to ``torch.nn.utils.rnn.pad_sequence(..., batch_first=True)`` over
    ``torch.tensor``-wrapped lists of Python ints.
    c                 s   s   � | ]}t |ƒV  qd S r   r†   )r!   Úsequencer#   r#   r$   r%   <  s   € z_pad_stack.<locals>.<genexpr>©ÚdtypeN)r}   ÚnpÚzerosr:   Úint64rM   )r©   rv   Úpaddedr\   rª   r#   r#   r$   Ú
_pad_stack6  s
   r±   Úsamplesc              	   C   s~   t dd„ | D ƒƒt dd„ | D ƒƒt dd„ | D ƒƒtjdd„ | D ƒtjd�tjdd„ | D ƒtjd�tjdd„ | D ƒtjd�d	œS )
zDBuild the ONNX feed dict for one batch (replaces create_mini_batch).c                 S   ó   g | ]}|d  ‘qS )r“   r#   ©r!   r„   r#   r#   r$   r*   F  rf   z_collate.<locals>.<listcomp>c                 S   r³   )r”   r#   r´   r#   r#   r$   r*   G  rf   c                 S   r³   )r•   r#   r´   r#   r#   r$   r*   H  rf   c                 S   r³   )r–   r#   r´   r#   r#   r$   r*   J  rf   r«   c                 S   r³   )r—   r#   r´   r#   r#   r$   r*   L  rf   c                 S   r³   )r˜   r#   r´   r#   r#   r$   r*   M  rf   )r“   r”   r•   r–   Úchar_idsÚposition_ids)r±   r­   ÚasarrayÚfloat32r¯   )r²   r#   r#   r$   Ú_collateC  s   ÿør¹   c                   @   sä   e Zd ZdZ					ddeeejf dedee dee	 d	e
d
eeeejf  ddfdd„Zededeeef fdd„ƒZdedee fdd„Zdedefdd„Zdeeee f deeee   fdd„Zdee fdd„ZdS )ÚG2PWOnnxConverteraG  Grapheme-to-phoneme converter for Chinese, compatible with g2pW's API.

    ``model_dir`` must already contain ``g2pw.onnx``, ``config.py``,
    ``POLYPHONIC_CHARS.txt`` and ``MONOPHONIC_CHARS.txt``. Unlike upstream this
    never downloads anything; :func:`piper.phonemize_chinese.download_model`
    fetches the archive.
    ÚbopomofoNFr   Ústyler   r   Úenable_non_tradional_chineseÚdata_dirr   c                 C   sâ  t |ƒ}|durt |ƒnt|ƒ}t ¡ }tjj|_tjj|_	d|_
tjt|d ƒ|d�| _|  |d ¡| _|p;| jd | _|pC| jd | _dd	lm} | | j¡| _d
d„ t|d ƒD ƒ| _dd„ t|d ƒD ƒ| _| jd rqtnt}	|	| jƒ\| _| _t| j ¡ ƒ| _t | _!t"|d ddd��}
t# $|
¡| _%W d  ƒ n1 s¡w   Y  dd„ | j&dœ| | _'t"|d ddd��}
t# $|
¡| _(W d  ƒ n1 sËw   Y  || _)i | _*|rít|d ƒD ]}| +d¡\}}|| j*|< qÞdS dS )zInitialize converter.Nr|   z	g2pw.onnx)Úsess_optionsz	config.pyr   r   r   )ÚBertTokenizerc                 S   ó   g | ]}|  d ¡‘qS ©ú	©Úsplit©r!   Úliner#   r#   r$   r*   ‚  ó    
ÿz.G2PWOnnxConverter.__init__.<locals>.<listcomp>zPOLYPHONIC_CHARS.txtc                 S   rÁ   rÂ   rÄ   rÆ   r#   r#   r$   r*   …  rÈ   zMONOPHONIC_CHARS.txtr   r   Úrúutf-8©Úencodingc                 S   s   | S r   r#   )Úxr#   r#   r$   Ú<lambda>˜  s    z,G2PWOnnxConverter.__init__.<locals>.<lambda>)r»   Úpinyinr   r   rÃ   ),r   r7   ÚonnxruntimeÚSessionOptionsÚGraphOptimizationLevelÚORT_ENABLE_ALLÚgraph_optimization_levelÚExecutionModeÚORT_SEQUENTIALÚexecution_modeÚintra_op_num_threadsÚInferenceSessionr¤   Úsession_g2pwÚ_load_configÚconfigr   r   ÚtransformersrÀ   Úfrom_pretrainedrO   Ú_read_linesra   Úmonophonic_charsrp   rn   rl   rm   rh   Úkeysrs   ÚPOS_TAGSÚpos_tagsÚopenÚjsonÚloadÚbopomofo_convert_dictÚ_convert_bopomofo_to_pinyinÚstyle_convert_funcÚchar_bopomofo_dictr½   Ús2t_dictrÅ   )rz   r   r¼   r   r   r½   r¾   r¿   rÀ   Ú
label_funcÚ	json_filerÇ   Ús_charÚt_charr#   r#   r$   r{   _  sj   
ÿ
ÿ
ÿ
ÿÿý
ÿýþý
ÿýýzG2PWOnnxConverter.__init__Úconfig_pathc                    sZ   ddl }|j d| ¡}|dur|jdusJ ‚|j |¡‰ |j ˆ ¡ ‡ fdd„t ¡ D ƒS )z>Read the settings this path uses out of the model's config.py.r   NÚg2pw_configc                    s   i | ]\}}|t ˆ ||ƒ“qS r#   )r1   )r!   ÚkeyÚdefault©Úmoduler#   r$   Ú
<dictcomp>±  s    ÿÿz2G2PWOnnxConverter._load_config.<locals>.<dictcomp>)r-   r.   Úspec_from_file_locationÚloaderÚmodule_from_specÚexec_moduleÚ_CONFIG_DEFAULTSÚitems)rð   r3   r4   r#   rô   r$   rÛ   ¨  s   
þzG2PWOnnxConverter._load_configc                 C   sD   |d }|dv s
J ‚| j  |dd… ¡}|r|| S t d|¡ dS )z@Convert one bopomofo syllable to pinyin, keeping the tone digit.éÿÿÿÿÚ12345Nz%Cannot convert bopomofo to pinyin: %s)rç   Úgetr›   rœ   )rz   r»   ÚtoneÚ	componentr#   r#   r$   rè   ¶  s   z-G2PWOnnxConverter._convert_bopomofo_to_pinyinÚsentencec                    s   d  ‡ fdd„|D ƒ¡S )z.Map simplified characters to traditional ones.rI   c                 3   s   � | ]
}ˆ j  ||¡V  qd S r   )rë   rÿ   )r!   rc   rŽ   r#   r$   r%   Ã  s   € z1G2PWOnnxConverter._convert_s2t.<locals>.<genexpr>)Újoin)rz   r  r#   rŽ   r$   Ú_convert_s2tÁ  s   zG2PWOnnxConverter._convert_s2tÚ	sentencesc              
      s^  t |tƒr|g}ˆjr(g }|D ]}ˆ |¡}t|ƒt|ƒks J ‚| |¡ q|}ˆ |¡\}}}}|s5|S tˆjˆj	ˆj
ˆj||ˆjd ˆjd d�‰ g }	tdtˆ ƒˆjƒD ]3}
‡ fdd„t|
t|
ˆj tˆ ƒƒƒD ƒ}ˆj g t|ƒ¡d }|	‡fdd„tj|dd	� ¡ D ƒ7 }	qVˆjd
 r–dd„ |	D ƒ}	|}t|||	ƒD ]\}}}ˆ |¡|| |< qž|S )zCGet a phoneme (or None for punctuation) per character per sentence.r   r   )r   r   r   c                    s   g | ]}ˆ | ‘qS r#   r#   r‚   )Úfeaturesr#   r$   r*   ä  s    ÿÿz.G2PWOnnxConverter.__call__.<locals>.<listcomp>c                    s   g | ]}ˆ j | ‘qS r#   )rl   ©r!   ÚpredrŽ   r#   r$   r*   é  s    rý   )Úaxisr   c                 S   s   g | ]	}|  d ¡d ‘qS )ro   r9   rÄ   r  r#   r#   r$   r*   ì  s    )Ú
isinstancer¤   r½   r  r:   r>   Ú_prepare_datarq   rO   rl   rm   rs   rÜ   rN   r   r~   rÚ   Úrunr¹   r­   ÚargmaxÚtolistrJ   ré   )rz   r  Útranslated_sentencesr  Útranslated_sentencert   ru   Úsent_idsÚpartial_resultsÚpredsr€   ÚbatchÚprobsÚresultsÚsent_idr   r  r#   )r  rz   r$   Ú__call__Å  sF   

ø
þ&
zG2PWOnnxConverter.__call__c                 C   sÒ   t | jƒ}t| jƒ}g }g }g }g }t|ƒD ]L\}}	dgt|	ƒ }
t|	ƒD ]7\}}||v r=| |	¡ | |¡ | |¡ q%||v rK|  || ¡|
|< q%|| jv r\|  | j| d ¡|
|< q%| |
¡ q||||fS )zGResolve what the lookup tables can, and collect the rest for the model.Nr   )	Úsetrs   r¥   rà   rM   r:   r>   ré   rê   )rz   r  ra   Úmonophonic_chars_dictrt   ru   r  r  r  r  Úpartial_resultr\   rc   r#   r#   r$   r  ô  s0   




ÿ
ÿ€zG2PWOnnxConverter._prepare_data)r»   NNFN)r    r¡   r¢   r£   r   r¤   ÚosÚPathLiker   r¦   r§   r{   r¨   r   r¥   r   rÛ   rè   r  r0   r  r  r#   r#   r#   r$   rº   V  s8    ùþýüûúù
øI*/rº   Úpathc                 C   sD   t | ddd��}| ¡  ¡  d¡W  d  ƒ S 1 sw   Y  dS )z-Read a file and split it into stripped lines.rÉ   rÊ   rË   Ú
N)rä   ÚreadÚstriprÅ   )r  Ú	text_filer#   r#   r$   rß     s   $ÿrß   )$r£   rå   Úloggingr  r;   Úpathlibr   Útypingr   r   r   r   Únumpyr­   rÐ   Ú	getLoggerr    r›   râ   r,   rû   r7   r¤   rH   r`   rn   rp   rq   r0   r¦   Úndarrayr±   r¥   r¹   rº   rß   r#   r#   r#   r$   Ú<module>   s:    
	û	&)	x( ;