o
    ²ƒðjÏ  ã                   @   s°   d Z ddlZddlmZ ddlmZmZ ddlZddl	m
Z
 eeƒjZdZdZdZed	ƒZh d
£Zeeeg d¢ƒƒZdddddddœZedƒZG dd„ deƒZG dd„ dƒZdS )zPython implementation of libtashkeel.

See: https://github.com/mush42/libtashkeel

Ported with the help of ChatGPT 2025-05-01.
é    N)ÚPath)ÚOptionalÚUnion)ÚInferenceSessionià.  Ú_ú#u   0123456789Ù Ù¡Ù¢Ù£Ù¤Ù¥Ù¦Ù§Ù¨Ù©>   õ   ÙŒõ   Ù�õ   ÙŽõ   Ù�õ   Ù�õ   Ù‘õ   Ù’)éR  iQ  iN  iO  iP  iK  iL  iM  u   Ù‘ÙŽu   Ù‘Ù‹u   Ù‘Ù�u   Ù‘ÙŒu   Ù‘Ù�u   Ù‘Ù�)u   ÙŽÙ‘u   Ù‹Ù‘u   Ù�Ù‘u   ÙŒÙ‘u   Ù�Ù‘u   Ù�Ù‘r   c                   @   s   e Zd ZdZdS )ÚTashkeelErrorzError for tashkeel.N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__© r   r   ú™/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/piper/tashkeel/__init__.pyr      s    r   c                   @   s|  e Zd ZdZefdeeef ddfdd„Zd*dede	e
 defd	d
„Zd*dedefdd„Zdee dee dedeee ee
 f fdd„Zdedee dee defdd„Zdedee dee dee
 de
defdd„Zdefdd„Z	d+dededeeee f fdd „Zdedeeee f fd!d"„Zdedee fd#d$„Zdee dee fd%d&„Zd'ee dee fd(d)„ZdS ),ÚTashkeelDiacritizerz0Add diacritics for Arabic text with libtashkeel.Ú	model_dirÚreturnNc                    sþ   t |ƒ}t|d ƒ| _t|d ddd��}t |¡| _W d  ƒ n1 s%w   Y  t|d ddd��}t |¡‰ dd	„ ˆ  ¡ D ƒ| _W d  ƒ n1 sMw   Y  ‡ fd
d„t	fD ƒ| _
t|d ddd��}t |¡| _W d  ƒ dS 1 sxw   Y  dS )zInitialize diacritizer.z
model.onnxzinput_id_map.jsonÚrzutf-8)ÚencodingNztarget_id_map.jsonc                 S   s   i | ]\}}||“qS r   r   )Ú.0ÚcÚir   r   r   Ú
<dictcomp>0   s    
ÿz0TashkeelDiacritizer.__init__.<locals>.<dictcomp>c                    s   h | ]}ˆ | ’qS r   r   ©r   r   ©Útarget_id_mapr   r   Ú	<setcomp>4   s    z/TashkeelDiacritizer.__init__.<locals>.<setcomp>zhint_id_map.json)r   r   ÚsessionÚopenÚjsonÚloadÚinput_id_mapÚitemsÚid_target_mapÚPADÚtarget_id_meta_charsÚhint_id_map)Úselfr   Úinput_id_map_fileÚtarget_id_map_fileÚhint_id_map_filer   r!   r   Ú__init__!   s0   
ÿý
ÿ

ÿü
ÿ"ýzTashkeelDiacritizer.__init__ÚtextÚtaskeen_thresholdc                 C   s
   |   |¡S )ú!Add diacritics using libtashkeel.)Ú
diacritize)r.   r3   r4   r   r   r   Ú__call__;   s   
zTashkeelDiacritizer.__call__c                 C   s®   |  ¡ }t|ƒtkrtdt› �ƒ‚|  |¡\}}| j|dd�\}}|  |¡}|  |¡}t|ƒ}|dkr5|S |  |||¡\}	}
|  	|	¡}|du rN|  
|||¡S |  ||||
|¡S )r5   zText length cannot exceed T)Únormalize_diacriticsr   N)ÚstripÚlenÚ
CHAR_LIMITr   Ú_to_valid_charsÚ_extract_chars_and_diacriticsÚ_input_to_idsÚ_hint_to_idsÚ_inferÚ_target_to_diacriticsÚ_annotate_text_with_diacriticsÚ&_annotate_text_with_diacritics_taskeen)r.   r3   r4   Ú
input_textÚremoved_charsÚ
diacriticsÚ	input_idsÚdiac_idsÚ
seq_lengthÚ
target_idsÚlogitsr   r   r   r6   ?   s&   
ÿ



ÿzTashkeelDiacritizer.diacritizerG   rH   rI   c                 C   sš   t j|t jd� d|¡}t j|t jd� d|¡}t j|gt jd� d¡}|||dœ}| j d|¡}|d  ¡  t j¡ 	¡ }	|d  ¡  t j
¡ 	¡ }
|	|
fS )zInfer target ids and logits.)Údtypeé   )Úchar_inputsÚdiac_inputsÚinput_lengthsNr   )ÚnpÚarrayÚint64Úreshaper$   ÚrunÚflattenÚastypeÚuint8ÚtolistÚfloat32)r.   rG   rH   rI   Úinput_ids_arrÚdiac_ids_arrÚinput_len_arrÚinputsÚoutputsrJ   rK   r   r   r   r@   \   s   ýzTashkeelDiacritizer._inferrD   rF   rE   c                 C   sZ   g }t |ƒ}|D ]}|  |¡rq||v r| |¡ q| |¡ | t|dƒ¡ qd |¡S ©NÚ )ÚiterÚ_is_diacritic_charÚappendÚnextÚjoin)r.   rD   rF   rE   ÚoutputÚ	diac_iterr   r   r   r   rB   s   s   


z2TashkeelDiacritizer._annotate_text_with_diacriticsrK   Ú	thresholdc                 C   sx   g }t ||ƒ}|D ]-}|  |¡rq	||v r| |¡ q	| |¡ t|dƒ\}	}
|
|kr1| t¡ q	| |	¡ q	d |¡S )N)ra   g        ra   )Úziprc   rd   re   ÚSUKOONrf   )r.   rD   rF   rE   rK   ri   rg   rh   r   ÚdiacÚlogitr   r   r   rC   „   s   



z:TashkeelDiacritizer._annotate_text_with_diacritics_taskeenc                 C   s   |t v S ©N)ÚARABIC_DIACRITICS)r.   r   r   r   r   rc   �   s   z&TashkeelDiacritizer._is_diacritic_charTr8   c           	      C   s´   |  d t¡¡}g }g }d}t|ƒdg D ]}|  |¡r!||7 }q| |¡ | |¡ d}q|r4| ¡  |r;| d¡ |rSt|ƒD ]\}}|| jvrRt	 
|d¡||< qAd |¡|fS )Nra   ú r   )Úlstriprf   ro   Úlistrc   rd   ÚpopÚ	enumerater-   ÚNORMALIZED_DIAC_MAPÚget)	r.   r3   r8   Úclean_charsrF   Úpending_diacr   r   Údr   r   r   r=       s(   





€z1TashkeelDiacritizer._extract_chars_and_diacriticsc                 C   s^   g }t ƒ }|D ] }|| jv s|tv r| |¡ q|tv r"| t¡ q| |¡ qd |¡|fS r`   )Úsetr(   ro   rd   ÚNUMERALSÚNUMERAL_SYMBOLÚaddrf   )r.   r3   ÚvalidÚinvalidr   r   r   r   r<   ½   s   z#TashkeelDiacritizer._to_valid_charsc                    ó   ‡ fdd„|D ƒS )Nc                    ó   g | ]}ˆ j | ‘qS r   )r(   r    ©r.   r   r   Ú
<listcomp>Ê   ó    z5TashkeelDiacritizer._input_to_ids.<locals>.<listcomp>r   )r.   r3   r   r‚   r   r>   É   ó   z!TashkeelDiacritizer._input_to_idsc                    r€   )Nc                    r�   r   )r-   )r   ry   r‚   r   r   rƒ   Í   r„   z4TashkeelDiacritizer._hint_to_ids.<locals>.<listcomp>r   )r.   rF   r   r‚   r   r?   Ì   r…   z TashkeelDiacritizer._hint_to_idsrJ   c                    r€   )Nc                    s    g | ]}|ˆ j vrˆ j| ‘qS r   )r,   r*   )r   r   r‚   r   r   rƒ   Ð   s
    
þz=TashkeelDiacritizer._target_to_diacritics.<locals>.<listcomp>r   )r.   rJ   r   r‚   r   rA   Ï   s   
þz)TashkeelDiacritizer._target_to_diacriticsrn   )T)r   r   r   r   ÚTASHKEEL_DIRr   Ústrr   r2   r   Úfloatr7   r6   rr   ÚintÚtupler@   rz   rB   rC   Úboolrc   r=   r<   r>   r?   rA   r   r   r   r   r      sb    ÿÿÿ
þÿÿÿ
þþýüûú
ùÿÿÿ
þr   )r   r&   Úpathlibr   Útypingr   r   ÚnumpyrQ   Úonnxruntimer   Ú__file__Úparentr†   r;   r+   r|   rz   r{   ÚHARAKAT_CHARSÚmapÚchrro   ru   rk   Ú	Exceptionr   r   r   r   r   r   Ú<module>   s"    
