o
    5ôj_*  ã                   @   s6  d dl Z d dlZd dlmZ d dlmZmZmZ d dlZ	d dl
Z
d dlm  mZ d dl
mZmZ ddlmZ ddlmZ ddlmZ eG d	d
„ d
ƒƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZddd„ZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZ G dd„ dejƒZ!dS )é    N)Ú	dataclass)ÚDictÚIterableÚOptional)ÚTensorÚnné   )Údecode)Údetect_language)Ú
transcribec                   @   s^   e Zd ZU eed< eed< eed< eed< eed< eed< eed< eed< eed	< eed
< dS )ÚModelDimensionsÚn_melsÚn_audio_ctxÚn_audio_stateÚn_audio_headÚn_audio_layerÚn_vocabÚ
n_text_ctxÚn_text_stateÚn_text_headÚn_text_layerN)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__© r   r   ú`/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/whisper/model.pyr      s   
 r   c                       s&   e Zd Zdedef‡ fdd„Z‡  ZS )Ú	LayerNormÚxÚreturnc                    s   t ƒ  | ¡ ¡ |j¡S ©N)ÚsuperÚforwardÚfloatÚtypeÚdtype©Úselfr   ©Ú	__class__r   r   r#      s   zLayerNorm.forward)r   r   r   r   r#   Ú__classcell__r   r   r)   r   r      s    r   c                   @   s   e Zd Zdedefdd„ZdS )ÚLinearr   r    c                 C   s2   t  || j |j¡| jd u rd ¡S | j |j¡¡S r!   )ÚFÚlinearÚweightÚtor&   Úbiasr'   r   r   r   r#   $   s   ýýzLinear.forwardN)r   r   r   r   r#   r   r   r   r   r,   #   s    r,   c                       s2   e Zd Zdededee def‡ fdd„Z‡  ZS )ÚConv1dr   r/   r1   r    c                    s.   t ƒ  || |j¡|d u rd ¡S | |j¡¡S r!   )r"   Ú_conv_forwardr0   r&   )r(   r   r/   r1   r)   r   r   r3   -   s
   ÿ
ÿzConv1d._conv_forward)r   r   r   r   r   r3   r+   r   r   r)   r   r2   ,   s    ÿÿÿþr2   é'  c                 C   sˆ   |d dksJ ‚t  |¡|d d  }t | t |d ¡ ¡}t | ¡dd…t jf |t jdd…f  }tjt |¡t |¡gdd�S )z*Returns sinusoids for positional embeddingé   r   r   N©Údim)	ÚnpÚlogÚtorchÚexpÚarangeÚnewaxisÚcatÚsinÚcos)ÚlengthÚchannelsZmax_timescaleZlog_timescale_incrementZinv_timescalesZscaled_timer   r   r   Ú	sinusoids5   s
   *rC   c                
       sv   e Zd Zdedef‡ fdd„Z			ddedee dee d	ee fd
d„Z	ddedededee fdd„Z	‡  Z
S )ÚMultiHeadAttentionÚn_stateÚn_headc                    sH   t ƒ  ¡  || _t||ƒ| _t||dd�| _t||ƒ| _t||ƒ| _d S )NF)r1   )r"   Ú__init__rF   r,   ÚqueryÚkeyÚvalueÚout)r(   rE   rF   r)   r   r   rG   ?   s   
zMultiHeadAttention.__init__Nr   ÚxaÚmaskÚkv_cachec           
      C   sˆ   |   |¡}|d u s|d u s| j|vr)|  |d u r|n|¡}|  |d u r%|n|¡}n
|| j }|| j }|  ||||¡\}}	|  |¡|	fS r!   )rH   rI   rJ   Úqkv_attentionrK   )
r(   r   rL   rM   rN   ÚqÚkÚvZwvÚqkr   r   r   r#   G   s   


zMultiHeadAttention.forwardrP   rQ   rR   c                 C   s  |j \}}}|| j d }|jg |j d d… ¢| j‘d‘R Ž  dddd¡| }|jg |j d d… ¢| j‘d‘R Ž  dddd¡| }|jg |j d d… ¢| j‘d‘R Ž  dddd¡}|| }	|d urm|	|d |…d |…f  }	|	 ¡ }	tj|	dd� |j¡}
|
|  dddd¡j	dd�|	 
¡ fS )	Ng      Ð¿r5   éÿÿÿÿr   r   é   r6   )Ú	start_dim)ÚshaperF   ÚviewÚpermuter$   r-   Úsoftmaxr0   r&   ÚflattenÚdetach)r(   rP   rQ   rR   rM   Ún_batchÚn_ctxrE   ÚscalerS   Úwr   r   r   rO   ]   s   440$z MultiHeadAttention.qkv_attention©NNNr!   )r   r   r   r   rG   r   r   Údictr#   rO   r+   r   r   r)   r   rD   >   s0    ûþýü
ûÿÿÿÿÿrD   c                
       sZ   e Zd Zddededef‡ fdd„Z			dded	ee d
ee dee fdd„Z	‡  Z
S )ÚResidualAttentionBlockFrE   rF   Úcross_attentionc                    s|   t ƒ  ¡  t||ƒ| _t|ƒ| _|rt||ƒnd | _|r t|ƒnd | _|d }t 	t
||ƒt ¡ t
||ƒ¡| _t|ƒ| _d S )Né   )r"   rG   rD   Úattnr   Úattn_lnÚ
cross_attnÚcross_attn_lnr   Ú
Sequentialr,   ÚGELUÚmlpÚmlp_ln)r(   rE   rF   rd   Zn_mlpr)   r   r   rG   p   s   

ÿÿzResidualAttentionBlock.__init__Nr   rL   rM   rN   c                 C   sZ   || j |  |¡||d�d  }| jr!|| j|  |¡||d�d  }||  |  |¡¡ }|S )N©rM   rN   r   )rN   )rf   rg   rh   ri   rl   rm   )r(   r   rL   rM   rN   r   r   r   r#   �   s
   zResidualAttentionBlock.forward)Fra   )r   r   r   r   ÚboolrG   r   r   rb   r#   r+   r   r   r)   r   rc   o   s    ûþýüûrc   c                
       s@   e Zd Zdededededef
‡ fdd„Zdefd	d
„Z‡  ZS )ÚAudioEncoderr   r^   rE   rF   Ún_layerc                    sp   t ƒ  ¡  t|ˆddd�| _tˆˆdddd�| _|  dt|ˆƒ¡ t ‡ ‡fdd„t	|ƒD ƒ¡| _
tˆƒ| _d S )	NrU   r   )Úkernel_sizeÚpaddingr5   )rr   Ústriders   Úpositional_embeddingc                    s   g | ]}t ˆˆ ƒ‘qS r   ©rc   ©Ú.0Ú_©rF   rE   r   r   Ú
<listcomp>™   s    z)AudioEncoder.__init__.<locals>.<listcomp>)r"   rG   r2   Úconv1Úconv2Úregister_bufferrC   r   Ú
ModuleListÚrangeÚblocksr   Úln_post)r(   r   r^   rE   rF   rq   r)   rz   r   rG   �   s   
ÿzAudioEncoder.__init__r   c                 C   s€   t  |  |¡¡}t  |  |¡¡}| ddd¡}|jdd… | jjks&J dƒ‚|| j  |j¡}| j	D ]}||ƒ}q2|  
|¡}|S )zt
        x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)
            the mel spectrogram of the audio
        r   r5   r   Nzincorrect audio shape)r-   Úgelur|   r}   rY   rW   ru   r0   r&   r�   r‚   )r(   r   Úblockr   r   r   r#   �   s   


zAudioEncoder.forward)r   r   r   r   rG   r   r#   r+   r   r   r)   r   rp   �   s    ÿÿÿÿÿrp   c                
       sN   e Zd Zdededededef
‡ fdd„Zdd	ed
edee fdd„Z‡  Z	S )ÚTextDecoderr   r^   rE   rF   rq   c                    s†   t ƒ  ¡  t |ˆ¡| _t t |ˆ¡¡| _t 	‡ ‡fdd„t
|ƒD ƒ¡| _tˆƒ| _t ||¡ tj ¡ d¡}| jd|dd� d S )Nc                    s   g | ]	}t ˆˆ d d�‘qS )T)rd   rv   rw   rz   r   r   r{   º   s    ÿÿz(TextDecoder.__init__.<locals>.<listcomp>r   rM   F©Ú
persistent)r"   rG   r   Ú	EmbeddingÚtoken_embeddingÚ	Parameterr:   Úemptyru   r   r€   r�   r   ÚlnÚfill_r8   ÚinfÚtriu_r~   )r(   r   r^   rE   rF   rq   rM   r)   rz   r   rG   ±   s   
þÿ
zTextDecoder.__init__Nr   rL   rN   c                 C   sš   |rt t| ¡ ƒƒjd nd}|  |¡| j|||jd  …  }| |j¡}| jD ]}|||| j	|d�}q)|  
|¡}|t | jj |j¡dd¡  ¡ }|S )zç
        x : torch.LongTensor, shape = (batch_size, <= n_ctx)
            the text tokens
        xa : torch.Tensor, shape = (batch_size, n_audio_ctx, n_audio_state)
            the encoded audio features to be attended on
        r   r   rT   rn   )ÚnextÚiterÚvaluesrW   r‰   ru   r0   r&   r�   rM   rŒ   r:   Ú	transposer/   r$   )r(   r   rL   rN   Úoffsetr„   Úlogitsr   r   r   r#   Ä   s   ÿÿ

þzTextDecoder.forwardr!   )
r   r   r   r   rG   r   r   rb   r#   r+   r   r   r)   r   r…   °   s    ÿÿÿÿÿ$r…   c                       s¾   e Zd Zdef‡ fdd„Zdefdd„Zdejfdd	„Z	d
ejdejfdd„Z
dejd
ejdeeejf fdd„Zedd„ ƒZedd„ ƒZedd„ ƒZddee fdd„ZeZeZeZ‡  ZS )ÚWhisperÚdimsc                    s¢   t ƒ  ¡  || _t| jj| jj| jj| jj| jjƒ| _	t
| jj| jj| jj| jj| jjƒ| _tj| jj| jjtjd�}d|| jjd d …< | jd| ¡ dd� d S )N©r&   Tr5   Úalignment_headsFr†   )r"   rG   r—   rp   r   r   r   r   r   Úencoderr…   r   r   r   r   r   Údecoderr:   Úzerosro   r~   Ú	to_sparse)r(   r—   Z	all_headsr)   r   r   rG   Þ   s*   
ûû	ÿzWhisper.__init__Údumpc                 C   sP   t jt t |¡¡td� ¡ }t 	|¡ 
| jj| jj¡}| jd| ¡ dd� d S )Nr˜   r™   Fr†   )r8   Ú
frombufferÚgzipÚ
decompressÚbase64Ú	b85decodero   Úcopyr:   Ú
from_numpyÚreshaper—   r   r   r~   r�   )r(   rž   ÚarrayrM   r   r   r   Úset_alignment_heads÷   s   ÿþ
ÿzWhisper.set_alignment_headsÚmelc                 C   s
   |   |¡S r!   )rš   )r(   r©   r   r   r   Úembed_audio   s   
zWhisper.embed_audioÚtokensÚaudio_featuresc                 C   s   |   ||¡S r!   )r›   )r(   r«   r¬   r   r   r   r•     s   zWhisper.logitsr    c                 C   s   |   ||  |¡¡S r!   )r›   rš   )r(   r©   r«   r   r   r   r#     s   zWhisper.forwardc                 C   s   t |  ¡ ƒjS r!   )r�   Ú
parametersÚdevice©r(   r   r   r   r®     s   zWhisper.devicec                 C   s   | j jdkS )Ni™Ê  )r—   r   r¯   r   r   r   Úis_multilingual  s   zWhisper.is_multilingualc                 C   s   | j jd t| jƒ S )Ni5Ê  )r—   r   r   r°   r¯   r   r   r   Únum_languages  s   zWhisper.num_languagesNÚcachec                    sP   ˆ duri ˆ ¥ni ‰ g ‰‡ ‡fdd„‰dt jf‡‡fdd„}ˆj |¡ ˆ ˆfS )a˜  
        The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value
        tensors calculated for the previous positions. This method returns a dictionary that stores
        all caches, and the necessary hooks for the key and value projection modules that save the
        intermediate tensors to be reused during later calculations.

        Returns
        -------
        cache : Dict[nn.Module, torch.Tensor]
            A dictionary object mapping the key/value projection modules to its cache
        hooks : List[RemovableHandle]
            List of PyTorch RemovableHandle objects to stop the hooks to be called
        Nc                    sP   | ˆ vs|j d ˆjjkr|ˆ | < ˆ |  S tjˆ |  |gdd� ¡ ˆ | < ˆ |  S )Nr   r6   )rW   r—   r   r:   r>   r\   )Úmodulery   Úoutput)r²   r(   r   r   Úsave_to_cache(  s
   ÿz5Whisper.install_kv_cache_hooks.<locals>.save_to_cacheÚlayerc                    s6   t | tƒrˆ  | j ˆ¡¡ ˆ  | j ˆ¡¡ d S d S r!   )Ú
isinstancerD   ÚappendrI   Úregister_forward_hookrJ   )r¶   )Úhooksrµ   r   r   Úinstall_hooks0  s   
þz5Whisper.install_kv_cache_hooks.<locals>.install_hooks)r   ÚModuler›   Úapply)r(   r²   r»   r   )r²   rº   rµ   r(   r   Úinstall_kv_cache_hooks  s   zWhisper.install_kv_cache_hooksr!   )r   r   r   r   rG   Úbytesr¨   r:   r   rª   r•   r   Ústrr#   Úpropertyr®   r°   r±   r   rb   r¾   Údetect_language_functionr
   Útranscribe_functionr   Údecode_functionr	   r+   r   r   r)   r   r–   Ý   s,    	ÿÿ
þ


!r–   )r4   )"r¢   r    Údataclassesr   Útypingr   r   r   Únumpyr8   r:   Ztorch.nn.functionalr   Ú
functionalr-   r   Údecodingr	   rÄ   r
   rÂ   r   rÃ   r   r   r,   r2   rC   r¼   rD   rc   rp   r…   r–   r   r   r   r   Ú<module>   s,    	
		1 !-