o
    èôjŸW  ã                   @   s¬  d dl Z d dl mZ d dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
 d dlmZ d dl Z d dlmZ d dl mZ d d	lmZ d
ededefdd„ZG dd„ dejƒZe
G dd„ dƒƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZde jfdeded ed!e jdef
d"d#„Zd$ed%edefd&d'„Zd(d)„ ZG d*d+„ d+ejƒZ G d,d-„ d-ejƒZ!G d.d/„ d/ejƒZ"G d0d1„ d1e jjƒZ#dS )2é    N)Únn)ÚWN)Úsequence_mask)Úweight_norm)Ú	dataclass)ÚOptional)ÚTensor)Ú
functionalÚnÚkÚreturnc                 C   s    | | dkr| S | | | |  S ©Nr   © )r
   r   r   r   úQ/var/www/html/academy/tools/voices/clone/seed-vc/modules/diffusion_transformer.pyÚfind_multiple   s   r   c                       s:   e Zd ZdZd
‡ fdd„Zddededefdd	„Z‡  ZS )ÚAdaptiveLayerNormzAdaptive Layer Normalizationr   Nc                    s:   t t| ƒ ¡  t |d| ¡| _|| _|| _| jj| _d S )Né   )	Úsuperr   Ú__init__r   ÚLinearÚproject_layerÚnormÚd_modelÚeps)Úselfr   r   ©Ú	__class__r   r   r   !   s
   zAdaptiveLayerNorm.__init__ÚinputÚ	embeddingc                 C   s@   |d u r	|   |¡S tj|  |¡| jdd�\}}||   |¡ | S )Néÿÿÿÿ)Úsplit_size_or_sectionsÚdim)r   ÚtorchÚsplitr   r   )r   r   r   ÚweightÚbiasr   r   r   Úforward(   s   

ýzAdaptiveLayerNorm.forward)r   N©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r&   Ú__classcell__r   r   r   r   r      s     r   c                   @   s¾   e Zd ZU dZeed< dZeed< dZeed< dZeed< dZ	eed	< d
Z
eed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dZeed< dd„ Zd
S )Ú	ModelArgsi   Ú
block_sizei }  Ú
vocab_sizeé    Ún_layerÚn_headi   r!   NÚintermediate_sizer   Ún_local_headsé@   Úhead_dimé'  Ú	rope_baseçñhãˆµøä>Únorm_epsFÚhas_cross_attentionr   Úcontext_dimÚuvit_skip_connectionÚtime_as_tokenc                 C   sJ   | j dkr	| j| _ | jd u r#d| j }td| d ƒ}t|dƒ| _d S d S )Nr   é   r   é   é   )r4   r2   r3   r!   Úintr   )r   Ú
hidden_dimÚn_hiddenr   r   r   Ú__post_init__D   s   


ýzModelArgs.__post_init__)r(   r)   r*   r.   rB   Ú__annotations__r/   r1   r2   r!   r3   r4   r6   r8   Úfloatr:   r;   Úboolr<   r=   r>   rE   r   r   r   r   r-   3   s    
 r-   c                       sŒ   e Zd Zdeddf‡ fdd„Zddd„Z					dd	ed
edee dee dee dee dee defdd„Ze	de
fdd„ƒZ‡  ZS )ÚTransformerÚconfigr   Nc                    sf   t ƒ  ¡  ˆ | _t ‡ fdd„tˆ jƒD ƒ¡| _tˆ j	t
ˆ j	ˆ jd�ƒ| _d | _d | _d| _d| _d S )Nc                 3   s   � | ]}t ˆ ƒV  qd S r'   )ÚTransformerBlock)Ú.0Ú_©rJ   r   r   Ú	<genexpr>R   s   € z'Transformer.__init__.<locals>.<genexpr>©r   r   )r   r   rJ   r   Ú
ModuleListÚranger1   Úlayersr   r!   ÚRMSNormr:   r   Ú	freqs_cisZ
mask_cacheÚmax_batch_sizeÚmax_seq_length©r   rJ   r   rN   r   r   N   s   
 
zTransformer.__init__Fc                    sþ   ˆ j |krˆ j|krd S ˆ jjˆ jj }t|dƒ}|ˆ _ |ˆ _ˆ jjjj	}ˆ jjjj
}tˆ jjˆ jjˆ jj|ƒ |¡ˆ _t tjˆ j ˆ j tjd�¡ |¡ˆ _|ˆ _ˆ jjˆ _ˆ jrw‡ fdd„tˆ jjƒD ƒˆ _‡ fdd„tˆ jjƒD ƒˆ _d S g ˆ _g ˆ _d S )Né   ©Údtypec                    s    g | ]}|ˆ j jd  k r|‘qS ©r   ©rJ   r1   ©rL   Úi©r   r   r   Ú
<listcomp>j   ó     z,Transformer.setup_caches.<locals>.<listcomp>c                    s    g | ]}|ˆ j jd  kr|‘qS r\   r]   r^   r`   r   r   ra   k   rb   )rW   rV   rJ   r!   r2   r   r   r   r$   r[   ÚdeviceÚprecompute_freqs_cisr.   r6   r8   ÚtorU   r"   ÚtrilÚonesrH   Úcausal_maskÚuse_kv_cacher=   rR   r1   Úlayers_emit_skipÚlayers_receive_skip)r   rV   rW   ri   r6   r[   rc   r   r`   r   Úsetup_cachesZ   s*   
ÿÿ$
 
zTransformer.setup_cachesÚxÚcÚ	input_posÚmaskÚcontextÚcontext_input_posÚcross_attention_maskc                 C   sð   | j d us	J dƒ‚|d u r*| js| jr| jd d |f }n| jd d |f }|d|f }| j | }|d ur9| j | }	nd }	g }
t| jƒD ]-\}}| jrT|| jv rT|
 d¡}nd }||||||||	||ƒ	}| jro|| j	v ro|
 
|¡ qB|  ||¡}|S )Nz Caches must be initialized first.r   )rU   Útrainingri   rh   Ú	enumeraterS   r=   rk   Úpoprj   Úappendr   )r   rm   rn   ro   rp   rq   rr   rs   rU   Úcontext_freqs_cisZskip_in_x_listr_   ÚlayerÚ	skip_in_xr   r   r   r&   p   s*   	

€zTransformer.forwardÚnamec                 C   s   | t  |¡ƒS r'   )r-   Ú	from_name)Úclsr{   r   r   r   r|   ‘   s   zTransformer.from_name©F)NNNNN)r(   r)   r*   r-   r   rl   r   r   r&   ÚclassmethodÚstrr|   r,   r   r   r   r   rI   M   s6    
ùÿþýüûúù
ø!rI   c                       sr   e Zd Zdeddf‡ fdd„Z				ddededed	ed
edee dee dee dee defdd„Z‡  ZS )rK   rJ   r   Nc                    sÄ   t ƒ  ¡  t|ƒ| _t|ƒ| _t|jt|j|j	d�ƒ| _
t|jt|j|j	d�ƒ| _|jrDd| _t|dd�| _t|jt|j|j	d�ƒ| _nd| _|jrYt |jd |j¡| _d| _nd| _|j| _d S )NrP   T)Úis_cross_attentionFr   )r   r   Ú	AttentionÚ	attentionÚFeedForwardÚfeed_forwardr   r!   rT   r:   Úffn_normÚattention_normr;   Úcross_attentionÚcross_attention_normr=   r   r   Úskip_in_linearr>   rX   r   r   r   r   —   s   


zTransformerBlock.__init__rm   rn   ro   rU   rp   rq   rx   rs   rz   c
              	   C   s�   | j rd n|}| jr|	d ur|  tj||	gdd�¡}||  |  ||¡|||¡ }
| jr;|
|  |  	|
|¡|||||¡ }
|
|  
|  |
|¡¡ }|S )Nr   ©r!   )r>   r=   rŠ   r"   Úcatrƒ   r‡   r;   rˆ   r‰   r…   r†   )r   rm   rn   ro   rU   rp   rq   rx   rs   rz   ÚhÚoutr   r   r   r&   ­   s    zTransformerBlock.forward)NNNN)	r(   r)   r*   r-   r   r   r   r&   r,   r   r   r   r   rK   –   s6    ÷ÿþýüûúùø	÷
örK   c                       sb   e Zd Zddedef‡ fdd„Z			ddeded	ed
ee dee dee defdd„Z‡  Z	S )r‚   FrJ   r�   c                    sÐ   t ƒ  ¡  |j|j dksJ ‚|jd|j  |j }|r;tj|j|j|j dd�| _tj|j	d|j |j dd�| _
n
tj|j|dd�| _tj|j|j |jdd�| _d | _|j| _|j| _|j| _|j| _d S )Nr   r   F©r%   )r   r   r!   r2   r4   r6   r   r   Úwqr<   ÚwkvÚwqkvÚwoÚkv_cache)r   rJ   r�   Ztotal_head_dimr   r   r   r   Ã   s   
"zAttention.__init__Nrm   rU   rp   ro   rq   rx   r   c                 C   sz  |j \}}}	| j| j }
|d u r#|  |¡j|
|
|
gdd�\}}}|}n|  |¡}|  |¡j|
|
gdd�\}}|j d }| ||| j| j¡}| ||| j| j¡}| ||| j| j¡}t	||ƒ}t	||d urf|n|ƒ}t
dd„ |||fƒ\}}}| jd ur…| j |||¡\}}|j| j| j dd�}|j| j| j dd�}tj||||dd�}| dd¡ ¡  ||| j| j ¡}|  |¡}|S )	Nr   r‹   é   c                 S   s   |   dd¡S )Nr•   r   )Ú	transpose)rm   r   r   r   Ú<lambda>ø   s    z#Attention.forward.<locals>.<lambda>g        )Ú	attn_maskÚ	dropout_pr   )Úshaper4   r6   r’   r#   r�   r‘   Úviewr2   Úapply_rotary_embÚmapr”   ÚupdateÚrepeat_interleaveÚFÚscaled_dot_product_attentionr–   Ú
contiguousr“   )r   rm   rU   rp   ro   rq   rx   ÚbszZseqlenrM   Zkv_sizeÚqr   ÚvZcontext_seqlenÚyr   r   r   r&   Þ   s,    



"
zAttention.forwardr~   )NNN)
r(   r)   r*   r-   rH   r   r   r   r&   r,   r   r   r   r   r‚   Â   s(    úÿþýüûúùr‚   c                       s8   e Zd Zdeddf‡ fdd„Zdedefdd„Z‡  ZS )	r„   rJ   r   Nc                    sP   t ƒ  ¡  tj|j|jdd�| _tj|j|jdd�| _tj|j|jdd�| _d S )NFr�   )	r   r   r   r   r!   r3   Úw1Úw3Úw2rX   r   r   r   r     s   
zFeedForward.__init__rm   c                 C   s    |   t |  |¡¡|  |¡ ¡S r'   )r©   r    Úsilur§   r¨   ©r   rm   r   r   r   r&     s    zFeedForward.forward)r(   r)   r*   r-   r   r   r&   r,   r   r   r   r   r„     s    r„   c                       sB   e Zd Zddedef‡ fdd„Zdd„ Zded	efd
d„Z‡  Z	S )rT   r9   r!   r   c                    s&   t ƒ  ¡  || _t t |¡¡| _d S r'   )r   r   r   r   Ú	Parameterr"   rg   r$   )r   r!   r   r   r   r   r     s   
zRMSNorm.__init__c                 C   s$   |t  t j|| ddd�| j ¡ S )Nr   T)r!   Úkeepdim)r"   ÚrsqrtÚmeanr   r«   r   r   r   Ú_norm  s   $zRMSNorm._normrm   r   c                 C   s   |   | ¡ ¡ |¡}|| j S r'   )r°   rG   Útype_asr$   )r   rm   Úoutputr   r   r   r&     s   
zRMSNorm.forward)r9   )
r(   r)   r*   rB   rG   r   r°   r   r&   r,   r   r   r   r   rT     s    rT   r7   Úseq_lenÚn_elemÚbaser[   c                 C   sz   d|t  d|d¡d |d …  ¡ |   }t j| |jd�}t  ||¡}t  t  |¡|¡}t j|j|j	gdd�}|j
|d�S )Ng      ð?r   r   )rc   r   r‹   rZ   )r"   ÚarangerG   rc   ÚouterÚpolarÚ	ones_likeÚstackÚrealÚimagre   )r³   r´   rµ   r[   ÚfreqsÚtrU   Úcacher   r   r   rd      s   *rd   rm   rU   c                 C   s    |   ¡ jg | jd d… ¢d‘d‘R Ž }| d| d¡d| d¡d¡}t |d |d  |d |d   |d |d  |d |d   gd¡}| d¡}| | ¡S )Nr   r   r•   r@   ).r   ).r•   )	rG   Úreshaperš   r›   Úsizer"   rº   Úflattenr±   )rm   rU   ZxshapedZx_out2r   r   r   rœ   ,  s   &þû

rœ   c                 C   s   | d|  d¡  |  d¡ S )Nr•   )Ú	unsqueeze)rm   ÚshiftÚscaler   r   r   Úmodulate;  s   rÆ   c                       s2   e Zd ZdZd	‡ fdd„	Zdd„ Zdd„ Z‡  ZS )
ÚTimestepEmbedderz>
    Embeds scalar timesteps into vector representations.
    rA   c              	      sŠ   t ƒ  ¡  t tj||dd�t ¡ tj||dd�¡| _|| _d| _d| _	|d }t
 t | j¡ t
jd|t
jd� | ¡}|  d|¡ d S )	NTr�   r7   iè  r   r   )ÚstartÚendr[   r½   )r   r   r   Ú
Sequentialr   ÚSiLUÚmlpÚfrequency_embedding_sizeZ
max_periodrÅ   r"   ÚexpÚmathÚlogr¶   Úfloat32Úregister_buffer)r   Úhidden_sizerÍ   Úhalfr½   r   r   r   r   G  s   
ý"ÿzTimestepEmbedder.__init__c                 C   sx   | j |dd…df  ¡  | jd  }tjt |¡t |¡gdd�}| jd r:tj|t |dd…dd…f ¡gdd�}|S )ai  
        Create sinusoidal timestep embeddings.
        :param t: a 1-D Tensor of N indices, one per batch element.
                          These may be fractional.
        :param dim: the dimension of the output.
        :param max_period: controls the minimum frequency of the embeddings.
        :return: an (N, D) Tensor of positional embeddings.
        Nr   r‹   r   r•   )	rÅ   rG   r½   r"   rŒ   ÚcosÚsinrÍ   Ú
zeros_like)r   r¾   Úargsr   r   r   r   Útimestep_embeddingX  s
   $
(z#TimestepEmbedder.timestep_embeddingc                 C   s   |   |¡}|  |¡}|S r'   )rÙ   rÌ   )r   r¾   Zt_freqZt_embr   r   r   r&   i  s   

zTimestepEmbedder.forward)rA   )r(   r)   r*   r+   r   rÙ   r&   r,   r   r   r   r   rÇ   C  s
    rÇ   c                       s*   e Zd ZdZ‡ fdd„Zddd„Z‡  ZS )ÚStyleEmbedderzs
    Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.
    c                    sJ   t ƒ  ¡  |dk}t t|ƒ|¡| _ttj||dd�ƒ| _|| _	|| _
d S )Nr   Tr�   )r   r   r   Ú	EmbeddingrB   Zembedding_tabler   r   Ústyle_inÚ
input_sizeÚdropout_prob)r   rÝ   rÓ   rÞ   Zuse_cfg_embeddingr   r   r   r   s  s   

zStyleEmbedder.__init__Nc                 C   s@   | j dk}|r	|s|d ur|  ||¡}|}|S |  |¡}|}|S r   )rÞ   Z
token_droprÜ   )r   ÚlabelsÚtrainZforce_drop_idsZuse_dropoutÚ
embeddingsr   r   r   r&   {  s   

þzStyleEmbedder.forwardr'   ©r(   r)   r*   r+   r   r&   r,   r   r   r   r   rÚ   o  s    rÚ   c                       s(   e Zd ZdZ‡ fdd„Zdd„ Z‡  ZS )Ú
FinalLayerz!
    The final layer of DiT.
    c                    s`   t ƒ  ¡  tj|ddd�| _ttj||| | dd�ƒ| _t t 	¡ tj|d| dd�¡| _
d S )NFg�íµ ÷Æ°>)Úelementwise_affiner   Tr�   r   )r   r   r   Ú	LayerNormÚ
norm_finalr   r   ÚlinearrÊ   rË   ÚadaLN_modulation)r   rÓ   Z
patch_sizeÚout_channelsr   r   r   r   ˆ  s   

þzFinalLayer.__init__c                 C   s8   |   |¡jddd�\}}t|  |¡||ƒ}|  |¡}|S )Nr   r•   r‹   )rè   ÚchunkrÆ   ræ   rç   )r   rm   rn   rÄ   rÅ   r   r   r   r&   ‘  s   
zFinalLayer.forwardrâ   r   r   r   r   rã   „  s    	rã   c                       s.   e Zd Z‡ fdd„Zdd„ Zddd„Z‡  ZS )	ÚDiTc              
      s  t t| ƒ ¡  t|jdƒr|jjnd| _t|jdƒr|jjnd| _t|jdƒr+|jjnd| _td|jj|jj	|jj
|jj
|jj	 d| j| jd�}t|ƒ| _|jj| _|jj| _|jj	| _	ttj|jj|jj
dd	�ƒ| _|jj| _|jj| _|jj| _t |jj|jj
¡| _tj|jj|jj
dd	�| _|jj| _t|jj
ƒ| _t d¡}|  d
|¡ |jj| _| jdk�rt|j j
ƒ| _!t |jj
|j j
¡| _"t #|j j
|jjd¡| _$t%|j j
|j j&|j j'|j j(|j j
|j j)dd�| _ t*|j j
d|j j
ƒ| _+t |jj
|j j
¡| _,|j j-| _.|jj-|j j-k�sJ ‚nt /t |jj
|jj
¡t 0¡ t |jj
|jj¡¡| _1|jj-| _2|jj3| _3t d|jj
¡| _4|jj5| _5t |jj
|jj |jj
¡| _6t |jj
|jjd  |j7j8| j2 | j   |jj
¡| _9| j�rt |j7j8|jj
¡| _:d S d S )Nr>   FÚstyle_as_tokenr=   i @  i   )r.   r1   r2   r!   r6   r/   r=   r>   Tr�   ro   Úwavenetr•   )Úhidden_channelsÚkernel_sizeÚdilation_rateZn_layersZgin_channelsÚ	p_dropoutZcausalr   );r   rë   r   Úhasattrr>   rì   r=   r-   ÚdepthÚ	num_headsrC   rI   ÚtransformerÚin_channelsré   r   r   r   Z
x_embedderÚcontent_typeÚcontent_codebook_sizeZcontent_dimrÛ   Zcond_embedderÚcond_projectionÚ	is_causalrÇ   Ú
t_embedderr"   r¶   rÒ   Úfinal_layer_typerí   Út_embedder2Úconv1ÚConv1dÚconv2r   rï   rð   Ú
num_layersrñ   rã   Úfinal_layerÚres_projectionZstyle_conditionZwavenet_style_conditionrÊ   rË   Ú	final_mlpÚtransformer_style_conditionÚclass_dropout_probZcontent_mask_embedderÚlong_skip_connectionÚskip_linearZstyle_encoderr!   Úcond_x_merge_linearrÜ   )r   rØ   Z
model_argsro   r   r   r   r   ˜  s‚   ø










ú
ÿ
ý


ÿþÿzDiT.__init__c                 C   s   | j j||dd� d S )NF)ri   )rõ   rl   )r   rV   rW   r   r   r   rl   ä  s   zDiT.setup_cachesFc                 C   s¬  d}| j rt d¡| jk rd}| j s|rd}| j}	| ¡ \}
}}|  |¡}|	|ƒ}| dd¡}| dd¡}tj|||gdd�}| j	r\| j
s\tj||d d …d d d …f  d|d¡gdd�}|rp|d| jd …f d |d| jd …f< |  |¡}| j
r’|  |¡}|r„t |¡n|}tj| d¡|gdd�}| jr¡tj| d¡|gdd�}t|| j
 | j ƒ |j¡ d¡}| jd | d¡… }| jsÓ|d d …d d d …f  dd| d¡d¡nd }|  || d¡||¡}| jrí|d d …dd …f n|}| j
rü|d d …dd …f n|}| j�r|  tj||gdd�¡}| jd	k�rI|  |¡}| dd¡}|  |¡}| j||| d¡d
� dd¡|  |¡ }|  ||¡ dd¡}|   |¡}|S |  !|¡}| dd¡}|S )NFr•   Tr   r   r‹   .r   rí   )Úg)"rt   r"   Úrandr  rù   rÁ   rû   r–   rŒ   r  rì   Úrepeatrö   r	  rÜ   r×   rÃ   r>   r   re   rc   ro   rú   rõ   r  r  rü   rþ   rý   rí   r  r  r   r  )r   rm   Úprompt_xÚx_lensr¾   ÚstyleÚcondZmask_contentZclass_dropoutZcond_in_moduleÚBrM   ÚTÚt1Zx_inZx_maskro   Zx_mask_expandedZx_resÚt2r   r   r   r&   æ  sX   

.$

"2

 ÿ

þzDiT.forwardr~   )r(   r)   r*   r   rl   r&   r,   r   r   r   r   rë   —  s    Lrë   )$r"   r   rÏ   Zmodules.wavenetr   Úmodules.commonsr   Ztorch.nn.utilsr   Údataclassesr   Útypingr   Útorch.nnr   r	   r    rB   r   ÚModuler   r-   rI   rK   r‚   r„   rT   Úbfloat16r[   rd   rœ   rÆ   rÇ   rÚ   rã   rë   r   r   r   r   Ú<module>   sP    I,Eþÿÿÿþ
ý,