o
    ›ôj_‚  ã                   @   sF  d dl mZmZmZ d dlZd dlm  mZ d dlmZ ddl	m
Z
mZ ddlmZ ddlmZmZmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZ e e¡Zdejdej de!de!fdd„Z"eG dd„ dejƒƒZ#eG dd„ dejƒƒZ$eG dd„ dejƒƒZ%eG dd„ dejƒƒZ&G dd„ dejƒZ'G dd„ dejƒZ(dS )é    )ÚAnyÚDictÚOptionalN)Únné   )Ú	deprecateÚlogging)Úmaybe_allow_in_graphé   )ÚGEGLUÚGELUÚApproximateGELU)Ú	AttentionÚJointAttnProcessor2_0)ÚSinusoidalPositionalEmbedding)ÚAdaLayerNormÚAdaLayerNormContinuousÚAdaLayerNormZeroÚRMSNormÚffÚhidden_statesÚ	chunk_dimÚ
chunk_sizec                    sf   |j | | dkrtd|j | › d|› d�ƒ‚|j | | }tj‡ fdd„|j||d�D ƒ|d�}|S )Nr   z)`hidden_states` dimension to be chunked: z$ has to be divisible by chunk size: z[. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.c                    s   g | ]}ˆ |ƒ‘qS © r   )Ú.0Ú	hid_slice©r   r   úm/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/diffusers/models/attention.pyÚ
<listcomp>(   s    z)_chunked_feed_forward.<locals>.<listcomp>©Údim)ÚshapeÚ
ValueErrorÚtorchÚcatÚchunk)r   r   r   r   Ú
num_chunksÚ	ff_outputr   r   r   Ú_chunked_feed_forward   s   ÿþr(   c                       sN   e Zd ZdZdedededef‡ fdd„Zdejd	ejd
ejfdd„Z‡  Z	S )ÚGatedSelfAttentionDenseat  
    A gated self-attention dense layer that combines visual features and object features.

    Parameters:
        query_dim (`int`): The number of channels in the query.
        context_dim (`int`): The number of channels in the context.
        n_heads (`int`): The number of heads to use for attention.
        d_head (`int`): The number of channels in each head.
    Ú	query_dimÚcontext_dimÚn_headsÚd_headc                    sˆ   t ƒ  ¡  t ||¡| _t|||d�| _t|dd�| _t 	|¡| _
t 	|¡| _|  dt t d¡¡¡ |  dt t d¡¡¡ d| _d S )N)r*   ÚheadsÚdim_headÚgeglu©Úactivation_fnÚ
alpha_attnç        Úalpha_denseT)ÚsuperÚ__init__r   ÚLinearÚlinearr   ÚattnÚFeedForwardr   Ú	LayerNormÚnorm1Únorm2Úregister_parameterÚ	Parameterr#   ÚtensorÚenabled)Úselfr*   r+   r,   r-   ©Ú	__class__r   r   r7   :   s   

z GatedSelfAttentionDense.__init__ÚxÚobjsÚreturnc              
   C   s‚   | j s|S |jd }|  |¡}|| j ¡ |  |  tj||gdd�¡¡d d …d |…d d …f   }|| j	 ¡ |  
|  |¡¡  }|S )Nr
   r   )rB   r!   r9   r3   Útanhr:   r=   r#   r$   r5   r   r>   )rC   rF   rG   Zn_visualr   r   r   ÚforwardK   s   

BzGatedSelfAttentionDense.forward)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr7   r#   ÚTensorrJ   Ú__classcell__r   r   rD   r   r)   .   s    
$r)   c                       sV   e Zd ZdZd‡ fdd„	Zddee defdd	„Zd
ej	dej	dej	fdd„Z
‡  ZS )ÚJointTransformerBlocka$  
    A Transformer block following the MMDiT architecture, introduced in Stable Diffusion 3.

    Reference: https://arxiv.org/abs/2403.03206

    Parameters:
        dim (`int`): The number of channels in the input and output.
        num_attention_heads (`int`): The number of heads to use for multi-head attention.
        attention_head_dim (`int`): The number of channels in each head.
        context_pre_only (`bool`): Boolean to determine if we should add some blocks associated with the
            processing of `context` conditions.
    Fc                    s
  t ƒ  ¡  || _|rdnd}t|ƒ| _|dkr#t||ddddd�| _n|dkr-t|ƒ| _ntd|› d	�ƒ‚tt	d
ƒr>t
ƒ }ntdƒ‚t|d ||| |||d|d�	| _tj|ddd�| _t||dd�| _|swtj|ddd�| _t||dd�| _nd | _d | _d | _d| _d S )NZada_norm_continousÚada_norm_zeroFç�íµ ÷Æ°>TÚ
layer_norm)Úelementwise_affineÚepsÚbiasÚ	norm_typezUnknown context_norm_type: z>, currently only support `ada_norm_continous`, `ada_norm_zero`Úscaled_dot_product_attentionzYThe current PyTorch version does not support the `scaled_dot_product_attention` function.)	r*   Úcross_attention_dimZadded_kv_proj_dimr/   r.   Úout_dimÚcontext_pre_onlyrX   Ú	processor©rV   rW   úgelu-approximate)r    Údim_outr2   r   )r6   r7   r]   r   r=   r   Únorm1_contextr"   ÚhasattrÚFr   r   r:   r   r<   r>   r;   r   Únorm2_contextÚ
ff_contextÚ_chunk_sizeÚ
_chunk_dim)rC   r    Únum_attention_headsÚattention_head_dimr]   Zcontext_norm_typer^   rD   r   r   r7   g   sL   


ÿ
ÿ
ÿ÷
zJointTransformerBlock.__init__r   r   r    c                 C   ó   || _ || _d S ©N©rg   rh   ©rC   r   r    r   r   r   Úset_chunk_feed_forwardš   ó   
z,JointTransformerBlock.set_chunk_feed_forwardr   Úencoder_hidden_statesÚtembc                 C   s|  | j ||d�\}}}}}| jr|  ||¡}	n| j||d�\}	}
}}}| j||	d�\}}| d¡| }|| }|  |¡}|d|d d …d f   |d d …d f  }| jd ur_t| j|| j	| jƒ}n|  |¡}| d¡| }|| }| jrxd }||fS |
 d¡| }|| }|  
|¡}	|	d|d d …d f   |d d …d f  }	| jd ur¬t| j|	| j	| jƒ}n|  |	¡}|| d¡|  }||fS )N)Úemb)r   rq   r
   )r=   r]   rb   r:   Ú	unsqueezer>   rg   r(   r   rh   re   rf   )rC   r   rq   rr   Únorm_hidden_statesÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlpZnorm_encoder_hidden_statesZ
c_gate_msaZc_shift_mlpZc_scale_mlpZ
c_gate_mlpÚattn_outputZcontext_attn_outputr'   Zcontext_ff_outputr   r   r   rJ   Ÿ   s@   ÿ
ÿ
(

ò
(
ÿ
zJointTransformerBlock.forward)F©r   )rK   rL   rM   rN   r7   r   rO   ro   r#   ÚFloatTensorrJ   rQ   r   r   rD   r   rR   X   s    3ÿÿÿrR   c                -       sN  e Zd ZdZ																					d1d
edededee dedee dedededededededededee dee dee dee dee dedef,‡ fd d!„Z	d2d#ee d
efd$d%„Z
							d3d&ejd'eej d(eej d)eej d*eej d+eeef d,eej d-eeeejf  d.ejfd/d0„Z‡  ZS )4ÚBasicTransformerBlockaä  
    A basic Transformer block.

    Parameters:
        dim (`int`): The number of channels in the input and output.
        num_attention_heads (`int`): The number of heads to use for multi-head attention.
        attention_head_dim (`int`): The number of channels in each head.
        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
        cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
        activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
        num_embeds_ada_norm (:
            obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`.
        attention_bias (:
            obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.
        only_cross_attention (`bool`, *optional*):
            Whether to use only cross-attention layers. In this case two cross attention layers are used.
        double_self_attention (`bool`, *optional*):
            Whether to use two self-attention layers. In this case no cross attention layers are used.
        upcast_attention (`bool`, *optional*):
            Whether to upcast the attention computation to float32. This is useful for mixed precision training.
        norm_elementwise_affine (`bool`, *optional*, defaults to `True`):
            Whether to use learnable elementwise affine parameters for normalization.
        norm_type (`str`, *optional*, defaults to `"layer_norm"`):
            The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`.
        final_dropout (`bool` *optional*, defaults to False):
            Whether to apply a final dropout after the last feed-forward layer.
        attention_type (`str`, *optional*, defaults to `"default"`):
            The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`.
        positional_embeddings (`str`, *optional*, defaults to `None`):
            The type of positional embeddings to apply to.
        num_positional_embeddings (`int`, *optional*, defaults to `None`):
            The maximum number of positional embeddings to apply.
    r4   Nr0   FTrU   çñhãˆµøä>Údefaultr    ri   rj   r[   r2   Únum_embeds_ada_normÚattention_biasÚonly_cross_attentionÚdouble_self_attentionÚupcast_attentionÚnorm_elementwise_affinerY   Únorm_epsÚfinal_dropoutÚattention_typeÚpositional_embeddingsÚnum_positional_embeddingsÚ-ada_norm_continous_conditioning_embedding_dimÚada_norm_biasÚff_inner_dimÚff_biasÚattention_out_biasc              
      sj  t ƒ  ¡  |	| _|d uo|dk| _|d uo|dk| _|dk| _|dk| _|dk| _|dv r<|d u r<td|› d|› d	�ƒ‚|| _	|| _
|rL|d u rLtd
ƒ‚|dkrXt||d�| _nd | _|dkrft||ƒ| _n#|dkrqt||ƒ| _n|dkr€t|||||dƒ| _n	tj|||d�| _t||||||	r“|nd ||d�| _|d us |
rÔ|dkr«t||ƒ| _n|dkrºt|||||dƒ| _nt |||¡| _t||
sÈ|nd ||||||d�| _nd | _d | _|dkrét|||||dƒ| _n|dv röt |||¡| _n|dkrýd | _t||||||d�| _|dk�s|dk�rt||||ƒ| _|dk�r-t t d|¡|d  ¡| _d | _ d| _!d S )NrS   Úada_normÚada_norm_singlerU   Úada_norm_continuous)r�   rS   z`norm_type` is set to zw, but `num_embeds_ada_norm` is not defined. Please make sure to define `num_embeds_ada_norm` if setting `norm_type` to Ú.z\If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined.Ú
sinusoidal)Úmax_seq_lengthZrms_normr_   )r*   r.   r/   ÚdropoutrX   r[   r„   Úout_bias)r*   r[   r.   r/   r–   rX   r„   r—   )rS   r�   rU   r’   Úlayer_norm_i2vgen)r–   r2   r‡   Ú	inner_dimrX   Úgatedzgated-text-imageé   g      à?r   )"r6   r7   r‚   Úuse_ada_layer_norm_zeroÚuse_ada_layer_normZuse_ada_layer_norm_singleÚuse_layer_normZuse_ada_layer_norm_continuousr"   rY   r€   r   Ú	pos_embedr   r=   r   r   r   r<   r   Úattn1r>   Úattn2Únorm3r;   r   r)   Úfuserr@   r#   ÚrandnÚscale_shift_tablerg   rh   )rC   r    ri   rj   r–   r[   r2   r€   r�   r‚   rƒ   r„   r…   rY   r†   r‡   rˆ   r‰   rŠ   r‹   rŒ   r�   rŽ   r�   rD   r   r   r7   ø   sÄ   



ÿÿÿú	
øú	

øú	ú


zBasicTransformerBlock.__init__r   r   c                 C   rk   rl   rm   rn   r   r   r   ro   ’  rp   z,BasicTransformerBlock.set_chunk_feed_forwardr   Úattention_maskrq   Úencoder_attention_maskÚtimestepÚcross_attention_kwargsÚclass_labelsÚadded_cond_kwargsrH   c	                 C   s|  |d ur|  dd ¡d urt d¡ |jd }	| jdkr"|  ||¡}
na| jdkr7| j||||jd�\}
}}}}nL| jdv rB|  |¡}
nA| jdkrP|  ||d	 ¡}
n3| jd
kr| jd  | |	dd¡ j	ddd�\}}}}}}|  |¡}
|
d|  | }
|
 
d¡}
ntdƒ‚| jd ur�|  |
¡}
|d ur•| ¡ ni }| dd ¡}| j|
f| jr¦|nd |dœ|¤Ž}| jdkr»| d¡| }n	| jd
krÄ|| }|| }|jdkrÒ| 
d¡}|d urÞ|  ||d ¡}| jd u�r8| jdkrð|  ||¡}
n'| jdv rû|  |¡}
n| jd
k�r|}
n| jdk�r|  ||d	 ¡}
ntdƒ‚| jd u�r(| jd
k�r(|  |
¡}
| j|
f||dœ|¤Ž}|| }| jdk�rG|  ||d	 ¡}
n| jd
k�sR|  |¡}
| jdk�rl|
d|d d …d f   |d d …d f  }
| jd
k�r|  |¡}
|
d|  | }
| jd u�r�t| j|
| j| jƒ}n|  |
¡}| jdk�r£| d¡| }n
| jd
k�r­|| }|| }|jdk�r¼| 
d¡}|S )NÚscalezSPassing `scale` to `cross_attention_kwargs` is deprecated. `scale` will be ignored.r   r�   rS   )Úhidden_dtype)rU   r˜   r’   Zpooled_text_embr‘   r›   éÿÿÿÿr
   r   zIncorrect norm usedZgligen)rq   r¦   é   rG   )rS   rU   r˜   zIncorrect norm)ÚgetÚloggerÚwarningr!   rY   r=   Údtyper¥   Úreshaper%   Úsqueezer"   rŸ   ÚcopyÚpopr    r‚   rt   Úndimr£   r¡   r>   r¢   rg   r(   r   rh   )rC   r   r¦   rq   r§   r¨   r©   rª   r«   Ú
batch_sizeru   rv   rw   rx   ry   Z	shift_msaZ	scale_msaZgligen_kwargsrz   r'   r   r   r   rJ   —  s¨   




ÿ


þ


ÿýü






ÿýü
(


zBasicTransformerBlock.forward)r4   Nr0   NFFFFTrU   r~   Fr   NNNNNTTr{   )NNNNNNN)rK   rL   rM   rN   rO   r   ÚstrÚboolÚfloatr7   ro   r#   rP   Ú
LongTensorr   r   rJ   rQ   r   r   rD   r   r}   Ô   s¾    'èþýüúùø	÷
öõôóòñðïîíìëêéè ÷þýüûú
ùø	÷
ör}   c                       sx   e Zd ZdZ	ddededededee f
‡ fdd	„Zd
ee fdd„Z	ddej	dedeej	 dej	fdd„Z
‡  ZS )ÚTemporalBasicTransformerBlocka÷  
    A basic Transformer block for video like data.

    Parameters:
        dim (`int`): The number of channels in the input and output.
        time_mix_inner_dim (`int`): The number of channels for temporal attention.
        num_attention_heads (`int`): The number of heads to use for multi-head attention.
        attention_head_dim (`int`): The number of channels in each head.
        cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention.
    Nr    Útime_mix_inner_dimri   rj   r[   c                    s¬   t ƒ  ¡  ||k| _t |¡| _t||dd�| _t |¡| _t	|||d d�| _
|d ur;t |¡| _t	||||d�| _nd | _d | _t |¡| _t|dd�| _d | _d | _d S )Nr0   )ra   r2   )r*   r.   r/   r[   )r*   r[   r.   r/   r1   )r6   r7   Úis_resr   r<   Únorm_inr;   Úff_inr=   r   r    r>   r¡   r¢   r   rg   rh   )rC   r    r¿   ri   rj   r[   rD   r   r   r7     s:   

ýü
ü
z&TemporalBasicTransformerBlock.__init__r   c                 K   s   || _ d| _d S )Nr
   rm   )rC   r   Úkwargsr   r   r   ro   T  s   
z4TemporalBasicTransformerBlock.set_chunk_feed_forwardr   Ú
num_framesrq   rH   c                 C   sv  |j d }|j \}}}|| }|d d d …f  ||||¡}| dddd¡}| || ||¡}|}|  |¡}| jd urEt| j|| j| jƒ}n|  |¡}| jrQ|| }|  	|¡}	| j
|	d d�}
|
| }| jd urv|  |¡}	| j|	|d�}
|
| }|  |¡}	| jd ur‹t| j|	| j| jƒ}n|  |	¡}| jr˜|| }n|}|d d d …f  ||||¡}| dddd¡}| || ||¡}|S )Nr   r   r
   é   )rq   )r!   r´   ÚpermuterÁ   rg   r(   rÂ   rh   rÀ   r=   r    r¡   r>   r¢   r   )rC   r   rÄ   rq   r¹   Zbatch_framesÚ
seq_lengthÚchannelsÚresidualru   rz   r'   r   r   r   rJ   Z  s>   










z%TemporalBasicTransformerBlock.forwardrl   )rK   rL   rM   rN   rO   r   r7   ro   r#   rP   rJ   rQ   r   r   rD   r   r¾     s2    úþýüûú5
üþýüûr¾   c                       sT   e Zd Z				ddedededed	ed
ee dedef‡ fdd„Zdd„ Z‡  ZS )ÚSkipFFTransformerBlockr4   NFTr    ri   rj   Úkv_input_dimÚkv_input_dim_proj_use_biasr[   r�   r�   c
           
   	      sv   t ƒ  ¡  ||krt |||¡| _nd | _t|dƒ| _t|||||||	d�| _t|dƒ| _	t|||||||	d�| _
d S )NrT   )r*   r.   r/   r–   rX   r[   r—   )r*   r[   r.   r/   r–   rX   r—   )r6   r7   r   r8   Ú	kv_mapperr   r=   r   r    r>   r¡   )
rC   r    ri   rj   rË   rÌ   r–   r[   r�   r�   rD   r   r   r7   •  s0   
ù
ùzSkipFFTransformerBlock.__init__c                 C   s‚   |d ur|  ¡ ni }| jd ur|  t |¡¡}|  |¡}| j|fd|i|¤Ž}|| }|  |¡}| j|fd|i|¤Ž}|| }|S )Nrq   )r¶   rÍ   rd   Úsilur=   r    r>   r¡   )rC   r   rq   r©   ru   rz   r   r   r   rJ   ¿  s,   

ÿþý
ÿþýzSkipFFTransformerBlock.forward)r4   NFT)	rK   rL   rM   rO   r»   r   r7   rJ   rQ   r   r   rD   r   rÊ   ”  s.    öþýüûúø	÷
ö*rÊ   c                       sh   e Zd ZdZ							dded	ee d
ededededef‡ fdd„Z	de
jde
jfdd„Z‡  ZS )r;   aª  
    A feed-forward layer.

    Parameters:
        dim (`int`): The number of channels in the input.
        dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`.
        mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension.
        dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.
        activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.
        final_dropout (`bool` *optional*, defaults to False): Apply a final dropout.
        bias (`bool`, defaults to True): Whether to use a bias in the linear layer.
    Nr¯   r4   r0   FTr    ra   Úmultr–   r2   r‡   rX   c	           
         sì   t ƒ  ¡  |d u rt|| ƒ}|d ur|n|}|dkr"t|||d�}	|dkr/t||d|d�}	n|dkr;t|||d�}	n|dkrFt|||d�}	t g ¡| _| j 	|	¡ | j 	t 
|¡¡ | j 	tj|||d�¡ |rt| j 	t 
|¡¡ d S d S )NÚgelu)rX   r`   rI   )ÚapproximaterX   r0   zgeglu-approximate)r6   r7   rO   r   r   r   r   Ú
ModuleListÚnetÚappendÚDropoutr8   )
rC   r    ra   rÏ   r–   r2   r‡   r™   rX   Úact_fnrD   r   r   r7   ê  s&   
ÿzFeedForward.__init__r   rH   c                 O   sD   t |ƒdks| dd ¡d urd}tdd|ƒ | jD ]}||ƒ}q|S )Nr   r¬   zðThe `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`.z1.0.0)Úlenr°   r   rÓ   )rC   r   ÚargsrÃ   Údeprecation_messageÚmoduler   r   r   rJ     s   

zFeedForward.forward)Nr¯   r4   r0   FNT)rK   rL   rM   rN   rO   r   r¼   rº   r»   r7   r#   rP   rJ   rQ   r   r   rD   r   r;   Ü  s2    ÷þýüûúù	÷$r;   ))Útypingr   r   r   r#   Útorch.nn.functionalr   Ú
functionalrd   Úutilsr   r   Zutils.torch_utilsr	   Úactivationsr   r   r   Zattention_processorr   r   Ú
embeddingsr   Únormalizationr   r   r   r   Ú
get_loggerrK   r±   ÚModulerP   rO   r(   r)   rR   r}   r¾   rÊ   r;   r   r   r   r   Ú<module>   s2   
){  ? H