o
    ŸôjëÁ  ã                   @   s  d dl Z d dlmZmZmZmZ d dlZd dlZd dl	m
  mZ d dlm
Z
 ddlmZ ddlmZmZ ddlmZ 					
dcdejdededededefdd„Z	dddd„Zdd„ Zdd„ ZG dd„ de
jƒZdedd„Zdfd d!„Zdgd#ed$eej ef d%efd&d'„Z!d(ejd)eejeej f d*eejejf fd+d,„Z"G d-d.„ d.e
jƒZ#G d/d0„ d0e
jƒZ$G d1d2„ d2e
jƒZ%G d3d4„ d4e
jƒZ&G d5d6„ d6e
jƒZ'G d7d8„ d8e
jƒZ(G d9d:„ d:e
jƒZ)G d;d<„ d<e
jƒZ*G d=d>„ d>e
jƒZ+G d?d@„ d@e
jƒZ,G dAdB„ dBe
jƒZ-G dCdD„ dDe
jƒZ.G dEdF„ dFe
jƒZ/G dGdH„ dHe
jƒZ0G dIdJ„ dJe
jƒZ1G dKdL„ dLe
jƒZ2G dMdN„ dNe
jƒZ3G dOdP„ dPe
jƒZ4G dQdR„ dRe
jƒZ5dSdT„ Z6G dUdV„ dVe
jƒZ7G dWdX„ dXe
jƒZ8G dYdZ„ dZe
jƒZ9G d[d\„ d\e
jƒZ:G d]d^„ d^e
jƒZ;G d_d`„ d`e
jƒZ<G dadb„ dbe
jƒZ=dS )hé    N)ÚListÚOptionalÚTupleÚUnion)Únné   )Ú	deprecateé   )ÚFP32SiLUÚget_activation)Ú	AttentionFé'  Ú	timestepsÚembedding_dimÚflip_sin_to_cosÚdownscale_freq_shiftÚscaleÚ
max_periodc           	      C   sö   t | jƒdksJ dƒ‚|d }t |¡ tjd|tj| jd� }|||  }t |¡}| dd…df  	¡ |ddd…f  }|| }tj
t |¡t |¡gdd�}|rktj
|dd…|d…f |dd…d|…f gdd�}|d dkrytjj |d	¡}|S )
a¬  
    This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.

    :param timesteps: a 1-D Tensor of N indices, one per batch element.
                      These may be fractional.
    :param embedding_dim: the dimension of the output. :param max_period: controls the minimum frequency of the
    embeddings. :return: an [N x dim] Tensor of positional embeddings.
    r	   zTimesteps should be a 1d-arrayr   r   )ÚstartÚendÚdtypeÚdeviceNéÿÿÿÿ©Údim)r   r	   r   r   )ÚlenÚshapeÚmathÚlogÚtorchÚarangeÚfloat32r   ÚexpÚfloatÚcatÚsinÚcosr   Ú
functionalÚpad)	r   r   r   r   r   r   Úhalf_dimÚexponentÚemb© r,   ún/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/diffusers/models/embeddings.pyÚget_timestep_embedding   s   ÿ
$2r.   ç      ð?é   c           
      C   sÄ   t |tƒr	||f}tj|d tjd�|d |  | }tj|d tjd�|d |  | }t ||¡}tj|dd�}| dd|d |d g¡}t| |ƒ}	|r`|dkr`tj	t 
|| g¡|	gdd�}	|	S )z©
    grid_size: int of the grid height and width return: pos_embed: [grid_size*grid_size, embed_dim] or
    [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
    r   ©r   r	   ©Úaxisr   )Ú
isinstanceÚintÚnpr    r!   ÚmeshgridÚstackÚreshapeÚ!get_2d_sincos_pos_embed_from_gridÚconcatenateÚzeros)
Ú	embed_dimÚ	grid_sizeÚ	cls_tokenZextra_tokensÚinterpolation_scaleÚ	base_sizeÚgrid_hÚgrid_wÚgridÚ	pos_embedr,   r,   r-   Úget_2d_sincos_pos_embedF   s   
$$
rF   c                 C   sN   | d dkr
t dƒ‚t| d |d ƒ}t| d |d ƒ}tj||gdd�}|S )Nr   r   ú embed_dim must be divisible by 2r	   r2   )Ú
ValueErrorÚ!get_1d_sincos_pos_embed_from_gridr6   r;   )r=   rD   Úemb_hÚemb_wr+   r,   r,   r-   r:   \   s   r:   c                 C   s‚   | d dkr
t dƒ‚tj| d tjd�}|| d  }dd|  }| d¡}t d	||¡}t |¡}t |¡}tj||gd
d�}|S )zu
    embed_dim: output dimension for each position pos: a list of positions to be encoded: size (M,) out: (M, D)
    r   r   rG   r1   g       @r/   r   r   zm,d->mdr	   r2   )	rH   r6   r    Úfloat64r9   Úeinsumr%   r&   r;   )r=   ÚposÚomegaÚoutZemb_sinZemb_cosr+   r,   r,   r-   rI   h   s   


rI   c                       sH   e Zd ZdZ												
d‡ fdd„	Zdd„ Zdd„ Z‡  ZS )Ú
PatchEmbedz:2D Image to Patch Embedding with support for SD3 cropping.éà   r0   é   é   FTr	   ÚsincosNc                    s  t ƒ  ¡  || ||  }|| _|| _|| _tj||||f||d�| _|r/tj|ddd�| _	nd | _	|| _
|| || | _| _|| | _|	| _|rM|}nt|d ƒ}|
d u r\d | _d S |
dkr‚t||| j| jd�}|rndnd}| jd	t |¡ ¡  d
¡|d� d S td|
› �ƒ‚)N)Úkernel_sizeÚstrideÚbiasFg�íµ ÷Æ°>)Úelementwise_affineÚepsç      à?rU   )rA   r@   TrE   r   )Ú
persistentzUnsupported pos_embed_type: )ÚsuperÚ__init__ÚflattenÚ
layer_normÚpos_embed_max_sizer   ÚConv2dÚprojÚ	LayerNormÚnormÚ
patch_sizeÚheightÚwidthrA   r@   r5   rE   rF   Úregister_bufferr   Ú
from_numpyr#   Ú	unsqueezerH   )Úselfrg   rh   rf   Úin_channelsr=   r`   r_   rX   r@   Zpos_embed_typera   Únum_patchesr>   rE   r\   ©Ú	__class__r,   r-   r^   €   s6   
ÿ

ÿ$zPatchEmbed.__init__c                 C   sÜ   | j du r	tdƒ‚|| j }|| j }|| j kr$td|› d| j › d�ƒ‚|| j kr5td|› d| j › d�ƒ‚| j | d }| j | d }| j d| j | j d	¡}|dd…||| …||| …dd…f }| dd	|jd	 ¡}|S )
z2Crops positional embeddings for SD3 compatibility.Nz.`pos_embed_max_size` must be set for cropping.zHeight (z/) cannot be greater than `pos_embed_max_size`: Ú.zWidth (r   r	   r   )ra   rH   rf   rE   r9   r   )rl   rg   rh   ÚtopÚleftZspatial_pos_embedr,   r,   r-   Úcropped_pos_embed³   s$   



ÿ
ÿ(zPatchEmbed.cropped_pos_embedc                 C   s  | j d ur|jdd … \}}n|jd | j |jd | j }}|  |¡}| jr1| d¡ dd¡}| jr9|  |¡}| jd u rD| 	|j
¡S | j rN|  ||¡}n,| j|ksX| j|krwt| jjd ||f| j| jd�}t |¡ ¡  d¡ 	|j¡}n| j}||  	|j
¡S )Néþÿÿÿr   r   r	   )r=   r>   rA   r@   r   )ra   r   rf   rc   r_   Ú	transposer`   re   rE   Útor   rt   rg   rh   rF   rA   r@   r   rj   r#   rk   r   )rl   Úlatentrg   rh   rE   r,   r,   r-   ÚforwardÊ   s,   
"



üzPatchEmbed.forward)rR   rR   r0   rS   rT   FTTr	   rU   N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r^   rt   ry   Ú__classcell__r,   r,   ro   r-   rQ   }   s     ô3rQ   Tc           
      C   s”   |\}}t j|d |d |d dt jd�}t j|d |d |d dt jd�}t  ||¡}t j|dd�}| ddg|jdd… ¢¡}t| ||d�}	|	S )	a  
    RoPE for image tokens with 2d structure.

    Args:
    embed_dim: (`int`):
        The embedding dimension size
    crops_coords (`Tuple[int]`)
        The top-left and bottom-right coordinates of the crop.
    grid_size (`Tuple[int]`):
        The grid size of the positional embedding.
    use_real (`bool`):
        If True, return real part and imaginary part separately. Otherwise, return complex numbers.

    Returns:
        `torch.Tensor`: positional embdding with shape `( grid_size * grid_size, embed_dim/2)`.
    r   F)Úendpointr   r	   r2   r   N©Úuse_real)r6   Úlinspacer!   r7   r8   r9   r   Ú!get_2d_rotary_pos_embed_from_grid)
r=   Úcrops_coordsr>   r�   r   ÚstoprB   rC   rD   rE   r,   r,   r-   Úget_2d_rotary_pos_embedé   s   ""r†   c                 C   sž   | d dksJ ‚t | d |d  d¡|d�}t | d |d  d¡|d�}|rDtj|d |d gdd�}tj|d |d gdd�}||fS tj||gdd�}|S )Né   r   r   r   r€   r	   r   )Úget_1d_rotary_pos_embedr9   r   r$   )r=   rD   r�   rJ   rK   r&   r%   r+   r,   r,   r-   rƒ     s   rƒ   ç     ˆÃ@r   rN   Úthetac           	      C   s¦   t |tƒr
t |¡}d|t d| d¡d| d …  ¡ |    }t |¡ |j¡}t 	||¡ ¡ }|rH| 
¡ jddd�}| ¡ jddd�}||fS t t |¡|¡}|S )ay  
    Precompute the frequency tensor for complex exponentials (cis) with given dimensions.

    This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' and the end
    index 'end'. The 'theta' parameter scales the frequencies. The returned tensor contains complex values in complex64
    data type.

    Args:
        dim (`int`): Dimension of the frequency tensor.
        pos (`np.ndarray` or `int`): Position indices for the frequency tensor. [S] or scalar
        theta (`float`, *optional*, defaults to 10000.0):
            Scaling factor for frequency computation. Defaults to 10000.0.
        use_real (`bool`, *optional*):
            If True, return real part and imaginary part separately. Otherwise, return complex numbers.

    Returns:
        `torch.Tensor`: Precomputed frequency tensor with complex exponentials. [S, D/2]
    r/   r   r   Nr	   r   )r4   r5   r6   r    r   r#   rj   rw   r   Úouterr&   Úrepeat_interleaver%   ÚpolarÚ	ones_like)	r   rN   rŠ   r�   ÚfreqsÚtZ	freqs_cosZ	freqs_sinÚ	freqs_cisr,   r,   r-   rˆ     s   

*rˆ   Úxr‘   Úreturnc                 C   sœ   |\}}|d }|d }|  | j¡|  | j¡}}| jg | jdd… ¢d‘d‘R Ž  d¡\}}tj| |gdd� d¡}|  ¡ | | ¡ |    | j	¡}|S )a3  
    Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings
    to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are
    reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting
    tensors contain rotary embeddings and are returned as real tensors.

    Args:
        x (`torch.Tensor`):
            Query or key tensor to apply rotary embeddings. [B, H, S, D] xk (torch.Tensor): Key tensor to apply
        freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],)

    Returns:
        Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings.
    )NNNr   r   r   rS   )
rw   r   r9   r   Úunbindr   r8   r_   r#   r   )r’   r‘   r&   r%   Úx_realÚx_imagZ	x_rotatedrP   r,   r,   r-   Úapply_rotary_emb6  s   , r—   c                       sL   e Zd Z					ddededededee f
‡ fd	d
„Zddd„Z‡  ZS )ÚTimestepEmbeddingÚsiluNTrm   Útime_embed_dimÚact_fnÚout_dimÚpost_act_fnc           	         sˆ   t ƒ  ¡  t |||¡| _|d urtj||dd�| _nd | _t|ƒ| _|d ur*|}n|}t |||¡| _|d u r=d | _	d S t|ƒ| _	d S )NF)rX   )
r]   r^   r   ÚLinearÚlinear_1Ú	cond_projr   ÚactÚlinear_2Úpost_act)	rl   rm   rš   r›   rœ   r�   Úcond_proj_dimZsample_proj_biasÚtime_embed_dim_outro   r,   r-   r^   U  s   



zTimestepEmbedding.__init__c                 C   sV   |d ur||   |¡ }|  |¡}| jd ur|  |¡}|  |¡}| jd ur)|  |¡}|S ©N)r    rŸ   r¡   r¢   r£   )rl   ÚsampleÚ	conditionr,   r,   r-   ry   u  s   





zTimestepEmbedding.forward)r™   NNNTr¦   )	rz   r{   r|   r5   Ústrr   r^   ry   r~   r,   r,   ro   r-   r˜   T  s$    øþýüûú r˜   c                       s2   e Zd Zdededef‡ fdd„Zdd„ Z‡  ZS )Ú	TimestepsÚnum_channelsr   r   c                    s    t ƒ  ¡  || _|| _|| _d S r¦   )r]   r^   r«   r   r   )rl   r«   r   r   ro   r,   r-   r^   …  s   

zTimesteps.__init__c                 C   s   t || j| j| jd�}|S )N)r   r   )r.   r«   r   r   )rl   r   Zt_embr,   r,   r-   ry   ‹  s   üzTimesteps.forward)	rz   r{   r|   r5   Úboolr#   r^   ry   r~   r,   r,   ro   r-   rª   „  s    rª   c                       s6   e Zd ZdZ	ddedef‡ fdd	„Zd
d„ Z‡  ZS )ÚGaussianFourierProjectionz-Gaussian Fourier embeddings for noise levels.é   r/   TFÚembedding_sizer   c                    s^   t ƒ  ¡  tjt |¡| dd�| _|| _|| _|r-tjt |¡| dd�| _	| j	| _d S d S )NF)Úrequires_grad)
r]   r^   r   Ú	Parameterr   ÚrandnÚweightr   r   ÚW)rl   r¯   r   Zset_W_to_weightr   r   ro   r,   r-   r^   ˜  s   
üz"GaussianFourierProjection.__init__c                 C   s†   | j rt  |¡}|d d …d f | jd d d …f  d tj }| jr2tjt |¡t |¡gdd�}|S tjt |¡t |¡gdd�}|S )Nr   r   r   )	r   r   r³   r6   Úpir   r$   r&   r%   )rl   r’   Zx_projrP   r,   r,   r-   ry   ¦  s   
,ÿz!GaussianFourierProjection.forward)r®   r/   TTF)	rz   r{   r|   r}   r5   r#   r^   ry   r~   r,   r,   ro   r-   r­   •  s    ÿÿÿr­   c                       s4   e Zd ZdZd	dedef‡ fdd„Zdd„ Z‡  ZS )
ÚSinusoidalPositionalEmbeddinga[  Apply positional information to a sequence of embeddings.

    Takes in a sequence of embeddings with shape (batch_size, seq_length, embed_dim) and adds positional embeddings to
    them

    Args:
        embed_dim: (int): Dimension of the positional embedding.
        max_seq_length: Maximum sequence length to apply positional embeddings

    é    r=   Úmax_seq_lengthc                    s    t ƒ  ¡  t |¡ d¡}t t d|d¡t d¡ |  ¡}t d||¡}t 	|| ¡|dd d …dd d…f< t 
|| ¡|dd d …dd d…f< |  d|¡ d S )Nr	   r   r   r‰   Úpe)r]   r^   r   r    rk   r"   r   r   r<   r%   r&   ri   )rl   r=   r¸   ÚpositionÚdiv_termr¹   ro   r,   r-   r^   ¿  s   
$""z&SinusoidalPositionalEmbedding.__init__c                 C   s*   |j \}}}|| jd d …d |…f  }|S r¦   )r   r¹   )rl   r’   Ú_Ú
seq_lengthr,   r,   r-   ry   È  s   z%SinusoidalPositionalEmbedding.forward)r·   ©rz   r{   r|   r}   r5   r^   ry   r~   r,   r,   ro   r-   r¶   ³  s    	r¶   c                       s:   e Zd ZdZdedededef‡ fdd„Zdd	„ Z‡  ZS )
ÚImagePositionalEmbeddingsa�  
    Converts latent image classes into vector embeddings. Sums the vector embeddings with positional embeddings for the
    height and width of the latent space.

    For more details, see figure 10 of the dall-e paper: https://arxiv.org/abs/2102.12092

    For VQ-diffusion:

    Output vector embeddings are used as input for the transformer.

    Note that the vector embeddings for the transformer are different than the vector embeddings from the VQVAE.

    Args:
        num_embed (`int`):
            Number of embeddings for the latent pixels embeddings.
        height (`int`):
            Height of the latent image i.e. the number of height embeddings.
        width (`int`):
            Width of the latent image i.e. the number of width embeddings.
        embed_dim (`int`):
            Dimension of the produced vector embeddings. Used for the latent pixel, height, and width embeddings.
    Ú	num_embedrg   rh   r=   c                    sV   t ƒ  ¡  || _|| _|| _|| _t | j|¡| _t | j|¡| _	t | j|¡| _
d S r¦   )r]   r^   rg   rh   rÀ   r=   r   Ú	Embeddingr+   Ú
height_embÚ	width_emb)rl   rÀ   rg   rh   r=   ro   r,   r-   r^   æ  s   
z"ImagePositionalEmbeddings.__init__c                 C   s¨   |   |¡}|  tj| j|jd� d| j¡¡}| d¡}|  tj| j	|jd� d| j	¡¡}| d¡}|| }| d| j| j	 d¡}||d d …d |j
d …d d …f  }|S )N©r   r	   r   r   )r+   rÂ   r   r    rg   r   Úviewrk   rÃ   rh   r   )rl   Úindexr+   rÂ   rÃ   Úpos_embr,   r,   r-   ry   ø  s   
"
"
$z!ImagePositionalEmbeddings.forwardr¾   r,   r,   ro   r-   r¿   Î  s    þýüûr¿   c                       s<   e Zd ZdZ‡ fdd„Zd
dd„Zd
dejfdd	„Z‡  Z	S )ÚLabelEmbeddinga7  
    Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance.

    Args:
        num_classes (`int`): The number of classes.
        hidden_size (`int`): The size of the vector embeddings.
        dropout_prob (`float`): The probability of dropping a label.
    c                    s4   t ƒ  ¡  |dk}t || |¡| _|| _|| _d S ©Nr   )r]   r^   r   rÁ   Úembedding_tableÚnum_classesÚdropout_prob)rl   rË   Úhidden_sizerÌ   Zuse_cfg_embeddingro   r,   r-   r^     s
   

zLabelEmbedding.__init__Nc                 C   sH   |du rt j|jd |jd�| jk }nt  |dk¡}t  || j|¡}|S )zB
        Drops labels to enable classifier-free guidance.
        Nr   rÄ   r	   )r   Úrandr   r   rÌ   ÚtensorÚwhererË   )rl   ÚlabelsÚforce_drop_idsZdrop_idsr,   r,   r-   Ú
token_drop   s
   zLabelEmbedding.token_droprÑ   c                 C   s6   | j dk}| jr
|s|d ur|  ||¡}|  |¡}|S rÉ   )rÌ   ÚtrainingrÓ   rÊ   )rl   rÑ   rÒ   Zuse_dropoutÚ
embeddingsr,   r,   r-   ry   +  s
   

zLabelEmbedding.forwardr¦   )
rz   r{   r|   r}   r^   rÓ   r   Ú
LongTensorry   r~   r,   r,   ro   r-   rÈ     s
    	
rÈ   c                	       sN   e Zd Z				ddedededef‡ fdd	„Zd
ejdejfdd„Z‡  ZS )ÚTextImageProjectioné   rT   é
   Útext_embed_dimÚimage_embed_dimÚcross_attention_dimÚnum_image_text_embedsc                    s6   t ƒ  ¡  || _t || j| ¡| _t ||¡| _d S r¦   )r]   r^   rÝ   r   rž   Úimage_embedsÚ	text_proj)rl   rÚ   rÛ   rÜ   rÝ   ro   r,   r-   r^   4  s   
zTextImageProjection.__init__Útext_embedsrÞ   c                 C   s@   |j d }|  |¡}| || jd¡}|  |¡}tj||gdd�S )Nr   r   r	   r   )r   rÞ   r9   rÝ   rß   r   r$   )rl   rà   rÞ   Ú
batch_sizeZimage_text_embedsr,   r,   r-   ry   A  s
   


zTextImageProjection.forward)rØ   rT   rT   rÙ   ©	rz   r{   r|   r5   r^   r   ÚTensorry   r~   r,   r,   ro   r-   r×   3  s    ûþýüûr×   c                       sB   e Zd Z			ddededef‡ fdd„Zdejfd	d
„Z‡  ZS )ÚImageProjectionrT   r·   rÛ   rÜ   rÝ   c                    s4   t ƒ  ¡  || _t || j| ¡| _t |¡| _d S r¦   )r]   r^   rÝ   r   rž   rÞ   rd   re   )rl   rÛ   rÜ   rÝ   ro   r,   r-   r^   O  s   
zImageProjection.__init__rÞ   c                 C   s2   |j d }|  |¡}| || jd¡}|  |¡}|S )Nr   r   )r   rÞ   r9   rÝ   re   )rl   rÞ   rá   r,   r,   r-   ry   [  s
   


zImageProjection.forward)rT   rT   r·   râ   r,   r,   ro   r-   rä   N  s    üþýürä   c                       s.   e Zd Zd‡ fdd„	Zdejfdd„Z‡  ZS )ÚIPAdapterFullImageProjectionrØ   c                    s8   t ƒ  ¡  ddlm} |||ddd�| _t |¡| _d S ©Nr	   ©ÚFeedForwardÚgelu)ÚmultÚactivation_fn)r]   r^   Ú	attentionrè   Úffr   rd   re   )rl   rÛ   rÜ   rè   ro   r,   r-   r^   f  s   
z%IPAdapterFullImageProjection.__init__rÞ   c                 C   s   |   |  |¡¡S r¦   )re   rí   )rl   rÞ   r,   r,   r-   ry   m  s   z$IPAdapterFullImageProjection.forward)rØ   rØ   ©rz   r{   r|   r^   r   rã   ry   r~   r,   r,   ro   r-   rå   e  s    rå   c                       s.   e Zd Zd‡ fdd„	Zdejfdd„Z‡  ZS )	ÚIPAdapterFaceIDImageProjectionrØ   r	   c                    sH   t ƒ  ¡  ddlm} || _|| _|||| |dd�| _t |¡| _	d S ræ   )
r]   r^   rì   rè   Ú
num_tokensrÜ   rí   r   rd   re   )rl   rÛ   rÜ   rê   rð   rè   ro   r,   r-   r^   r  s   
z'IPAdapterFaceIDImageProjection.__init__rÞ   c                 C   s&   |   |¡}| d| j| j¡}|  |¡S )Nr   )rí   r9   rð   rÜ   re   )rl   rÞ   r’   r,   r,   r-   ry   {  s   

z&IPAdapterFaceIDImageProjection.forward)rØ   rØ   r	   r	   rî   r,   r,   ro   r-   rï   q  s    	rï   c                       s(   e Zd Zd‡ fdd„	Zddd„Z‡  ZS )	ÚCombinedTimestepLabelEmbeddingsçš™™™™™¹?c                    s:   t ƒ  ¡  tdddd�| _td|d�| _t|||ƒ| _d S )Nr®   Tr	   ©r«   r   r   ©rm   rš   )r]   r^   rª   Ú	time_projr˜   Útimestep_embedderrÈ   Úclass_embedder)rl   rË   r   Zclass_dropout_probro   r,   r-   r^   ‚  s   
z(CombinedTimestepLabelEmbeddings.__init__Nc                 C   s2   |   |¡}|  |j|d�¡}|  |¡}|| }|S ©Nr1   )rõ   rö   rw   r÷   )rl   ÚtimestepÚclass_labelsÚhidden_dtypeÚtimesteps_projÚtimesteps_embÚconditioningr,   r,   r-   ry   ‰  s
   

z'CombinedTimestepLabelEmbeddings.forward)rò   r¦   ©rz   r{   r|   r^   ry   r~   r,   r,   ro   r-   rñ   �  s    rñ   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )Ú"CombinedTimestepTextProjEmbeddingsc                    s<   t ƒ  ¡  tdddd�| _td|d�| _t||dd�| _d S )Nr®   Tr   ró   rô   r™   )r›   )r]   r^   rª   rõ   r˜   rö   ÚPixArtAlphaTextProjectionÚtext_embedder)rl   r   Úpooled_projection_dimro   r,   r-   r^   •  s   
z+CombinedTimestepTextProjEmbeddings.__init__c                 C   s4   |   |¡}|  |j|jd�¡}|  |¡}|| }|S rø   )rõ   rö   rw   r   r  )rl   rù   Zpooled_projectionrü   rý   Úpooled_projectionsrþ   r,   r,   r-   ry   œ  s
   

z*CombinedTimestepTextProjEmbeddings.forwardrÿ   r,   r,   ro   r-   r   ”  s    r   c                	       s8   e Zd Zd
dedededef‡ fdd„Zdd	„ Z‡  ZS )ÚHunyuanDiTAttentionPoolNÚspacial_dimr=   Ú	num_headsÚ
output_dimc                    sp   t ƒ  ¡  t t |d |¡|d  ¡| _t ||¡| _t ||¡| _	t ||¡| _
t ||p0|¡| _|| _d S )Nr	   r[   )r]   r^   r   r±   r   r²   Úpositional_embeddingrž   Úk_projÚq_projÚv_projÚc_projr  )rl   r  r=   r  r  ro   r,   r-   r^   ª  s   
 
z HunyuanDiTAttentionPool.__init__c              	   C   s  |  ddd¡}tj|jddd�|gdd�}|| jd d …d d d …f  |j¡ }tjdi d|d d… “d|“d	|“d
|j	d “d| j
“d| jj“d| jj“d| jj“dd “dt | jj| jj| jjg¡“dd “dd “dd“dd“d| jj“d| jj“dd“d| j“dd“Ž\}}| d¡S )Nr	   r   r   T©r   Úkeepdimr   ÚqueryÚkeyÚvalueÚembed_dim_to_checkr   r  Úq_proj_weightÚk_proj_weightÚv_proj_weightÚin_proj_weightÚin_proj_biasÚbias_kÚbias_vÚadd_zero_attnFÚ	dropout_pÚout_proj_weightÚout_proj_biasÚuse_separate_proj_weightrÔ   Úneed_weightsr,   )Úpermuter   r$   Úmeanr	  rw   r   ÚFÚmulti_head_attention_forwardr   r  r  r³   r
  r  rX   r  rÔ   Úsqueeze)rl   r’   r¼   r,   r,   r-   ry   ³  sV   $
ÿþý
üûúùø	÷
öõôóòñðïîí
zHunyuanDiTAttentionPool.forwardr¦   ©rz   r{   r|   r5   r^   ry   r~   r,   r,   ro   r-   r  §  s     	r  c                       s(   e Zd Zd	‡ fdd„	Zd
dd„Z‡  ZS )Ú-HunyuanCombinedTimestepTextSizeStyleEmbeddingrØ   r®   é   c                    sn   t ƒ  ¡  tdddd�| _td|d�| _t||d|d�| _t 	d|¡| _
d	| | }t||d
 |dd�| _d S )Nr®   Tr   ró   rô   é   )r  r  r	   é   r‡   Ú	silu_fp32)Úin_featuresrÍ   Úout_featuresr›   )r]   r^   rª   rõ   r˜   rö   r  Úpoolerr   rÁ   Ústyle_embedderr  Úextra_embedder)rl   r   r  Úseq_lenrÜ   Zextra_in_dimro   r,   r-   r^   Ð  s   
ÿüz6HunyuanCombinedTimestepTextSizeStyleEmbedding.__init__Nc                 C   s‚   |   |¡}|  |j|d�¡}|  |¡}t| d¡dddƒ}|j|d�}| dd¡}|  |¡}	tj|||	gdd�}
||  	|
¡ }|S )	Nr1   r   r®   Tr   r*  r	   r   )
rõ   rö   rw   r.  r.   rÅ   r/  r   r$   r0  )rl   rù   Úencoder_hidden_statesZimage_meta_sizeÚstylerû   rü   rý   r  Zstyle_embeddingZ
extra_condrþ   r,   r,   r-   ry   ã  s   


z5HunyuanCombinedTimestepTextSizeStyleEmbedding.forward)rØ   r®   r(  r¦   rÿ   r,   r,   ro   r-   r'  Ï  s    r'  c                       s4   e Zd Zd	dededef‡ fdd„Zdd„ Z‡  ZS )
ÚTextTimeEmbeddingé@   Úencoder_dimrš   r  c                    s@   t ƒ  ¡  t |¡| _t||ƒ| _t ||¡| _t |¡| _	d S r¦   )
r]   r^   r   rd   Únorm1ÚAttentionPoolingÚpoolrž   rc   Únorm2)rl   r6  rš   r  ro   r,   r-   r^   ú  s
   
zTextTimeEmbedding.__init__c                 C   s,   |   |¡}|  |¡}|  |¡}|  |¡}|S r¦   )r7  r9  rc   r:  )rl   Úhidden_statesr,   r,   r-   ry     s
   



zTextTimeEmbedding.forward)r5  r&  r,   r,   ro   r-   r4  ù  s    r4  c                       sB   e Zd Zddededef‡ fdd„Zdejd	ejfd
d„Z‡  ZS )ÚTextImageTimeEmbeddingrT   r*  rÚ   rÛ   rš   c                    s6   t ƒ  ¡  t ||¡| _t |¡| _t ||¡| _d S r¦   )r]   r^   r   rž   rß   rd   Ú	text_normÚ
image_proj)rl   rÚ   rÛ   rš   ro   r,   r-   r^   
  s   
zTextImageTimeEmbedding.__init__rà   rÞ   c                 C   s&   |   |¡}|  |¡}|  |¡}|| S r¦   )rß   r=  r>  )rl   rà   rÞ   Ztime_text_embedsÚtime_image_embedsr,   r,   r-   ry     s   


zTextImageTimeEmbedding.forward)rT   rT   r*  râ   r,   r,   ro   r-   r<  	  s    r<  c                       s8   e Zd Zd
dedef‡ fdd„Zdejfdd	„Z‡  ZS )ÚImageTimeEmbeddingrT   r*  rÛ   rš   c                    s(   t ƒ  ¡  t ||¡| _t |¡| _d S r¦   )r]   r^   r   rž   r>  rd   Ú
image_norm©rl   rÛ   rš   ro   r,   r-   r^     s   
zImageTimeEmbedding.__init__rÞ   c                 C   s   |   |¡}|  |¡}|S r¦   )r>  rA  )rl   rÞ   r?  r,   r,   r-   ry   !  s   

zImageTimeEmbedding.forward©rT   r*  râ   r,   r,   ro   r-   r@    s    r@  c                       s>   e Zd Zddedef‡ fdd„Zdejdejfd	d
„Z‡  ZS )ÚImageHintTimeEmbeddingrT   r*  rÛ   rš   c                    sâ   t ƒ  ¡  t ||¡| _t |¡| _t tjddddd�t 	¡ tjddddd�t 	¡ tjdddddd�t 	¡ tjddddd�t 	¡ tjdddddd�t 	¡ tjddddd�t 	¡ tjdd	dddd�t 	¡ tjd	d
ddd�¡| _
d S )NrS   r0   r	   )Úpaddingr·   r   )rE  rW   é`   r®   r‡   )r]   r^   r   rž   r>  rd   rA  Ú
Sequentialrb   ÚSiLUÚinput_hint_blockrB  ro   r,   r-   r^   )  s(   

ñzImageHintTimeEmbedding.__init__rÞ   Úhintc                 C   s&   |   |¡}|  |¡}|  |¡}||fS r¦   )r>  rA  rI  )rl   rÞ   rJ  r?  r,   r,   r-   ry   ?  s   


zImageHintTimeEmbedding.forwardrC  râ   r,   r,   ro   r-   rD  (  s    rD  c                       s&   e Zd Zd‡ fdd„	Zdd„ Z‡  ZS )r8  Nc                    s~   t ƒ  ¡  || _t t d|¡|d  ¡| _tj||| jd�| _	tj||| jd�| _
tj||| jd�| _|| _|| j | _d S )Nr	   r[   r1   )r]   r^   r   r   r±   r   r²   r	  rž   r
  r  r  r  Údim_per_head)rl   r  r=   r   ro   r,   r-   r^   J  s   
zAttentionPooling.__init__c                    sô   |  ¡ \‰ }}‡ ‡fdd„}|jddd�ˆj |j¡ }tj||gdd�}|ˆ |¡ƒ}|ˆ |¡ƒ}|ˆ 	|¡ƒ}dt
 t
 ˆj¡¡ }	t d||	 ||	 ¡}
tj|
 ¡ dd� |
j¡}
t d	|
|¡}| ˆ dd¡ dd
¡}|d d …dd d …f S )Nc                    sF   |   ˆ dˆjˆj¡} |  dd¡} |  ˆ ˆj dˆj¡} |  dd¡} | S )Nr   r	   r   )rÅ   r  rK  rv   r9   )r’   ©Úbsrl   r,   r-   r   W  s
   z'AttentionPooling.forward.<locals>.shaper	   Tr  r   zbct,bcs->btsr   zbts,bcs->bctr   r   )Úsizer"  r	  rw   r   r   r$   r  r
  r  r   ÚsqrtrK  rM   Úsoftmaxr#   Útyper9   rv   )rl   r’   Úlengthrh   r   Zclass_tokenÚqÚkÚvr   r³   Úar,   rL  r-   ry   T  s   zAttentionPooling.forwardr¦   rÿ   r,   r,   ro   r-   r8  G  s    
r8  c                 C   sŠ   |j dd… \}}dt | ¡|   }|d j|j|jd�}|| d¡ }tj| ¡ | 	¡ fdd�}| 
dd	d
dd¡ ||| d d ¡}|S )zÉ
    Args:
        embed_dim: int
        box: a 3-D tensor [B x N x 4] representing the bounding boxes for GLIGEN pipeline
    Returns:
        [B x N x embed_dim] tensor of positional embeddings
    Nr   éd   )NNN)r   r   r   r   r   r	   rS   r‡   )r   r   r    rw   r   r   rk   r8   r%   r&   r!  r9   )r=   Úboxrá   Ú	num_boxesr+   r,   r,   r-   Ú#get_fourier_embeds_from_boundingboxy  s   	$rZ  c                       s2   e Zd Zd‡ fdd„	Z					d	dd„Z‡  ZS )
ÚGLIGENTextBoundingboxProjectionú	text-onlyr)  c              
      sf  t ƒ  ¡  || _|| _|| _|d d | _t|tƒr|d }|dkrMt 	t 
| j| j d¡t ¡ t 
dd¡t ¡ t 
d|¡¡| _tj t | jg¡¡| _nX|dkr¥t 	t 
| j| j d¡t ¡ t 
dd¡t ¡ t 
d|¡¡| _t 	t 
| j| j d¡t ¡ t 
dd¡t ¡ t 
d|¡¡| _tj t | jg¡¡| _tj t | jg¡¡| _tj t | jg¡¡| _d S )Nr   r‡   r   r\  é   z
text-image)r]   r^   Úpositive_lenrœ   Úfourier_embedder_dimZposition_dimr4   Útupler   rG  rž   rH  Úlinearsr   r±   r<   Únull_positive_featureÚlinears_textÚlinears_imageÚnull_text_featureÚnull_image_featureÚnull_position_feature)rl   r^  rœ   Zfeature_typeZfourier_freqsro   r,   r-   r^   �  sD   



û

û

ûz(GLIGENTextBoundingboxProjection.__init__Nc                 C   s$  |  d¡}t| j|ƒ}| j ddd¡}	|| d| |	  }|d urA| j ddd¡}
|| d| |
  }|  tj||gdd�¡}|S |  d¡}|  d¡}| j	 ddd¡}| j
 ddd¡}|| d| |  }|| d| |  }|  tj||gdd�¡}|  tj||gdd�¡}tj||gdd�}|S )Nr   r	   r   )rk   rZ  r_  rg  rÅ   rb  ra  r   r$   re  rf  rc  rd  )rl   ÚboxesÚmasksZpositive_embeddingsZphrases_masksZimage_masksZphrases_embeddingsZimage_embeddingsZxyxy_embeddingZ	xyxy_nullZpositive_nullÚobjsZ	text_nullZ
image_nullZ	objs_textZ
objs_imager,   r,   r-   ry   ¸  s&   


ñ
z'GLIGENTextBoundingboxProjection.forward)r\  r)  )NNNNNrÿ   r,   r,   ro   r-   r[  Ž  s    -ør[  c                       s0   e Zd ZdZddef‡ fdd„Zdd„ Z‡  ZS )	Ú)PixArtAlphaCombinedTimestepSizeEmbeddingszº
    For PixArt-Alpha.

    Reference:
    https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L164C9-L168C29
    FÚuse_additional_conditionsc                    sl   t ƒ  ¡  || _tdddd�| _td|d�| _|| _|r4tdddd�| _td|d�| _	td|d�| _
d S d S )Nr®   Tr   ró   rô   )r]   r^   Zoutdimrª   rõ   r˜   rö   rl  Úadditional_condition_projÚresolution_embedderÚaspect_ratio_embedder)rl   r   Zsize_emb_dimrl  ro   r,   r-   r^   ó  s   
ýz2PixArtAlphaCombinedTimestepSizeEmbeddings.__init__c                 C   s�   |   |¡}|  |j|d�¡}| jrD|  | ¡ ¡ |¡}|  |¡ |d¡}|  | ¡ ¡ |¡}	|  |	¡ |d¡}	|t	j
||	gdd� }
|
S |}
|
S )Nr1   r   r	   r   )rõ   rö   rw   rl  rm  r_   rn  r9   ro  r   r$   )rl   rù   Ú
resolutionÚaspect_ratiorá   rû   rü   rý   Zresolution_embZaspect_ratio_embrþ   r,   r,   r-   ry      s   
þz1PixArtAlphaCombinedTimestepSizeEmbeddings.forward©F)rz   r{   r|   r}   r¬   r^   ry   r~   r,   r,   ro   r-   rk  ë  s    rk  c                       s*   e Zd ZdZd‡ fdd„	Zdd„ Z‡  ZS )	r  zÊ
    Projects caption embeddings. Also handles dropout for classifier-free guidance.

    Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py
    NÚ	gelu_tanhc                    sŠ   t ƒ  ¡  |d u r|}tj||dd�| _|dkr tjdd�| _n|dkr*t ¡ | _n|dkr3tƒ | _nt	d|› �ƒ‚tj||dd�| _
d S )	NT)r,  r-  rX   rs  Útanh)Úapproximater™   r+  zUnknown activation function: )r]   r^   r   rž   rŸ   ÚGELUÚact_1rH  r
   rH   r¢   )rl   r,  rÍ   r-  r›   ro   r,   r-   r^     s   

z"PixArtAlphaTextProjection.__init__c                 C   s"   |   |¡}|  |¡}|  |¡}|S r¦   )rŸ   rw  r¢   )rl   Úcaptionr;  r,   r,   r-   ry   &  s   


z!PixArtAlphaTextProjection.forward)Nrs  )rz   r{   r|   r}   r^   ry   r~   r,   r,   ro   r-   r    s    r  c                       sD   e Zd Z				ddedededed	d
f
‡ fdd„Zdd„ Z‡  ZS )Ú!IPAdapterPlusImageProjectionBlockrT   r5  r0   r‡   Ú
embed_dimsÚdim_headÚheadsÚ	ffn_ratior“   Nc              
      sf   t ƒ  ¡  ddlm} t |¡| _t |¡| _t|||dd�| _	t 
t |¡|||d|dd�¡| _d S )Nr	   rç   F)Ú	query_dimr{  r|  Úout_biasré   )rë   rê   rX   )r]   r^   rì   rè   r   rd   Úln0Úln1r   ÚattnrG  rí   )rl   rz  r{  r|  r}  rè   ro   r,   r-   r^   .  s   
ü
þz*IPAdapterPlusImageProjectionBlock.__init__c                 C   sH   |   |¡}|  |¡}tj||gdd�}|  ||¡| }|  |¡| }|S )Nru   r   )r€  r�  r   r$   r‚  rí   )rl   r’   ÚlatentsÚresidualr2  r,   r,   r-   ry   E  s   

z)IPAdapterPlusImageProjectionBlock.forward)rT   r5  r0   r‡   )rz   r{   r|   r5   r#   r^   ry   r~   r,   r,   ro   r-   ry  -  s"    ûþýüûúry  c                       sn   e Zd ZdZ								dd	ed
edededededededdf‡ fdd„Zdejdejfdd„Z	‡  Z
S )ÚIPAdapterPlusImageProjectionaä  Resampler of IP-Adapter Plus.

    Args:
        embed_dims (int): The feature dimension. Defaults to 768. output_dims (int): The number of output channels,
        that is the same
            number of the channels in the `unet.config.cross_attention_dim`. Defaults to 1024.
        hidden_dims (int):
            The number of hidden channels. Defaults to 1280. depth (int): The number of blocks. Defaults
        to 8. dim_head (int): The number of head channels. Defaults to 64. heads (int): Parallel attention heads.
        Defaults to 16. num_queries (int):
            The number of queries. Defaults to 8. ffn_ratio (float): The expansion ratio
        of feedforward network hidden
            layer channels. Defaults to 4.
    rT   rØ   é   r‡   r5  r0   r)  rz  Úoutput_dimsÚhidden_dimsÚdepthr{  r|  Únum_queriesr}  r“   Nc	           	         sx   t ƒ  ¡  t t d|ˆ¡ˆd  ¡| _t |ˆ¡| _t ˆ|¡| _	t 
|¡| _t ‡ ‡‡‡fdd„t|ƒD ƒ¡| _d S )Nr	   r[   c                    s   g | ]	}t ˆˆ ˆˆƒ‘qS r,   ©ry  ©Ú.0r¼   ©r{  r}  r|  rˆ  r,   r-   Ú
<listcomp>r  ó    z9IPAdapterPlusImageProjection.__init__.<locals>.<listcomp>)r]   r^   r   r±   r   r²   rƒ  rž   Úproj_inÚproj_outrd   Únorm_outÚ
ModuleListÚrangeÚlayers)	rl   rz  r‡  rˆ  r‰  r{  r|  rŠ  r}  ro   rŽ  r-   r^   ^  s   

ÿz%IPAdapterPlusImageProjection.__init__r’   c                 C   sP   | j  | d¡dd¡}|  |¡}| jD ]
}|}||||ƒ}q|  |¡}|  |¡S )z‹Forward pass.

        Args:
            x (torch.Tensor): Input Tensor.
        Returns:
            torch.Tensor: Output Tensor.
        r   r	   )rƒ  ÚrepeatrN  r‘  r–  r’  r“  )rl   r’   rƒ  Úblockr„  r,   r,   r-   ry   u  s   



z$IPAdapterPlusImageProjection.forward)rT   rØ   r†  r‡   r5  r0   r)  r‡   ©rz   r{   r|   r}   r5   r#   r^   r   rã   ry   r~   r,   r,   ro   r-   r…  N  s<    ÷þýüûúùø	÷
ör…  c                       s€   e Zd ZdZ												dd
edededededededededededdf‡ fdd„Zdejdejfdd„Z	‡  Z
S )Ú"IPAdapterFaceIDPlusImageProjectiona—  FacePerceiverResampler of IP-Adapter Plus.

    Args:
        embed_dims (int): The feature dimension. Defaults to 768. output_dims (int): The number of output channels,
        that is the same
            number of the channels in the `unet.config.cross_attention_dim`. Defaults to 1024.
        hidden_dims (int):
            The number of hidden channels. Defaults to 1280. depth (int): The number of blocks. Defaults
        to 8. dim_head (int): The number of head channels. Defaults to 64. heads (int): Parallel attention heads.
        Defaults to 16. num_tokens (int): Number of tokens num_queries (int): The number of queries. Defaults to 8.
        ffn_ratio (float): The expansion ratio of feedforward network hidden
            layer channels. Defaults to 4.
        ffproj_ratio (float): The expansion ratio of feedforward network hidden
            layer channels (for ID embeddings). Defaults to 4.
    rT   r†  r]  r‡   r5  r0   r)  r   rz  r‡  rˆ  Úid_embeddings_dimr‰  r{  r|  rð   rŠ  r}  Úffproj_ratior“   Nc                    s¦   t ƒ  ¡  ddlm} || _ˆ| _d | _d| _d| _||ˆ| d|d�| _	t
 ˆ¡| _t
 |ˆ¡| _t
 ˆ|¡| _t
 |¡| _t
 ‡ ‡‡‡fdd„t|ƒD ƒ¡| _d S )	Nr	   rç   Fr/   ré   )rë   rê   c                    s   g | ]	}t ˆˆ ˆˆƒ‘qS r,   r‹  rŒ  ©r{  rz  r}  r|  r,   r-   r�  º  r�  z?IPAdapterFaceIDPlusImageProjection.__init__.<locals>.<listcomp>)r]   r^   rì   rè   rð   r=   Úclip_embedsÚshortcutÚshortcut_scalerc   r   rd   re   rž   r‘  r’  r“  r”  r•  r–  )rl   rz  r‡  rˆ  r›  r‰  r{  r|  rð   rŠ  r}  rœ  rè   ro   r�  r-   r^   š  s   

ÿz+IPAdapterFaceIDPlusImageProjection.__init__Ú	id_embedsc                 C   s¦   |  | jj¡}|  |¡}| d| j| j¡}|  |¡}|}|  | j¡}| d|j	d |j	d ¡}| j
D ]
}|}||||ƒ}q2|  |¡}|  |¡}| jrQ|| j|  }|S )zŸForward pass.

        Args:
            id_embeds (torch.Tensor): Input Tensor (ID embeds).
        Returns:
            torch.Tensor: Output Tensor.
        r   r   rS   )rw   rž  r   rc   r9   rð   r=   re   r‘  r   r–  r’  r“  rŸ  r   )rl   r¡  rƒ  rž  r’   r˜  r„  rP   r,   r,   r-   ry   ½  s   




z*IPAdapterFaceIDPlusImageProjection.forward)rT   rT   r†  r]  r‡   r5  r0   r‡   r)  r‡   r   r™  r,   r,   ro   r-   rš  ‰  sN    ôþýüûúùø	÷
öõôó#rš  c                       sJ   e Zd Zdeeej eej f f‡ fdd„Zdee	j
 fdd„Z‡  ZS )ÚMultiIPAdapterImageProjectionÚIPAdapterImageProjectionLayersc                    s   t ƒ  ¡  t |¡| _d S r¦   )r]   r^   r   r”  Úimage_projection_layers)rl   r£  ro   r,   r-   r^   Ú  s   
z&MultiIPAdapterImageProjection.__init__rÞ   c                 C   sÚ   g }t |tƒsd}tdd|dd� | d¡g}t|ƒt| jƒkr/tdt|ƒ› dt| jƒ› �ƒ‚t|| jƒD ]5\}}|jd	 |jd }}| 	|| f|jd
d …  ¡}||ƒ}| 	||f|jdd …  ¡}| 
|¡ q5|S )NzÒYou have passed a tensor as `image_embeds`.This is deprecated and will be removed in a future release. Please make sure to update your script to pass `image_embeds` as a list of tensors to supress this warning.zimage_embeds not a listz1.0.0F)Ústandard_warnr	   zGimage_embeds must have the same length as image_projection_layers, got z and r   r   )r4   Úlistr   rk   r   r¤  rH   Úzipr   r9   Úappend)rl   rÞ   Zprojected_image_embedsÚdeprecation_messageZimage_embedZimage_projection_layerrá   Z
num_imagesr,   r,   r-   ry   Þ  s"   
ÿÿz%MultiIPAdapterImageProjection.forward)rz   r{   r|   r   r   r   ÚModuler   r^   r   rã   ry   r~   r,   r,   ro   r-   r¢  Ù  s    &r¢  )Fr	   r	   r   )Fr   r/   r0   )Trr  )r‰   F)>r   Útypingr   r   r   r   Únumpyr6   r   Útorch.nn.functionalr   r'   r#  Úutilsr   Úactivationsr
   r   Úattention_processorr   rã   r5   r¬   r#   r.   rF   r:   rI   rª  rQ   r†   rƒ   Úndarrayrˆ   r—   r˜   rª   r­   r¶   r¿   rÈ   r×   rä   rå   rï   rñ   r   r  r'  r4  r<  r@  rD  r8  rZ  r[  rk  r  ry  r…  rš  r¢  r,   r,   r,   r-   Ú<module>   sˆ   úÿþýüû
ú,
ÿ
l
"!ÿþ
ý0A$(*2]%!;P