o
    šôj˜&  ã                   @   s  d dl Z d dlmZmZmZ d dlZd dlmZ d dlm  m	Z
 ddlmZ ddlmZ ddlmZmZ G dd	„ d	ejƒZG d
d„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZeddƒrkejZn	G dd„ dejƒZG dd„ dejƒZG dd„ dejƒZdS )é    N)ÚDictÚOptionalÚTupleé   )Úis_torch_versioné   )Úget_activation)ÚCombinedTimestepLabelEmbeddingsÚ)PixArtAlphaCombinedTimestepSizeEmbeddingsc                       sF   e Zd ZdZdedef‡ fdd„Zdejdejdejfd	d
„Z‡  Z	S )ÚAdaLayerNormzÛ
    Norm layer modified to incorporate timestep embeddings.

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        num_embeddings (`int`): The size of the embeddings dictionary.
    Úembedding_dimÚnum_embeddingsc                    sH   t ƒ  ¡  t ||¡| _t ¡ | _t ||d ¡| _tj	|dd�| _
d S )Nr   F)Úelementwise_affine)ÚsuperÚ__init__ÚnnÚ	EmbeddingÚembÚSiLUÚsiluÚLinearÚlinearÚ	LayerNormÚnorm©Úselfr   r   ©Ú	__class__© úq/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/diffusers/models/normalization.pyr   %   s
   

zAdaLayerNorm.__init__ÚxÚtimestepÚreturnc                 C   s@   |   |  |  |¡¡¡}t |d¡\}}|  |¡d|  | }|S )Nr   r   )r   r   r   ÚtorchÚchunkr   )r   r    r!   r   ÚscaleÚshiftr   r   r   Úforward,   s   zAdaLayerNorm.forward)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Úintr   r#   ÚTensorr'   Ú__classcell__r   r   r   r   r      s    $r   c                       sŽ   e Zd ZdZddedee f‡ fdd„Z				ddejdeej d	eej	 d
eej
 deej deejejejejejf fdd„Z‡  ZS )ÚAdaLayerNormZerozÕ
    Norm layer adaptive layer norm zero (adaLN-Zero).

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        num_embeddings (`int`): The size of the embeddings dictionary.
    Nr   r   c                    s\   t ƒ  ¡  |d urt||ƒ| _nd | _t ¡ | _tj|d| dd�| _tj	|ddd�| _
d S )Né   T©ÚbiasFç�íµ ÷Æ°>)r   Úeps)r   r   r	   r   r   r   r   r   r   r   r   r   r   r   r   r   <   s   

zAdaLayerNormZero.__init__r    r!   Úclass_labelsÚhidden_dtyper   r"   c                 C   s€   | j d ur| j |||d�}|  |  |¡¡}|jddd�\}}}}	}
}|  |¡d|d d …d f   |d d …d f  }|||	|
|fS )N)r6   r0   r   ©Údim)r   r   r   r$   r   )r   r    r!   r5   r6   r   Ú	shift_msaÚ	scale_msaÚgate_msaÚ	shift_mlpÚ	scale_mlpÚgate_mlpr   r   r   r'   G   s   
.zAdaLayerNormZero.forward©N)NNNN)r(   r)   r*   r+   r,   r   r   r#   r-   Ú
LongTensorÚdtyper   r'   r.   r   r   r   r   r/   3   s(    úþýüûúùr/   c                       s„   e Zd ZdZddedef‡ fdd„Z			ddejd	e	e
eejf  d
e	e de	ej deejejejejejf f
dd„Z‡  ZS )ÚAdaLayerNormSingleaL  
    Norm layer adaptive layer norm single (adaLN-single).

    As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3).

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        use_additional_conditions (`bool`): To use additional conditions for normalization or not.
    Fr   Úuse_additional_conditionsc                    sB   t ƒ  ¡  t||d |d�| _t ¡ | _tj|d| dd�| _d S )Né   )Úsize_emb_dimrC   r0   Tr1   )	r   r   r
   r   r   r   r   r   r   )r   r   rC   r   r   r   r   b   s   

ÿ
zAdaLayerNormSingle.__init__Nr!   Úadded_cond_kwargsÚ
batch_sizer6   r"   c                 C   s0   | j |fi |¤||dœ¤Ž}|  |  |¡¡|fS )N)rG   r6   )r   r   r   )r   r!   rF   rG   r6   Zembedded_timestepr   r   r   r'   l   s   zAdaLayerNormSingle.forward)F)NNN)r(   r)   r*   r+   r,   Úboolr   r#   r-   r   r   ÚstrrA   r   r'   r.   r   r   r   r   rB   W   s"    
ûþýüûúrB   c                       sZ   e Zd ZdZ	ddedededee def
‡ fd	d
„Zde	j
de	j
de	j
fdd„Z‡  ZS )ÚAdaGroupNormañ  
    GroupNorm layer modified to incorporate timestep embeddings.

    Parameters:
        embedding_dim (`int`): The size of each embedding vector.
        num_embeddings (`int`): The size of the embeddings dictionary.
        num_groups (`int`): The number of groups to separate the channels into.
        act_fn (`str`, *optional*, defaults to `None`): The activation function to use.
        eps (`float`, *optional*, defaults to `1e-5`): The epsilon value to use for numerical stability.
    Nçñhãˆµøä>r   Úout_dimÚ
num_groupsÚact_fnr4   c                    sF   t ƒ  ¡  || _|| _|d u rd | _nt|ƒ| _t ||d ¡| _d S )Nr   )	r   r   rM   r4   Úactr   r   r   r   )r   r   rL   rM   rN   r4   r   r   r   r   „   s   

zAdaGroupNorm.__init__r    r   r"   c                 C   sl   | j r|   |¡}|  |¡}|d d …d d …d d f }|jddd�\}}tj|| j| jd�}|d|  | }|S )Nr   r   r7   )r4   )rO   r   r$   ÚFÚ
group_normrM   r4   )r   r    r   r%   r&   r   r   r   r'   ’   s   

zAdaGroupNorm.forward)NrK   )r(   r)   r*   r+   r,   r   rI   Úfloatr   r#   r-   r'   r.   r   r   r   r   rJ   x   s    ÿÿÿÿÿÿ$rJ   c                       sL   e Zd Z				ddedef‡ fdd„Zdejd	ejd
ejfdd„Z‡  ZS )ÚAdaLayerNormContinuousTrK   Ú
layer_normr   Úconditioning_embedding_dimc                    sn   t ƒ  ¡  t ¡ | _tj||d |d�| _|dkr#t||||ƒ| _d S |dkr0t	|||ƒ| _d S t
d|› �ƒ‚)Nr   r1   rT   Úrms_normzunknown norm_type )r   r   r   r   r   r   r   r   r   ÚRMSNormÚ
ValueError)r   r   rU   r   r4   r2   Ú	norm_typer   r   r   r   Ÿ   s   

zAdaLayerNormContinuous.__init__r    Úconditioning_embeddingr"   c                 C   sj   |   |  |¡ |j¡¡}tj|ddd�\}}|  |¡d| d d …d d d …f  |d d …d d d …f  }|S )Nr   r   r7   )r   r   ÚtorA   r#   r$   r   )r   r    rZ   r   r%   r&   r   r   r   r'   ·   s   :zAdaLayerNormContinuous.forward)TrK   TrT   )	r(   r)   r*   r,   r   r#   r-   r'   r.   r   r   r   r   rS   ž   s    
ôþý$rS   z>=z2.1.0c                       s4   e Zd Zd
dededef‡ fdd„Zdd	„ Z‡  ZS )r   rK   Tr4   r   r2   c                    sx   t ƒ  ¡  || _t|tjƒr|f}t |¡| _|r4t	 
t |¡¡| _|r/t	 
t |¡¡| _d S d | _d S d | _d | _d S r?   )r   r   r4   Ú
isinstanceÚnumbersÚIntegralr#   ÚSizer8   r   Ú	ParameterÚonesÚweightÚzerosr2   )r   r8   r4   r   r2   r   r   r   r   Å   s   
$
zLayerNorm.__init__c                 C   s   t  || j| j| j| j¡S r?   )rP   rT   r8   rb   r2   r4   )r   Úinputr   r   r   r'   Ö   s   zLayerNorm.forward)rK   TT©r(   r)   r*   rR   rH   r   r'   r.   r   r   r   r   r   Ä   s    r   c                       s0   e Zd Zddedef‡ fdd„Zdd„ Z‡  ZS )	rW   Tr4   r   c                    sR   t ƒ  ¡  || _t|tjƒr|f}t |¡| _|r$t	 
t |¡¡| _d S d | _d S r?   )r   r   r4   r\   r]   r^   r#   r_   r8   r   r`   ra   rb   )r   r8   r4   r   r   r   r   r   Û   s   

zRMSNorm.__init__c                 C   s~   |j }| tj¡ d¡jddd�}|t || j ¡ }| jd ur8| jj tj	tj
fv r1| | jj ¡}|| j }|S | |¡}|S )Nr   éÿÿÿÿT)Úkeepdim)rA   r[   r#   Úfloat32ÚpowÚmeanÚrsqrtr4   rb   Úfloat16Úbfloat16)r   Úhidden_statesÚinput_dtypeÚvariancer   r   r   r'   ê   s   


þzRMSNorm.forward)Tre   r   r   r   r   rW   Ú   s    rW   c                       s$   e Zd Z‡ fdd„Zdd„ Z‡  ZS )ÚGlobalResponseNormc                    s>   t ƒ  ¡  t t ddd|¡¡| _t t ddd|¡¡| _d S )Nr   )r   r   r   r`   r#   rc   ÚgammaÚbeta)r   r8   r   r   r   r   ü   s   
zGlobalResponseNorm.__init__c                 C   s@   t j|dddd�}||jddd�d  }| j||  | j | S )Nr   )r   r   T)Úpr8   rg   rf   )r8   rg   r3   )r#   r   rj   rr   rs   )r   r    ÚgxÚnxr   r   r   r'     s   zGlobalResponseNorm.forward)r(   r)   r*   r   r'   r.   r   r   r   r   rq   ú   s    rq   )r]   Útypingr   r   r   r#   Útorch.nnr   Útorch.nn.functionalÚ
functionalrP   Úutilsr   Úactivationsr   Ú
embeddingsr	   r
   ÚModuler   r/   rB   rJ   rS   r   rW   rq   r   r   r   r   Ú<module>   s$   $!&
! 