o
    óôj{  ã                   @   s8   d Z ddlZG dd„ dejjƒZG dd„ dejjƒZdS )z+Positionwise feed forward layer definition.é    Nc                	       sV   e Zd ZdZej ¡ fdedededejj	f‡ fdd„Z
dejd	ejfd
d„Z‡  ZS )ÚPositionwiseFeedForwardaa  Positionwise feed forward layer.

    FeedForward are appied on each position of the sequence.
    The output dim is same with the input dim.

    Args:
        idim (int): Input dimenstion.
        hidden_units (int): The number of hidden units.
        dropout_rate (float): Dropout rate.
        activation (torch.nn.Module): Activation function
    ÚidimÚhidden_unitsÚdropout_rateÚ
activationc                    sF   t t| ƒ ¡  tj ||¡| _|| _tj |¡| _	tj ||¡| _
dS )z+Construct a PositionwiseFeedForward object.N)Úsuperr   Ú__init__ÚtorchÚnnÚLinearÚw_1r   ÚDropoutÚdropoutÚw_2)Úselfr   r   r   r   ©Ú	__class__© úe/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/transformer/positionwise_feed_forward.pyr   !   s
   z PositionwiseFeedForward.__init__ÚxsÚreturnc              	   C   s   |   |  |  |  |¡¡¡¡S )z†Forward function.

        Args:
            xs: input tensor (B, L, D)
        Returns:
            output tensor, (B, L, D)
        )r   r   r   r   )r   r   r   r   r   Úforward/   s   zPositionwiseFeedForward.forward©Ú__name__Ú
__module__Ú__qualname__Ú__doc__r	   r
   ÚReLUÚintÚfloatÚModuler   ÚTensorr   Ú__classcell__r   r   r   r   r      s    ûþýüûr   c                       s^   e Zd ZdZej ¡ fdedededededejj	f‡ fdd	„Z
d
ejdejfdd„Z‡  ZS )ÚMoEFFNLayera”  
    Mixture of expert with Positionwise feed forward layer
    See also figure 1 in https://arxiv.org/pdf/2305.15663.pdf
    The output dim is same with the input dim.

    Modified from https://github.com/Lightning-AI/lit-gpt/pull/823
                  https://github.com/mistralai/mistral-src/blob/b46d6/moe_one_file_ref.py#L203-L219
    Args:
        n_expert: number of expert.
        n_expert_per_token: The actual number of experts used for each frame
        idim (int): Input dimenstion.
        hidden_units (int): The number of hidden units.
        dropout_rate (float): Dropout rate.
        activation (torch.nn.Module): Activation function
    Ún_expertÚn_expert_per_tokenr   r   r   r   c                    sR   t t| ƒ ¡  tjjˆ|dd�| _tj ‡ ‡‡‡fdd„t|ƒD ƒ¡| _	|| _
d S )NF)Úbiasc                 3   s   � | ]
}t ˆˆˆˆ ƒV  qd S )N)r   )Ú.0Ú_©r   r   r   r   r   r   Ú	<genexpr>V   s   € ÿÿ
ÿz'MoEFFNLayer.__init__.<locals>.<genexpr>)r   r#   r   r	   r
   r   ÚgateÚ
ModuleListÚrangeÚexpertsr%   )r   r$   r%   r   r   r   r   r   r)   r   r   K   s   	
þ
zMoEFFNLayer.__init__r   r   c                 C   s¾   |  ¡ \}}}| d|¡}|  |¡}t || j¡\}}tjjj|dtj	d�j
|jd�}t |¡}	t| jƒD ]"\}
}||
k}t |¡\}}|	|  |||df ||| ƒ 7  < q5|	 |||¡S )z…Foward function.
        Args:
            xs: input tensor (B, L, D)
        Returns:
            output tensor, (B, L, D)

        éÿÿÿÿé   )ÚdimÚdtype)r2   N)ÚsizeÚviewr+   r	   Útopkr%   r
   Ú
functionalÚsoftmaxr   Útor2   Ú
zeros_likeÚ	enumerater.   Úwhere)r   r   ÚBÚLÚDÚrouterÚlogitsÚindicesÚweightsÚoutputÚiZexpertÚmaskÚ	batch_idxZ
ith_expertr   r   r   r   [   s(   
ÿþþ
ÿzMoEFFNLayer.forwardr   r   r   r   r   r#   :   s"    ùþýüûúùr#   )r   r	   r
   r    r   r#   r   r   r   r   Ú<module>   s   &