o
    šôj�I  ã                   @   sö   d dl mZmZmZ d dlZd dlm  mZ d dlmZ ddl	m
Z
mZ ddlmZ eƒ r6d dlmZmZ e e¡Zdd	„ Zd
d„ Zddefdd„ZG dd„ dejjƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZG dd„ dejƒZdS )é    )ÚOptionalÚTupleÚUnionN)Únné   )Ú	deprecateÚlogging)Úis_transformers_available)ÚCLIPTextModelÚCLIPTextModelWithProjectionc                 C   s`   g }t | ttfƒr't| jjjƒD ]\}}d|› d�}|j}| ||f¡ q|S t	d| j
j› �ƒ‚)Nútext_model.encoder.layers.z
.self_attnz.do not know how to get attention modules for: )Ú
isinstancer
   r   Ú	enumerateÚ
text_modelÚencoderÚlayersÚ	self_attnÚappendÚ
ValueErrorÚ	__class__Ú__name__)Útext_encoderZattn_modulesÚiÚlayerÚnameÚmod© r   úh/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/diffusers/models/lora.pyÚtext_encoder_attn_modules)   s   þr   c                 C   s`   g }t | ttfƒr't| jjjƒD ]\}}|j}d|› d�}| ||f¡ q|S t	d| j
j› �ƒ‚)Nr   z.mlpz(do not know how to get mlp modules for: )r   r
   r   r   r   r   r   Úmlpr   r   r   r   )r   Zmlp_modulesr   r   Zmlp_modr   r   r   r   Útext_encoder_mlp_modules7   s   þr    ç      ð?Ú
lora_scalec                 C   sp   t | ƒD ]\}}t|jtƒr||j_||j_||j_||j_qt| ƒD ]\}}t|j	tƒr5||j	_||j
_q#d S ©N)r   r   Úq_projÚPatchedLoraProjectionr"   Úk_projÚv_projÚout_projr    Úfc1Úfc2)r   r"   Ú_Zattn_moduleZ
mlp_moduler   r   r   Úadjust_lora_scale_text_encoderE   s   €€ýr,   c                       sN   e Zd Zd‡ fdd„	Zddddœ‡ fd	d
„
Zddd„Zdd„ Zdd„ Z‡  ZS )r%   é   Né   c           	         sp   d}t dd|ƒ tƒ  ¡  ddlm} || _| jjj}|d u r$| jjj}|| jj	| jj
||||d�| _|| _d S )NzsUse of `PatchedLoraProjection` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r%   ú1.0.0r   )ÚLoRALinearLayer)Únetwork_alphaÚdeviceÚdtypeÚrank)r   ÚsuperÚ__init__Zmodels.lorar0   Úregular_linear_layerÚweightr2   r3   Úin_featuresÚout_featuresÚlora_linear_layerr"   )	Úselfr7   r"   r1   r4   r3   Údeprecation_messager0   r2   ©r   r   r   r6   T   s"   


ú
	zPatchedLoraProjection.__init__Ú F©ÚdestinationÚprefixÚ	keep_varsc                   s6   | j d u r| jj||||dœŽS tƒ j||||dœŽS )Nr@   )r;   r7   Ú
state_dictr5   )r<   rA   rB   rC   Úargsr>   r   r   rD   o   s
   
ÿz PatchedLoraProjection.state_dictr!   c           	   	   C   s  | j d u rd S | jjjj| jjjj}}| jjj ¡ }| j jjj ¡ }| j jjj ¡ }| j j	d ur;|| j j	 | j j
 }||t |d d d …f |d d d …f ¡d   }|rft |¡ ¡  ¡ rftd| › d�ƒ‚|j||d�| jj_d | _ | ¡ | _| ¡ | _|| _d S ©Nr   úaThis LoRA weight seems to be broken. Encountered NaN values when trying to fuse LoRA weights for ú .LoRA weights will not be fused.©r2   r3   )r;   r7   r8   Údatar3   r2   ÚfloatÚupÚdownr1   r4   ÚtorchÚbmmÚisnanÚanyÚitemr   ÚtoÚcpuÚw_upÚw_downr"   ©	r<   r"   Úsafe_fusingr3   r2   Úw_origrU   rV   Úfused_weightr   r   r   Ú
_fuse_loraw   s(   
0ÿÿ


z PatchedLoraProjection._fuse_lorac              	   C   s¸   t | dd ƒd urt | dd ƒd usd S | jjj}|j|j}}| jj|d� ¡ }| j	 |¡ ¡ }| ¡ | j
t |d d d …f |d d d …f ¡d   }|j||d�| jj_d | _d | _	d S ©NrU   rV   ©r2   r   rI   )Úgetattrr7   r8   rJ   r3   r2   rU   rS   rK   rV   r"   rN   rO   ©r<   rZ   r3   r2   rU   rV   Úunfused_weightr   r   r   Ú_unfuse_lora—   s    
6
z"PatchedLoraProjection._unfuse_lorac                 C   s>   | j d u rd| _ | jd u r|  |¡S |  |¡| j |  |¡  S )Nr!   )r"   r;   r7   )r<   Úinputr   r   r   Úforward§   s
   


zPatchedLoraProjection.forward)r-   Nr.   N©r!   F)	r   Ú
__module__Ú__qualname__r6   rD   r[   ra   rc   Ú__classcell__r   r   r>   r   r%   S   s    
 r%   c                       sr   e Zd ZdZ				ddedededee deeej	e
f  d	eej f‡ fd
d„Zdejdejfdd„Z‡  ZS )r0   a‘  
    A linear layer that is used with LoRA.

    Parameters:
        in_features (`int`):
            Number of input features.
        out_features (`int`):
            Number of output features.
        rank (`int`, `optional`, defaults to 4):
            The rank of the LoRA layer.
        network_alpha (`float`, `optional`, defaults to `None`):
            The value of the network alpha used for stable learning and preventing underflow. This value has the same
            meaning as the `--network_alpha` option in the kohya-ss trainer script. See
            https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning
        device (`torch.device`, `optional`, defaults to `None`):
            The device to use for the layer's weights.
        dtype (`torch.dtype`, `optional`, defaults to `None`):
            The dtype to use for the layer's weights.
    r.   Nr9   r:   r4   r1   r2   r3   c                    sŠ   t ƒ  ¡  d}tdd|ƒ tj||d||d�| _tj||d||d�| _|| _|| _|| _	|| _
tjj| jjd| d� tj | jj¡ d S )NzmUse of `LoRALinearLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r0   r/   F)Úbiasr2   r3   r-   ©Ústd)r5   r6   r   r   ÚLinearrM   rL   r1   r4   r:   r9   ÚinitÚnormal_r8   Úzeros_)r<   r9   r:   r4   r1   r2   r3   r=   r>   r   r   r6   Ä   s   
	zLoRALinearLayer.__init__Úhidden_statesÚreturnc                 C   óN   |j }| jjj }|  | |¡¡}|  |¡}| jd ur"|| j| j 9 }| |¡S r#   ©r3   rM   r8   rS   rL   r1   r4   ©r<   ro   Ú
orig_dtyper3   Zdown_hidden_statesZup_hidden_statesr   r   r   rc   Þ   ó   



zLoRALinearLayer.forward)r.   NNN)r   re   rf   Ú__doc__Úintr   rK   r   rN   r2   Ústrr3   r6   ÚTensorrc   rg   r   r   r>   r   r0   ¯   s(    ùþýüûúùr0   c                       s–   e Zd ZdZ					ddededed	eeeeef f d
eeeeef f deeeeef ef dee	 f‡ fdd„Z
dejdejfdd„Z‡  ZS )ÚLoRAConv2dLayera"  
    A convolutional layer that is used with LoRA.

    Parameters:
        in_features (`int`):
            Number of input features.
        out_features (`int`):
            Number of output features.
        rank (`int`, `optional`, defaults to 4):
            The rank of the LoRA layer.
        kernel_size (`int` or `tuple` of two `int`, `optional`, defaults to 1):
            The kernel size of the convolution.
        stride (`int` or `tuple` of two `int`, `optional`, defaults to 1):
            The stride of the convolution.
        padding (`int` or `tuple` of two `int` or `str`, `optional`, defaults to 0):
            The padding of the convolution.
        network_alpha (`float`, `optional`, defaults to `None`):
            The value of the network alpha used for stable learning and preventing underflow. This value has the same
            meaning as the `--network_alpha` option in the kohya-ss trainer script. See
            https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning
    r.   ©r-   r-   r   Nr9   r:   r4   Úkernel_sizeÚstrideÚpaddingr1   c           	         s€   t ƒ  ¡  d}tdd|ƒ tj|||||dd�| _tj||dddd�| _|| _|| _tj	j
| jjd| d	� tj	 | jj¡ d S )
NzmUse of `LoRAConv2dLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.rz   r/   F)r|   r}   r~   rh   r{   )r|   r}   rh   r-   ri   )r5   r6   r   r   ÚConv2drM   rL   r1   r4   rl   rm   r8   rn   )	r<   r9   r:   r4   r|   r}   r~   r1   r=   r>   r   r   r6     s   

zLoRAConv2dLayer.__init__ro   rp   c                 C   rq   r#   rr   rs   r   r   r   rc     ru   zLoRAConv2dLayer.forward)r.   r{   r{   r   N)r   re   rf   rv   rw   r   r   rx   r   rK   r6   rN   ry   rc   rg   r   r   r>   r   rz   ë   s.    øþýüûúùørz   c                       sz   e Zd ZdZddœdee f‡ fdd„Zdee fdd„Zddede	fdd„Z
dd„ Zddejdedejfdd„Z‡  ZS )ÚLoRACompatibleConvz;
    A convolutional layer that can be used with LoRA.
    N©Ú
lora_layerr‚   c                   ó,   d}t dd|ƒ tƒ j|i |¤Ž || _d S )NzpUse of `LoRACompatibleConv` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r€   r/   ©r   r5   r6   r‚   ©r<   r‚   rE   Úkwargsr=   r>   r   r   r6   0  ó   
zLoRACompatibleConv.__init__c                 C   ó   d}t dd|ƒ || _d S ©NznUse of `set_lora_layer()` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.Úset_lora_layerr/   ©r   r‚   ©r<   r‚   r=   r   r   r   rŠ   7  s   
z!LoRACompatibleConv.set_lora_layerr!   Fr"   rX   c           
      C   s   | j d u rd S | jjj| jjj}}| jj ¡ }| j jjj ¡ }| j jjj ¡ }| j jd ur8|| j j | j j	 }t
 |jdd�|jdd�¡}| |j¡}|||  }	|ret
 |	¡ ¡  ¡ retd| › d�ƒ‚|	j||d�| j_d | _ | ¡ | _| ¡ | _|| _d S )Nr-   ©Ú	start_dimrG   rH   rI   )r‚   r8   rJ   r3   r2   rK   rL   rM   r1   r4   rN   ÚmmÚflattenÚreshapeÚshaperP   rQ   rR   r   rS   rT   rU   rV   Ú_lora_scale)
r<   r"   rX   r3   r2   rY   rU   rV   ÚfusionrZ   r   r   r   r[   =  s,   
ÿÿ


zLoRACompatibleConv._fuse_lorac                 C   sÄ   t | dd ƒd urt | dd ƒd usd S | jj}|jj|jj}}| jj|d� ¡ | _| j |¡ ¡ | _t	 
| jjdd�| jjdd�¡}| |j¡}| ¡ | j|  }|j||d�| j_d | _d | _d S )NrU   rV   r]   r-   r�   rI   )r^   r8   rJ   r3   r2   rU   rS   rK   rV   rN   r�   r�   r‘   r’   r“   )r<   rZ   r3   r2   r”   r`   r   r   r   ra   _  s     
zLoRACompatibleConv._unfuse_loraro   Úscalerp   c              	   C   sj   | j dkrtj|| j| j d�}d}n| j}t || j| j| j|| j	| j
¡}| jd u r,|S |||  |¡  S )NÚzeros)Úmode)r   r   )Úpadding_modeÚFÚpadÚ _reversed_padding_repeated_twicer~   Úconv2dr8   rh   r}   ÚdilationÚgroupsr‚   )r<   ro   r•   r~   Zoriginal_outputsr   r   r   rc   q  s   
ÿ
zLoRACompatibleConv.forwardrd   ©r!   )r   re   rf   rv   r   rz   r6   rŠ   rK   Úboolr[   ra   rN   ry   rc   rg   r   r   r>   r   r€   +  s    "$r€   c                       s~   e Zd ZdZddœdee f‡ fdd„Zdee fdd„Zddede	fdd„Z
dd„ Zddejdedejf‡ fdd„Z‡  ZS )ÚLoRACompatibleLinearz4
    A Linear layer that can be used with LoRA.
    Nr�   r‚   c                   rƒ   )NzrUse of `LoRACompatibleLinear` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`.r¡   r/   r„   r…   r>   r   r   r6   ‡  r‡   zLoRACompatibleLinear.__init__c                 C   rˆ   r‰   r‹   rŒ   r   r   r   rŠ   Ž  s   
z#LoRACompatibleLinear.set_lora_layerr!   Fr"   rX   c           	   	   C   sü   | j d u rd S | jjj| jjj}}| jj ¡ }| j jjj ¡ }| j jjj ¡ }| j jd ur8|| j j | j j	 }||t
 |d d d …f |d d d …f ¡d   }|rct
 |¡ ¡  ¡ rctd| › d�ƒ‚|j||d�| j_d | _ | ¡ | _| ¡ | _|| _d S rF   )r‚   r8   rJ   r3   r2   rK   rL   rM   r1   r4   rN   rO   rP   rQ   rR   r   rS   rT   rU   rV   r“   rW   r   r   r   r[   “  s(   
0ÿÿ


zLoRACompatibleLinear._fuse_lorac              	   C   s´   t | dd ƒd urt | dd ƒd usd S | jj}|j|j}}| jj|d� ¡ }| j |¡ ¡ }| ¡ | j	t
 |d d d …f |d d d …f ¡d   }|j||d�| j_d | _d | _d S r\   )r^   r8   rJ   r3   r2   rU   rS   rK   rV   r“   rN   rO   r_   r   r   r   ra   ³  s    6
z!LoRACompatibleLinear._unfuse_loraro   r•   rp   c                    s8   | j d u rtƒ  |¡}|S tƒ  |¡||   |¡  }|S r#   )r‚   r5   rc   )r<   ro   r•   Úoutr>   r   r   rc   Ã  s
   
zLoRACompatibleLinear.forwardrd   rŸ   )r   re   rf   rv   r   r0   r6   rŠ   rK   r    r[   ra   rN   ry   rc   rg   r   r   r>   r   r¡   ‚  s     (r¡   rŸ   ) Útypingr   r   r   rN   Útorch.nn.functionalr   Ú
functionalr™   Úutilsr   r   Úutils.import_utilsr	   Útransformersr
   r   Ú
get_loggerr   Úloggerr   r    rK   r,   ÚModuler%   r0   rz   r   r€   rk   r¡   r   r   r   r   Ú<module>   s"   
\<@W