o
    ²ƒðjK  ã                   @   sÜ   d Z ddlZddlZddlZddlmZ ddlmZ ddlZ	ddl
Z
ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZmZ ddlmZmZmZ ddlm Z  e !e"¡Z#G dd„ de	j$ƒZ%dS )zPyTorch Lightning module.é    N)Úreduce)ÚOptional)Úautocast)Ú
functionalé   )Úslice_segments)ÚBatch)Údiscriminator_lossÚfeature_lossÚgenerator_lossÚkl_loss)Úmel_spectrogram_torchÚspec_to_mel_torch)ÚMultiPeriodDiscriminatorÚMultiResolutionDiscriminatorÚSynthesizerTrn)ÚMosPredictorc                O       s¬  e Zd Z												
		
																										 					!	dad"ed#ed$ed%ed&ed'ed(ed)ed*ed+ee d,ed-ed.ed/ed0ed1ed2ed3ed4ed5ed6ed7ed8ed9ed:ed;eeef d<eeef d=ed>ed?ed@edAedBedCedDee dEee dFee dGee dHe	fN‡ fdIdJ„Z
dbdKdL„ZdMefdNdO„ZdMedPefdQdR„ZdMedPefdSdT„ZdcdVdW„ZdXdY„ ZdZefd[d\„ZdZefd]d^„Zd_d`„ Z‡  ZS )dÚ	VitsModelé    é"V  é   r   Ú2©é   é   é   ©)r   é   )r   é   )r   é   ©é   r!   é   ©é   r$   r!   é   éP   ç        NéÀ   é   r   r   r   çš™™™™™¹?Fr   Té    ç-Cëâ6*?ç-Cëâ6?©çš™™™™™é?g®Gáz®ï?©g      à?gÍÌÌÌÌÌì?ç•Ö&è.>çåÐ"Ûùþï?ç§èH.ÿï?ç      ð?é-   ÚutmosÚ
batch_sizeÚsample_rateÚnum_symbolsÚnum_speakersÚfilter_lengthÚ
hop_lengthÚ
win_lengthÚmel_channelsÚmel_fminÚmel_fmaxÚinter_channelsÚhidden_channelsÚfilter_channelsÚn_headsÚn_layersÚkernel_sizeÚ	p_dropoutÚ
n_layers_qÚuse_spectral_normÚgin_channelsÚuse_sdpÚsegment_sizeÚuse_mrdÚlearning_rateÚlearning_rate_dÚbetasÚbetas_dÚepsÚlr_decayÚ
lr_decay_dÚinit_lr_ratioÚwarmup_epochsÚc_melÚc_klÚ	grad_clipÚvocoder_warmstart_ckptÚwarmstart_ckptÚ
mos_metricÚdatasetc.           0         sF  t ƒ  ¡  |  ¡  t| jjtƒrt | jj¡| j_t| jj	tƒr)t | jj	¡| j_	t| jj
tƒr9t | jj
¡| j_
t| jjtƒrIt | jj¡| j_t| jjtƒrYt | jj¡| j_ttj| jj
dƒ}/|/|krjtdƒ‚d| _|| _| jjdkr€| jjdkr€d| j_|*| _|+| _tdi d|“d| jjd d “d	| jj| jj “d
| jj“d| jj“d| jj“d| jj“d| jj“d| jj “d| jj!“d| jj"“d| jj“d| jj	“d| jj
“d| jj#“d| jj“d| jj“d| jj“d| jj$“Ž| _%t&| jj'd�| _(d | _)| jj*�r	t+| jj'd�| _)d | _,|,�r|, -¡ dk�r!t.|, -¡ ƒ| _,d S d S d S )Nr   z&Upsample rates do not match hop lengthFr   i   Ún_vocabÚspec_channelsr   rL   rA   rB   rC   rD   rE   rF   rG   ÚresblockÚresblock_kernel_sizesÚresblock_dilation_sizesÚupsample_ratesÚupsample_initial_channelÚupsample_kernel_sizesÚ
n_speakersrJ   rK   )rI   Únone© )/ÚsuperÚ__init__Úsave_hyperparametersÚ
isinstanceÚhparamsra   ÚstrÚastÚliteral_evalrb   rc   re   rP   r   ÚoperatorÚmulÚ
ValueErrorÚautomatic_optimizationr7   r:   rJ   Ú_vocoder_warmstart_ckptÚ_warmstart_ckptr   r;   rL   r<   rA   rB   rC   rD   rE   rF   rG   r`   rd   rK   Úmodel_gr   rI   Úmodel_dÚ	model_mrdrM   r   Ú_mos_predictorÚlowerr   )0Úselfr7   r8   r9   r:   r`   ra   rb   rc   rd   re   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   ÚkwargsÚexpected_hop_length©Ú	__class__rh   úœ/tmp/claude-1000/-var-www-html-academy/3f42efd1-c2b7-4d08-9b05-30d86bb35479/scratchpad/piper/venv/lib/python3.10/site-packages/piper/train/vits/lightning.pyrj      sœ   
Fÿÿÿÿþýüûúùø	÷
öõôóòñðïîíÿ
ÿÿzVitsModel.__init__c           
      C   s8   |d }|d }|d }| j j||||||d�^}}	|S )Nr   r   r   )Únoise_scaleÚlength_scaleÚnoise_scale_wÚsid)rw   Úinfer)
r|   ÚtextÚtext_lengthsÚscalesr…   r‚   rƒ   r„   ÚaudioÚ_rh   rh   r�   Úforward³   s   
ú	zVitsModel.forwardÚbatchc           4   
   C   sj  |j |j|j|j|j|j|jd ur|jnd f\}}}}}}}|  |||||¡\}	}
}}}}\}}}}}}t| j	j
dd��G t| ¡ | jj| jj| jj| jj| jjƒ}t||| jj| jj ƒ}t|	 d¡ ¡ | jj| jj| jj| jj| jj| jj| jjƒ}W d   ƒ n1 s†w   Y  t||| jj | jjƒ}|	dd |jd …f }	|  ||	¡\}}}}| jd ur»|  ||	¡\}}}}t| j	j
dd��_ t ||¡} t|||||ƒ}!t  |
 ¡ ¡}"| | jj! }#|!| jj" }$t#||ƒ}%t$|ƒ\}&}'|&|% |# |" |$ }(| %d¡})| %d¡}*| jd u�rt#||ƒ}*t$|ƒ\})}|(|) |* }(W d   ƒ n	1 �s&w   Y  |  ||	 &¡ ¡\}+}}}| jd u�rI|  ||	 &¡ ¡\},}-}}t| j	j
dd��) t'|+|ƒ\}.}/}0|.}1| %d¡}2| jd u�rst'|,|-ƒ\}2}}|1|2 }1W d   ƒ n	1 �s~w   Y  |  &¡ |! &¡ |" &¡ |% &¡ |& &¡ |. &¡ dœ}3| jd u�r°|) &¡ |3d< |* &¡ |3d	< |2 &¡ |3d
< |(|1|3fS )NF)Úenabledr   .éÿÿÿÿrh   )ÚmelÚklÚdurÚfmÚgenÚdiscÚmrd_genÚmrd_fmÚmrd_disc)(Úphoneme_idsÚphoneme_lengthsÚaudiosÚaudio_lengthsÚspectrogramsÚspectrogram_lengthsÚspeaker_idsrw   r   ÚdeviceÚtyper   Úfloatrm   r;   r>   r8   r?   r@   r   rL   r<   r   Úsqueezer=   Úshaperx   ry   ÚFÚl1_lossr   ÚtorchÚsumrW   rX   r
   r   Ú	new_zerosÚdetachr	   )4r|   r�   ÚxÚ	x_lengthsÚyr‹   ÚspecÚspec_lengthsrŸ   Úy_hatÚl_lengthÚ_attnÚ	ids_sliceÚ_x_maskÚz_maskÚ_zÚz_pÚm_pÚlogs_pÚ_m_qÚlogs_qr�   Úy_melÚ	y_hat_melÚ
_y_d_hat_rÚ	y_d_hat_gÚfmap_rÚfmap_gÚ_y_dr_rÚy_dr_gÚ
fmap_r_mrdÚ
fmap_g_mrdÚmel_l1r‘   Úloss_durÚloss_melÚloss_klÚloss_fmÚloss_genÚ_losses_genÚloss_gen_allÚloss_mrd_genÚloss_mrd_fmÚ	y_d_hat_rÚ
y_dr_r_detÚ
y_dr_g_detÚ	loss_discÚ_losses_disc_rÚ_losses_disc_gÚloss_disc_allÚloss_mrd_discÚmetricsrh   rh   r�   Ú_compute_lossÂ   s¼   ùøúýøò
ý




€í
ÿ
€öú
zVitsModel._compute_lossÚ	batch_idxc                 C   s–   |   ¡ \}}|  |¡\}}}| jd|| jd� | jdd„ | ¡ D ƒ| jd� | ¡  |  |¡ | ¡  | jd|| jd� | ¡  |  |¡ | ¡  d S )NÚloss_g©r7   c                 S   ó   i | ]
\}}d |› �|“qS )Útrain_rh   ©Ú.0ÚnameÚvaluerh   rh   r�   Ú
<dictcomp>?  ó    z+VitsModel.training_step.<locals>.<dictcomp>Úloss_d)	Ú
optimizersrÙ   Úlogr7   Úlog_dictÚitemsÚ	zero_gradÚmanual_backwardÚstep)r|   r�   rÚ   Úopt_gÚopt_drÛ   rå   rØ   rh   rh   r�   Útraining_step9  s   þ

zVitsModel.training_stepc                 C   sJ   |   |¡\}}}|}| jd|| jd� | jdd„ | ¡ D ƒ| jdd� |S )NÚval_lossrÜ   c                 S   rÝ   )Úval_rh   rß   rh   rh   r�   rã   W  rä   z-VitsModel.validation_step.<locals>.<dictcomp>T)r7   Ú	sync_dist)rÙ   rç   r7   rè   ré   )r|   r�   rÚ   rÛ   Ú_loss_drØ   rð   rh   rh   r�   Úvalidation_stepN  s   ýzVitsModel.validation_stepÚreturnc              	   C   s”  | j jrd S tt| dd ƒot| jdƒot| jjdƒƒ}| jd u}|s&|s&d S g }t| j j	j
ƒD ]ƒ\}}|j d¡ | j¡}t t|jƒg¡ | j¡}g d¢}|jd urZ|j | j¡nd }	t ¡ � | ||||	d� ¡ }
W d   ƒ n1 suw   Y  |rŽ| j |
| jj¡}|d urŽ| |¡ |r²|
dtdt|
ƒ ¡ ƒ  }|jp£t|ƒ}| jjj||| jj| jd	� q/|rÈt|ƒt|ƒ }| j d
|ddd� d S d S )NÚloggerÚ
experimentÚ	add_audior   )gòÒMbXå?r4   r/   )r…   r4   g{®Gáz„?)r8   Úglobal_stepÚval_mosT)Úprog_barrò   )!ÚtrainerÚsanity_checkingÚboolÚgetattrÚhasattrrö   r÷   rz   Ú	enumerateÚ
datamoduleÚtest_datasetr™   Ú	unsqueezeÚtor    r§   Ú
LongTensorÚlenÚ
speaker_idÚinference_moderª   Úscorerm   r8   ÚappendÚmaxÚabsr‡   rn   rø   rù   r¨   rç   )r|   Úhas_audio_loggerÚmos_enabledÚ
mos_scoresÚutt_idxÚtest_uttr‡   rˆ   r‰   r…   Ú
test_audior
  Ú
norm_audioÚtagrú   rh   rh   r�   Úon_validation_epoch_end]  sR   
ÿý

ÿý
ÿ
ü€þz!VitsModel.on_validation_epoch_endc                 C   s¬   t | j ¡ ƒ}| jd ur|t | j ¡ ƒ7 }tjj| j ¡ | jj	| jj
| jjd�tjj|| jj| jj| jjd�g}tjjj|d | jjd�tjjj|d | jjd�g}||fS )N)ÚlrrP   rR   r   )Úgammar   )Úlistrx   Ú
parametersry   r§   ÚoptimÚAdamWrw   rm   rN   rP   rR   rO   rQ   Úlr_schedulerÚExponentialLRrS   rT   )r|   Údisc_paramsræ   Ú
schedulersrh   rh   r�   Úconfigure_optimizers•  s0   
üüùÿÿü	zVitsModel.configure_optimizersÚ	ckpt_pathc           	      C   s˜   t j|| jdd�}|d }|  ¡ }d}d}| ¡ D ]\}}| |¡s#q||v r7|| j|jkr7|||< |d7 }q| j|dd� t 	d|› d	|› �¡ d S )
NF©Úmap_locationÚweights_onlyÚ
state_dict)zmodel_g.dec.zmodel_g.enc_q.zmodel_g.flow.r   r   ©Ústrictz[warmstart] Copied z vocoder parameters from )
r§   Úloadr    r&  ré   Ú
startswithr¤   Úload_state_dictÚ_LOGGERÚinfo)	r|   r"  ÚckptÚold_sdÚnew_sdÚKEEP_PREFIXESÚcopiedÚkÚvrh   rh   r�   Ú_warmstart_vocoder_from_ckptµ  s   
€z&VitsModel._warmstart_vocoder_from_ckptc           	      C   s�   t j|| jdd�}|d }|  ¡ }d\}}| ¡ D ]\}}||v r2|| j|jkr2|||< |d7 }q|d7 }q| j|dd� t d|||¡ dS )	z>Copy all matching-shape params from a checkpoint (non-strict).Fr#  r&  )r   r   r   r'  z^[warmstart] Copied %s parameters from %s (%s skipped; new modules such as the MRD start fresh)N)	r§   r)  r    r&  ré   r¤   r+  r,  r-  )	r|   r"  r.  r/  r0  r2  Úskippedr3  r4  rh   rh   r�   Ú_warmstart_from_ckptÍ  s    

ûzVitsModel._warmstart_from_ckptc                 C   s@   | j d ur|  | j ¡ d | _ | jd ur|  | j¡ d | _d S d S ©N)ru   r5  rv   r7  )r|   rh   rh   r�   Úon_fit_startä  s   


þzVitsModel.on_fit_start)-r   r   r   r   r   r   r   r    r   r#   r%   r   r%   r&   r'   Nr(   r(   r)   r   r   r   r*   r   Fr   Tr+   Fr,   r-   r.   r0   r1   r2   r3   r4   r   r5   r4   NNNr6   Nr8  )rõ   N)Ú__name__Ú
__module__Ú__qualname__Úintr¢   r   rþ   Útuplern   Úobjectrj   rŒ   r   rÙ   rï   rô   r  r!  r5  r7  r9  Ú__classcell__rh   rh   r   r�   r      s   ½þýüûîíìëêéçæåäãâá à!ß"Þ#Ý$Ü)×+Õ,Ô
-Ó
.Ò/Ñ0Ð1Ï2Î3Í4Ì5Ë6Ê7É=ÃA¿C½ 
w
8 r   )&Ú__doc__ro   Úloggingrq   Ú	functoolsr   Útypingr   Ú	lightningÚLr§   r   Útorch.nnr   r¥   Úcommonsr   r]   r   Úlossesr	   r
   r   r   Úmel_processingr   r   Úmodelsr   r   r   Úmosr   Ú	getLoggerr:  r,  ÚLightningModuler   rh   rh   rh   r�   Ú<module>   s$    
