o
    óôjmA  ã                   @   s2  d dl Z d dlZd dlmZ d dlmZ d dlZd dl	Z	d dl
Z
d dlZd dlmZ d dlm  mZ d dlZd dlmZmZ h d£Zd4dd„Z			
					d5dd„Zd6dd„Zd7dd„Z		d8dd„Zd8dd„Zd4dd„Zd4dd„Zd4d d!„Z d9d#d$„Z!d:d&d'„Z"d;d)d*„Z#d<d,d-„Z$d=d/d0„Z%d>d2d3„Z&dS )?é    N)ÚBytesIO)Úpad_sequence)Úembedding_extractorÚonline_feature>   ÚwmaÚflacÚm4aÚwavÚopusÚmp3ÚoggÚtrainc                 c   s®   � | D ]Q}d|v sJ ‚|d }z*t  |¡jdd�D ]}| ¡ }tt|ƒƒD ]}| t|j| ƒ¡ i |¥V  q%qW q t	yT } zt
 d ||¡¡ W Y d}~qd}~ww dS )zÌ Give url or local file, return file descriptor
        Inplace operation.

        Args:
            data(Iterable[str]): url or local file list

        Returns:
            Iterable[{src, stream}]
    Úsrcé@   )Ú
batch_sizezFailed to open {}, ex info {}N)ÚpqÚParquetFileZiter_batchesÚ	to_pandasÚrangeÚlenÚupdateÚdictÚlocÚ	ExceptionÚloggingÚwarningÚformat)ÚdataÚmodeÚsampleÚurlÚdfÚiÚex© r$   úQ/var/www/html/academy/tools/voices/clone/CosyVoice/cosyvoice/dataset/processor.pyÚparquet_opener   s"   €
ýþ€ÿör&   é (  é
   éÈ   é   çü©ñÒMb@?c           
      c   s  � | D ]ƒ}t  t|d ƒ¡\|d< |d< |d jddd�|d< |d= |d  d¡|d  d }	|	|k r4q|	|kr9qt|d	 ƒ|k rBqt|d	 ƒ|krKqtd
u rXt|d ƒdkrXqtd
u rid|v rit|d ƒdkriq|	dkrƒt|d	 ƒ|	 |k rxqt|d	 ƒ|	 |krƒq|V  qdS )aX   Filter sample according to feature and label length
        Inplace operation.

        Args::
            data: Iterable[{key, wav, label, sample_rate}]
            max_length: drop utterance which is greater than max_length(10ms)
            min_length: drop utterance which is less than min_length(10ms)
            token_max_length: drop utterance which is greater than
                token_max_length, especially when use char unit for
                english modeling
            token_min_length: drop utterance which is
                less than token_max_length
            min_output_input_ratio: minimal ration of
                token_length / feats_length(10ms)
            max_output_input_ratio: maximum ration of
                token_length / feats_length(10ms)

        Returns:
            Iterable[{key, wav, label, sample_rate}]
    Ú
audio_dataÚspeechÚsample_rater   T)ÚdimÚkeepdimr*   éd   Ú
text_tokenFÚspeech_tokenÚreject_speech_tokenN)Ú
torchaudioÚloadr   ÚmeanÚsizer   r   )
r   Ú
max_lengthÚ
min_lengthZtoken_max_lengthZtoken_min_lengthZmin_output_input_ratioZmax_output_input_ratior   r   Ú
num_framesr$   r$   r%   Úfilter7   s2   € ér<   é"V  é€>  c                 c   s˜   � | D ]F}d|v sJ ‚d|v sJ ‚|d }|d }||kr2||k r"q||d< t jj||d�|ƒ|d< |d  ¡  ¡ }|dkrF|d  |  < |V  qdS )zð Resample data.
        Inplace operation.

        Args:
            data: Iterable[{key, wav, label, sample_rate}]
            resample_rate: target resample rate

        Returns:
            Iterable[{key, wav, label, sample_rate}]
    r.   r-   ©Ú	orig_freqÚnew_freqr*   N)r5   Ú
transformsÚResampleÚabsÚmax)r   Úresample_rateZmin_sample_rater   r   r.   ÚwaveformÚmax_valr$   r$   r%   Úresamplem   s(   €ÿÿòrI   é `  c              	   c   sˆ   � | D ]>}|d }|j d |kr(t d|j d | ¡}|dd…||| …f }ntj|t d||j d  ¡gdd�}||d< |V  qdS )zÒ Truncate data.

        Args:
            data: Iterable[{key, wav, label, sample_rate}]
            truncate_length: truncate length

        Returns:
            Iterable[{key, wav, label, sample_rate}]
    r-   r*   r   N©r/   )ÚshapeÚrandomÚrandintÚtorchÚconcatÚzeros)r   Ztruncate_lengthr   r   rG   Ústartr$   r$   r%   Útruncate‰   s   €
$ørS   éÿÿÿÿc              	   c   s¾   � | D ]Y}d|v sJ ‚d|v sJ ‚d|v sJ ‚d|v sJ ‚|dkrIt t |d jd | ¡ƒ}tj|d t d|| |d jd  ¡gdd�|d< ||d ƒjdd� dd¡|d	< |V  qd
S )z˜ Extract fbank

        Args:
            data: Iterable[{key, wav, label, sample_rate}]

        Returns:
            Iterable[{key, feat, label}]
    r.   r-   Úuttr2   rT   r*   rK   r   Úspeech_featN)	ÚintÚnpÚceilrL   rO   rP   rQ   ÚsqueezeÚ	transpose)r   Úfeat_extractorr;   r   r   Úindexr$   r$   r%   Úcompute_fbankž   s   €4 ör^   c                 c   s‚   � | D ];}|dkr|d j d | dksJ dƒ‚tjj|d dd�|d ƒ|d	< tj|d	 d
d�jdd� dd¡|d< |V  qdS )z  Extract whisper fbank

        Args:
            data: Iterable[{key, wav, label, sample_rate}]

        Returns:
            Iterable[{key, feat, label}]
    rT   r-   r*   r   z.speech length is not aligned with speech_tokenr.   r>   r?   Ú
speech_16ké€   )Ún_melsrK   Úwhisper_featN)rL   r5   rB   rC   ÚwhisperÚlog_mel_spectrogramrZ   r[   )r   r;   r   r   r$   r$   r%   Úcompute_whisper_fbank·   s   €	 &ûre   c           
      c   s  � |d | }| D ]|}d|v sJ ‚d|v sJ ‚d|v sJ ‚d|v s#J ‚|d }t j|jdd� ¡  d¡||d	�\}}t|dkƒd
k rUt j|jdd� ¡  d¡||d	�\}}t  |jdd� ¡  d¡|||¡}	tj	t
 |	¡ ddd¡|d jd dd� d¡}	|	|d< |V  q	dS )z• Extract f0

        Args:
            data: Iterable[{key, wav, label, sample_rate}]

        Returns:
            Iterable[{key, feat, label}]
    iè  r.   r-   rU   r2   r   rK   Údouble)Úframe_periodé   r*   rT   rV   Úlinear)r8   r   Ú
pitch_featN)ÚpwZharvestrZ   ÚnumpyÚastypeÚsumZdioZ	stonemaskÚFÚinterpolaterO   Ú
from_numpyÚviewrL   )
r   r.   Úhop_sizer   rg   r   rG   Z_f0ÚtÚf0r$   r$   r%   Ú
compute_f0È   s    €	&&"0ôrv   c                 c   sÆ   � | D ]]}d|vr-d|vr-t jj|d dd�|d ƒ|d< t |d ¡}| |d< |d< ntj|d tjd�|d< tj|d tjd�|d< |r]tj	|d d	d
�|d< tj	|d d	d
�|d< |V  qdS )z¬ Parse utt_embedding/spk_embedding

        Args:
            data: Iterable[{key, wav, label, sample_rate}]

        Returns:
            Iterable[{key, feat, label}]
    Úutt_embeddingÚspk_embeddingr.   r>   r?   r-   r_   ©Údtyper   rK   N)
r5   rB   rC   r   Ú	inferencerO   ÚtensorÚfloat32ro   Ú	normalize)r   r~   r   r   Ú	embeddingr$   r$   r%   Úparse_embeddingá   s   €	 õr€   c                 c   s\   � |ƒ }| D ]%}d|v sJ ‚|j |d |d�|d< d|v r(|j |d |d�|d< |V  qdS )z× Decode text to chars or BPE
        Inplace operation

        Args:
            data: Iterable[{key, wav, txt, sample_rate}]

        Returns:
            Iterable[{key, wav, txt, tokens, label, sample_rate}]
    Útext)Úallowed_specialr2   ZinstructÚinstruct_tokenN)Úencode)r   Úget_tokenizerr‚   r   Ú	tokenizerr   r$   r$   r%   Útokenizeø   s   €
ûr‡   é'  c                 c   s~   � g }t |d ƒ}| D ]$}| |¡ t|ƒ|kr/t |¡ |d|… D ]}|V  q#||d… }qt |¡ |D ]}|V  q7dS )zÇ Local shuffle the data

        Args:
            data: Iterable[{key, feat, label}]
            shuffle_size: buffer size for shuffle

        Returns:
            Iterable[{key, feat, label}]
    é   N)rW   Úappendr   rM   Úshuffle)r   Zshuffle_sizer   ÚbufZ
yield_sizer   Úxr$   r$   r%   r‹     s   €


€
ÿr‹   éô  c                 c   sn   � g }| D ]}|  |¡ t|ƒ|kr$|jdd„ d� |D ]}|V  qg }q|jdd„ d� |D ]}|V  q/dS )a{   Sort the data by feature length.
        Sort is used after shuffle and before batch, so we can group
        utts with similar lengths into a batch, and `sort_size` should
        be less than `shuffle_size`

        Args:
            data: Iterable[{key, feat, label}]
            sort_size: buffer size for sort

        Returns:
            Iterable[{key, feat, label}]
    c                 S   ó   | d   d¡S ©NrV   r   ©r8   ©r�   r$   r$   r%   Ú<lambda>6  ó    zsort.<locals>.<lambda>)Úkeyc                 S   r�   r�   r‘   r’   r$   r$   r%   r“   ;  r”   N)rŠ   r   Úsort)r   Ú	sort_sizer   rŒ   r   r�   r$   r$   r%   r–   $  s   €
€ÿr–   é   c                 c   sJ   � g }| D ]}|  |¡ t|ƒ|kr|V  g }qt|ƒdkr#|V  dS dS )zÍ Static batch the data by `batch_size`

        Args:
            data: Iterable[{key, feat, label}]
            batch_size: batch size

        Returns:
            Iterable[List[{key, feat, label}]]
    r   N)rŠ   r   )r   r   rŒ   r   r$   r$   r%   Ústatic_batch@  s   €

€
ÿr™   éà.  c                 c   sš   � g }d}| D ]8}d|v sJ ‚t |d tjƒsJ ‚|d  d¡}t||ƒ}|t|ƒd  }||kr:|V  |g}|}q| |¡ qt|ƒdkrK|V  dS dS )a   Dynamic batch the data until the total frames in batch
        reach `max_frames_in_batch`

        Args:
            data: Iterable[{key, feat, label}]
            max_frames_in_batch: max_frames in one batch

        Returns:
            Iterable[List[{key, feat, label}]]
    r   rV   r*   N)Ú
isinstancerO   ÚTensorr8   rE   r   rŠ   )r   Úmax_frames_in_batchr   rŒ   Zlongest_framesr   Znew_sample_framesZframes_after_paddingr$   r$   r%   Údynamic_batchT  s"   €

ÿrž   Ústaticc                 C   s8   |dkr	t | |ƒS |dkrt| |ƒS t d |¡¡ dS )z& Wrapper for static/dynamic batch
    rŸ   ÚdynamiczUnsupported batch type {}N)r™   rž   r   Úfatalr   )r   Z
batch_typer   r�   r   r$   r$   r%   Úbatchq  s
   

r¢   Fc                 #   sX  � | D �]¥‰ t ˆ tƒsJ ‚tjtjdd„ ˆ D ƒtjd�dd�}i }‡ fdd„|D ƒ|d< ‡ fdd„|D ƒ|d	< ‡ fd
d„|D ƒ}tjdd„ |D ƒtjd�|d< t|ddd�|d< ‡ fdd„|D ƒ}tjdd„ |D ƒtjd�|d< t|ddd�|d< tj‡ fdd„|D ƒdd�|d< tj‡ fdd„|D ƒdd�|d< t ‡ fdd„|D ƒ¡ ¡ rÈ‡ fdd„|D ƒ}	tjdd„ |	D ƒtjd�|d< t|	ddd�|d< t ‡ fdd„|D ƒ¡ ¡ r÷‡ fdd„|D ƒ}
tjd d„ |
D ƒtjd�|d!< t|
ddd�|d"< t ‡ fd#d„|D ƒ¡ ¡ �r'‡ fd$d„|D ƒ}tjd%d„ |D ƒtjd�|d&< t|ddd�|d'< |du �rn‡ fd(d„|D ƒ}tjd)d„ |D ƒtjd�|d*< t|ddd�|d+< ‡ fd,d„|D ƒ}tjd-d„ |D ƒtjd�|d.< t|ddd�|d/< |du �r”‡ fd0d„|D ƒ}tjd1d„ |D ƒtjd�|d2< t|ddd�|d3< |du �r |d |d4< n|d |d4< |V  qd5S )6zÎ Padding the data into training data

        Args:
            data: Iterable[List[{key, feat, label}]]

        Returns:
            Iterable[Tuple(keys, feats, labels, feats lengths, label lengths)]
    c                 S   s   g | ]	}|d    d¡‘qS )r-   r*   r‘   )Ú.0r�   r$   r$   r%   Ú
<listcomp>‡  s    zpadding.<locals>.<listcomp>ry   T)Ú
descendingc                    ó   g | ]}ˆ | d  ‘qS )rU   r$   ©r£   r"   ©r   r$   r%   r¤   ‰  ó    Úuttsc                    r¦   )r�   r$   r§   r¨   r$   r%   r¤   Š  r©   r�   c                    ó   g | ]}t  ˆ | d  ¡‘qS )r2   ©rO   r|   r§   r¨   r$   r%   r¤   ‹  ó    c                 S   ó   g | ]}|  d ¡‘qS ©r   r‘   r§   r$   r$   r%   r¤   Œ  ó    Útext_token_lenr   )Úbatch_firstÚpadding_valuer2   c                    r¦   )rV   r$   r§   r¨   r$   r%   r¤   Ž  r©   c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   �  r°   Úspeech_feat_lenrV   c                    r¦   )rw   r$   r§   r¨   r$   r%   r¤   ‘  r©   rK   rw   c                    r¦   )rx   r$   r§   r¨   r$   r%   r¤   ’  r©   rx   c                    ó   g | ]}d ˆ | v ‘qS ©rƒ   r$   r§   r¨   r$   r%   r¤   “  r©   c                    r«   r¶   r¬   r§   r¨   r$   r%   r¤   ”  r­   c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   •  r°   Úinstruct_token_lenrƒ   c                    rµ   ©rb   r$   r§   r¨   r$   r%   r¤   —  r©   c                    r¦   r¸   r$   r§   r¨   r$   r%   r¤   ˜  r©   c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   ™  r°   Úwhisper_feat_lenrb   c                    rµ   ©r3   r$   r§   r¨   r$   r%   r¤   ›  r©   c                    r«   rº   r¬   r§   r¨   r$   r%   r¤   œ  r­   c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   �  r°   Úspeech_token_lenr3   c                    s    g | ]}ˆ | d  j dd�‘qS )r-   r   rK   )rZ   r§   r¨   r$   r%   r¤   ¡  s     c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   ¢  r°   Ú
speech_lenr-   c                    r¦   )rj   r$   r§   r¨   r$   r%   r¤   ¤  r©   c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   ¥  r°   Zpitch_feat_lenrj   c                    r«   )r4   r¬   r§   r¨   r$   r%   r¤   ¨  r­   c                 S   r®   r¯   r‘   r§   r$   r$   r%   r¤   ©  r°   Úreject_speech_token_lenr4   r   N)	r›   ÚlistrO   Úargsortr|   Úint32r   ÚstackÚall)r   Zuse_spk_embeddingr   ZganZdpoÚorderr¢   r2   rV   rƒ   rb   r3   r-   rj   r4   r$   r¨   r%   Úpadding|  sV   €
	$  


ÖrÄ   )r   )r'   r(   r)   r*   r+   r*   r   )r=   r>   r   )rJ   r   )rT   r   )rˆ   r   )rŽ   r   )r˜   )rš   r   )rŸ   r˜   rš   r   )r   FF)'r   rM   Úpyarrow.parquetÚparquetr   Úior   rl   rX   rc   rO   r5   Ztorch.nn.utils.rnnr   Ztorch.nn.functionalÚnnÚ
functionalro   Zpyworldrk   Úcosyvoice.utils.onnxr   r   ZAUDIO_FORMAT_SETSr&   r<   rI   rS   r^   re   rv   r€   r‡   r‹   r–   r™   rž   r¢   rÄ   r$   r$   r$   r%   Ú<module>   sJ   

ù
6

ý








