o
    qôj9  ã                   @   s°  d dl Z d dlmZ d dlmZ d dlZd dlZd dlm	Z	 ddl
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ eddƒrXd d	l m!Z! nd d	l"m!Z! eƒ rjd dl#m$  m%Z& dd
l'm(Z( ddl)m*Z* e(e+ƒZ,			d&de-de.e/ de.de.de.de0de0dee! de1de1defdd„Z2			d'de-de.de.de.de.de0dee! dee/ de/fdd„Z3d(d e-d!e0de1ddfd"d#„Z4d)d e-d!e0ddfd$d%„Z5dS )*é    N)ÚPath)ÚOptional)Ú
load_modelé   )Ú
MODEL_NAMEÚOPTIMIZER_NAMEÚRNG_STATE_NAMEÚSAFE_MODEL_NAMEÚSAFE_WEIGHTS_NAMEÚSAMPLER_NAMEÚSCALER_NAMEÚSCHEDULER_NAMEÚWEIGHTS_NAMEÚget_pretty_nameÚis_cuda_availableÚis_hpu_availableÚis_mlu_availableÚis_musa_availableÚis_neuron_availableÚis_sdaa_availableÚis_torch_versionÚis_torch_xla_availableÚis_xpu_availableÚloadÚsavez>=z2.4.0)Ú
GradScaler)Ú
get_logger)ÚPartialStateFTÚ
output_dirÚmodel_statesÚ
optimizersÚ
schedulersÚdataloadersÚprocess_indexÚstepÚscalerÚsave_on_each_nodeÚsafe_serializationÚreturnc
           !      C   sb  t | ƒ} t|ƒD ]-\}
}|	stnt}|
dkr | dd|
› d�¡}|  |¡}t||||	d� t d|› �¡ qt|ƒD ].\}
}| 	¡ }|
dkrKt
› d�nt
› d|
› d�}|  |¡}t|||dd� t d|› �¡ q:t|ƒD ].\}
}| 	¡ }|
dkr~t› d�nt› d|
› d�}|  |¡}t|||dd� t d	|› �¡ qmt|ƒD ]e\}
}|
dkr­t› d�nt› d|
› d�}|  |¡}d
dlm}m} t|j|ƒrÙ| ¡ }t||ƒrÙt|||dd� t|ddƒrú|
dkrådnd|
› d�}|  |¡}| 	¡ }t ||¡ t d|
› d|› �¡ q |du�r"| 	¡ }|  t¡}t ||¡ t d|› �¡ i }t› d|› d�}||d< t ¡ |d< tj ¡ |d< t ¡ |d< tƒ �rNtj ¡ |d< tƒ �rZtj  ¡ |d< nt!ƒ �rftj" ¡ |d< nt#ƒ �rqtj$ ¡ |d< t%ƒ �r|tj& ¡ |d< t'ƒ �r‡tj( ¡ |d< t)ƒ �r’tj* ¡ |d< t+ƒ �rœt, ¡ |d< |  |¡} t || ¡ t d | › �¡ | S )!aw  
    Saves the current states of the models, optimizers, scaler, and RNG generators to a given directory.

    <Tip>

    If `safe_serialization` is `True`, models will be saved with `safetensors` while the rest are saved using native
    `pickle`.

    </Tip>

    Args:
        output_dir (`str` or `os.PathLike`):
            The name of the folder to save all relevant weights and states.
        model_states (`List[torch.nn.Module]`):
            A list of model states
        optimizers (`List[torch.optim.Optimizer]`):
            A list of optimizer instances
        schedulers (`List[torch.optim.lr_scheduler._LRScheduler]`):
            A list of learning rate schedulers
        dataloaders (`List[torch.utils.data.DataLoader]`):
            A list of dataloader instances to save their sampler states
        process_index (`int`):
            The current process index in the Accelerator state
        step (`int`):
            The current step in the internal step tracker
        scaler (`torch.amp.GradScaler`, *optional*):
            An optional gradient scaler instance to save;
        save_on_each_node (`bool`, *optional*):
            Whether to save on every node, or only the main node.
        safe_serialization (`bool`, *optional*, defaults to `True`):
            Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).
    r   Ú.Ú_)r&   r'   zModel weights saved in ú.binFzOptimizer state saved in zScheduler state saved in r   ©ÚIterableDatasetShardÚSeedableRandomSamplerÚuse_stateful_dataloaderúdl_state_dict.binÚdl_state_dict_zSampler state for dataloader z
 saved in NzGradient scaler state saved in ú.pklr$   Úrandom_stateÚnumpy_random_seedÚtorch_manual_seedÚtorch_xpu_manual_seedÚtorch_mlu_manual_seedÚtorch_sdaa_manual_seedÚtorch_musa_manual_seedÚtorch_hpu_manual_seedÚtorch_neuron_manual_seedÚtorch_cuda_manual_seedÚxm_seedzRandom states saved in )-r   Ú	enumerater   r
   ÚreplaceÚjoinpathr   ÚloggerÚinfoÚ
state_dictr   r   r   Údata_loaderr-   r.   Ú
isinstanceÚdatasetÚget_samplerÚgetattrÚtorchr   r   ÚrandomÚgetstateÚnpÚ	get_stateÚget_rng_stater   ÚxpuÚget_rng_state_allr   Úmlur   Úsdaar   Úmusar   Úhpur   Úneuronr   Úcudar   Úxm)!r   r   r    r!   r"   r#   r$   r%   r&   r'   ÚiÚstateÚweights_nameÚoutput_model_fileÚoptÚoptimizer_nameÚoutput_optimizer_fileÚ	schedulerÚscheduler_nameZoutput_scheduler_fileÚ
dataloaderÚsampler_nameZoutput_sampler_filer-   r.   ÚsamplerÚdataloader_state_dict_nameZ!output_dataloader_state_dict_filerC   Zoutput_scaler_fileÚstatesZstates_nameZoutput_states_file© rf   úi/var/www/html/academy/tools/voices/clone/sv-venv/lib/python3.10/site-packages/accelerate/checkpointing.pyÚsave_accelerator_state?   s‚   ,
"
"
"





rh   Ú	input_dirÚmodelsÚload_kwargsc	           #      K   s  t ƒ }
|dvrtdƒ‚|du rd}n|dkrtƒ j}|du r i }t| ƒ} t|ƒD ]D\}}|dkr5d|› �nd}|  t› |› d	�¡}| ¡ rSt	||fd
t
|ƒi|	¤Ž q(|  t› |› d�¡}t||d�}|j|fi |	¤Ž q(t d¡ t|ƒD ]+\}}|dkrƒt› d�nt› d|› d�}|  |¡}t|fd|i|¤Ž}||  |¡ qvt d¡ t|ƒD ]'\}}|dkr¸t› d�nt› d|› d�}|  |¡}t|fi |¤Ž}| |¡ q«t d¡ t|ƒD ]e\}}|dkrét› d�nt› d|› d�}|  |¡}ddlm}m} t|j|ƒ�r| ¡ }t||ƒ�r| t|ƒ¡}t|ddƒ�rA|dk�r$dnd|› d�}|  |¡}| ¡ �rAt|fi |¤Ž}| |¡ qÜt d¡ |du�r_|  t¡} t| ƒ}!| |!¡ t d¡ z—t|  t› d|› d�¡ƒ}"d|"v �rx|"d |
d< t |"d ¡ tj  |"d ¡ t! "|"d ¡ t#ƒ �ršt!j$ %|"d ¡ t&ƒ �r§t!j' %|"d ¡ nt(ƒ �r´t!j) %|"d  ¡ nt*ƒ �rÀt!j+ %|"d! ¡ t,ƒ �rÌt!j- %|"d" ¡ t.ƒ �rØt!j/ %|"d# ¡ t0ƒ �rät!j1 %|"d$ ¡ t2ƒ �rït3 "|"d% ¡ t d&¡ W |
S  t4�y   t d'¡ Y |
S w )(aO  
    Loads states of the models, optimizers, scaler, and RNG generators from a given directory.

    Args:
        input_dir (`str` or `os.PathLike`):
            The name of the folder to load all relevant weights and states.
        models (`List[torch.nn.Module]`):
            A list of model instances
        optimizers (`List[torch.optim.Optimizer]`):
            A list of optimizer instances
        schedulers (`List[torch.optim.lr_scheduler._LRScheduler]`):
            A list of learning rate schedulers
        dataloaders (`List[torch.utils.data.DataLoader]`):
            A list of dataloader instances used in your program
        process_index (`int`):
            The current process index in the Accelerator state
        scaler (`torch.amp.GradScaler`, *optional*):
            An optional *GradScaler* instance to load
        map_location (`str`, *optional*):
            What device to load the optimizer state onto. Should be one of either "cpu" or "on_device".
        load_kwargs (`dict`, *optional*):
            Additional arguments that can be passed to the `load` function.
        load_model_func_kwargs (`dict`, *optional*):
            Additional arguments that can be passed to the model's `load_state_dict` method.

    Returns:
        `dict`: Contains the `Accelerator` attributes to override while loading the state.
    )NÚcpuÚ	on_devicezaUnsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`Nrl   rm   r   r*   Ú z.safetensorsÚdevicer+   )Úmap_locationz%All model weights loaded successfullyrp   z(All optimizer states loaded successfullyz(All scheduler states loaded successfullyr   r,   r/   Fr0   r1   z1All dataloader sampler states loaded successfullyz$GradScaler state loaded successfullyr2   r$   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   z%All random states loaded successfullyzCould not load random states)5ÚdictÚ	TypeErrorr   ro   r   r>   r@   r	   Úexistsr   Ústrr   r   Úload_state_dictrA   rB   r   r   r   rD   r-   r.   rE   rF   rG   Zset_samplerrH   r   r   rJ   ÚsetstaterL   Ú	set_staterI   Úset_rng_stater   rO   Úset_rng_state_allr   rQ   r   rR   r   rS   r   rT   r   rU   r   rV   r   rW   Ú	Exception)#ri   rj   r    r!   r"   r#   r%   rp   rk   Úload_model_func_kwargsÚoverride_attributesrX   ÚmodelZendingÚinput_model_filerC   r\   r]   Úinput_optimizer_fileÚoptimizer_stater_   r`   Zinput_scheduler_fileZscheduler_statera   rb   Zinput_sampler_filer-   r.   rc   rd   Z input_dataloader_state_dict_fileÚinput_scaler_fileÚscaler_statere   rf   rf   rg   Úload_accelerator_state·   s¦   )ÿ
"

"

"



€





ýýrƒ   ÚpathÚindexc                 C   sD   t |ƒd|› d� }t dt| ƒ› d|› �¡ t|  ¡ ||d� dS )zL
    Saves the state of `obj` to `{path}/custom_checkpoint_{index}.pkl`
    Zcustom_checkpoint_r2   zSaving the state of z to )r&   N)r   rA   rB   r   r   rC   )Úobjr„   r…   r&   Úsave_locationrf   rf   rg   Úsave_custom_stateC  s   rˆ   c                 C   sB   |› d|› d�}t  dt| ƒ› d|› �¡ |  t|ddd�¡ dS )	zŽ
    Loads the state of `obj` at `{path}/custom_checkpoint_{index}.pkl`. Will always set `weights_only=False` when
    loading the state.
    z/custom_checkpoint_r2   zLoading the state of z from rl   F)rp   Úweights_onlyN)rA   rB   r   ru   r   )r†   r„   r…   Zload_locationrf   rf   rg   Úload_custom_stateM  s   rŠ   )NFT)NNN)r   F)r   )6rJ   Úpathlibr   Útypingr   ÚnumpyrL   rI   Úsafetensors.torchr   Úutilsr   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Ú	torch.ampr   Ztorch.cuda.ampÚtorch_xla.core.xla_modelÚcoreÚ	xla_modelrW   Úloggingr   rY   r   Ú__name__rA   rt   Úlistrq   ÚintÚboolrh   rƒ   rˆ   rŠ   rf   rf   rf   rg   Ú<module>   s„   \
öÿþýüûúùø	÷
ö
õ÷ÿþýüûúù	÷
õ 
