o
    ¤ôjö-  ã                	   @   sì  d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZ d dl	m
Z
mZmZmZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlm Z m!Z!m"Z" d dl#m$Z$m%Z% dZ&ervd dl'm(Z( G dd„ dƒZ)G dd„ dej*ƒZ+de$defdd„Z,dedefdd„Z-	d,dee.ef dee.eee%f f deee.  ddfdd„Z/dedee.ef fd d!„Z0d"ee.ef d#ee.ee.d$f f dee.ef fd%d&„Z1d'ee.ef d(ee.d$f d)eddfd*d+„Z2dS )-é    N)Úpartial)ÚBytesIO)ÚPath)ÚIOÚTYPE_CHECKINGÚAnyÚCallableÚDictÚOptionalÚOrderedDictÚSequenceÚSetÚTupleÚUnion)Úapply_to_collection)ÚTensor)Ú_TensorMeta)Ú	Parameter)Úoverride)Ú_TORCH_GREATER_EQUAL_2_0Ú_TORCH_GREATER_EQUAL_2_1Ú_TORCH_GREATER_EQUAL_2_2)Ú_PATHÚ	_Statefulzmeta.pt©ÚTypedStoragec                   @   s&  e Zd Zdedddededdf
dd	„Zedd
œdededede	de
d defdd„ƒZedd
œdededede
d deed f f
dd„ƒZe	d*dddededededede
e dddd fdd„ƒZdefdd„Ze	 	d+ded!edee d"e
e def
d#d$„ƒZd%edefd&d'„Zdefd(d)„ZdS ),Ú_NotYetLoadedTensorÚ
metatensorÚarchiveinfoÚ_LazyLoadingUnpicklerÚstorageinfoÚrebuild_argsÚreturnNc                 C   s   || _ || _|| _|| _d S ©N)r   r   r    r!   )Úselfr   r   r    r!   © r%   úr/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/utilities/load.pyÚ__init__,   s   
z_NotYetLoadedTensor.__init__©r   ÚfuncÚnew_typeÚargsÚstatec                   sJ   ||Ž }t |tƒr|j‰dtf‡ ‡‡fdd„}||_|S tj |ˆ |ˆ¡S )Nr"   c                      s    ˆƒ ‰ t j ‡ fdd„ˆdˆ¡S )Nc                      s   ˆ S r#   r%   r%   ©Útr%   r&   Ú<lambda>H   s    zP_NotYetLoadedTensor.rebuild_from_type_v2.<locals>._load_tensor.<locals>.<lambda>r%   )ÚtorchÚ_tensorÚ_rebuild_from_type_v2r%   ©r*   Úold_ltr,   r-   r&   Ú_load_tensorF   s   z>_NotYetLoadedTensor.rebuild_from_type_v2.<locals>._load_tensor)Ú
isinstancer   r5   r   r0   r1   r2   )Úclsr)   r*   r+   r,   r   Úretr5   r%   r3   r&   Úrebuild_from_type_v28   s   

z(_NotYetLoadedTensor.rebuild_from_type_v2ÚdataÚrequires_gradÚbackward_hooksc                   s@   t |tƒr|j‰dtf‡ ‡‡fdd„}||_|S tj |ˆˆ ¡S )Nr"   c                     s   ˆƒ } t j | ˆˆ ¡S r#   )r0   Ú_utilsÚ_rebuild_parameterr-   ©r<   r4   r;   r%   r&   r5   Z   s   z;_NotYetLoadedTensor.rebuild_parameter.<locals>._load_tensor)r6   r   r5   r   r0   r=   r>   )r7   r:   r;   r<   r   r5   r%   r?   r&   Úrebuild_parameterN   s   
	z%_NotYetLoadedTensor.rebuild_parameterÚstorager   Ústorage_offsetÚsizeÚstrideÚmetadatac             	   C   s<   ||||||f}	t j |||||||¡}
|j}t|
|||	ƒS r#   )r0   r=   Ú_rebuild_tensor_v2r   r   )r7   rA   rB   rC   rD   r;   r<   rE   r   r!   r   r    r%   r%   r&   Úrebuild_tensor_v2b   s   ÿz%_NotYetLoadedTensor.rebuild_tensor_v2c           	      C   s¬   ddl m}m} | j\}}}}}| jj}| jj d|› �|t	j
 |¡ |¡}| ¡ j}t ¡ � t d¡ |||dd�}W d   ƒ n1 sFw   Y  t	j
j|g| j¢R Ž S )Nr   )r   ÚUntypedStoragezdata/ÚignoreT)Úwrap_storageÚdtypeÚ	_internal)Útorch.storager   rH   r    r   rK   r   Úfile_readerÚget_storage_from_recordr0   r=   Ú_element_sizeÚ_typed_storageÚ_untyped_storageÚwarningsÚcatch_warningsÚsimplefilterrF   r!   )	r$   r   rH   Ú_ÚfnrC   rK   rA   Zutsr%   r%   r&   r5   v   s   ÿ


ýz _NotYetLoadedTensor._load_tensorr%   ÚtypesÚkwargsc                 C   s$   |pi }dd„ |D ƒ}||i |¤ŽS )Nc                 S   s"   g | ]}t |tƒr| ¡ n|‘qS r%   )r6   r   r5   )Ú.0Úargr%   r%   r&   Ú
<listcomp>�   s   " z:_NotYetLoadedTensor.__torch_function__.<locals>.<listcomp>r%   )r7   r)   rX   r+   rY   Zloaded_argsr%   r%   r&   Ú__torch_function__‡   s   z&_NotYetLoadedTensor.__torch_function__Únamec                 C   sF   |dv r
t | j|ƒS |dv rt |  ¡ |ƒS tdt| ƒj› d|› d�ƒ‚)N>   rK   ÚlayoutÚgradÚndimÚgrad_fnÚshapeÚis_metaÚ	output_nrrC   Úretains_gradÚvolatiler;   Únames>   ÚcudaÚ
contiguousÚhalfú'z' object has no attribute ')Úgetattrr   r5   ÚAttributeErrorÚtypeÚ__name__)r$   r^   r%   r%   r&   Ú__getattr__“   s
   z_NotYetLoadedTensor.__getattr__c                 C   s   | j j› dt| jƒ› d�S )Nú(ú))Ú	__class__rp   Úreprr   )r$   r%   r%   r&   Ú__repr__¬   s   z_NotYetLoadedTensor.__repr__r#   )r%   N)rp   Ú
__module__Ú__qualname__r   Útupler'   Úclassmethodr   r   Údictr
   r   r9   Úboolr   r   r@   ÚintrG   r5   r   r	   r]   Ústrrq   rv   r%   r%   r%   r&   r   +   sœ    þýüû
úùþýüûùøúþýüú
ù	øþýüûúùø
öõûþýüûúr   c                       s`   e Zd Zdedejddf‡ fdd„Zededede	f‡ fd	d
„ƒZ
ededdfdd„ƒZ‡  ZS )r   ÚfilerN   r"   Nc                    s   t ƒ  |¡ || _d S r#   )Úsuperr'   rN   )r$   r   rN   ©rt   r%   r&   r'   ²   s   
z_LazyLoadingUnpickler.__init__Úmoduler^   c                    sh   |dkr|dkrt tj| d�S |dkr|dkrt tj| d�S |dkr-|dkr-t tj| d�S tƒ  ||¡S )Nztorch._utilsrF   r(   ztorch._tensorr2   r>   )r   r   rG   r9   r@   r€   Ú
find_class)r$   r‚   r^   r�   r%   r&   rƒ   ¶   s   z _LazyLoadingUnpickler.find_classÚpidr   c                 C   sf   ddl m} |\}}}}}t ¡ � t d¡ ||ƒ jdd�}W d   ƒ n1 s)w   Y  ||_|S )Nr   r   rI   Úmeta)rK   Údevice)rM   r   rS   rT   rU   rK   r   )r$   r„   r   rV   r7   rA   r%   r%   r&   Úpersistent_loadÀ   s   

ýz%_LazyLoadingUnpickler.persistent_load)rp   rw   rx   r   r0   ÚPyTorchFileReaderr'   r   r~   r   rƒ   ry   r‡   Ú__classcell__r%   r%   r�   r&   r   ±   s    	r   Úfilenamer"   c                 C   s~   t stdƒ‚tj | ¡stdt| ƒ›d�ƒ‚t t| ƒ¡}t	| 
d¡ƒ�}t||ƒ}| ¡ W  d   ƒ S 1 s8w   Y  d S )Nz3Lazy-loading is only supported with PyTorch >= 2.0.zPath z! does not exist or is not a file.zdata.pkl)r   ÚNotImplementedErrorÚosÚpathÚisfileÚFileNotFoundErrorr~   r0   rˆ   r   Ú
get_recordr   Úload)rŠ   rN   ÚpklZmupr%   r%   r&   Ú
_lazy_loadÍ   s   
$þr“   Ú
collectionc                 C   s    dt dtfdd„}t| t |d�S )Nr.   r"   c                 S   s   |   ¡ S r#   )r5   r-   r%   r%   r&   r5   Ù   s   z*_materialize_tensors.<locals>._load_tensor)rK   Úfunction)r   r   r   )r”   r5   r%   r%   r&   Ú_materialize_tensorsØ   s   r–   ÚsourceÚdestinationÚkeysc                 C   sb   |du rt | ƒn|t | ƒ@ }|D ]}|  |¡}||v r*t|| tƒr*||  |¡ q|||< qdS )zÖTakes the state from the source destination and moves it into the destination dictionary.

    If an object in the destination follows the stateful protocol, it loads the source state via ``load_state_dict``.

    N)ÚsetÚpopr6   r   Úload_state_dict)r—   r˜   r™   Úkeyr,   r%   r%   r&   Ú_move_state_intoß   s   

ûrž   Úcheckpoint_folderc              
   C   s  t stdƒ‚ddlm} ddlm}m} trddlm} nddlm	} || ƒ}| 
¡ }i }|j ¡ D ]/\}}	t|	|ƒr@d||< q2t|	|ƒratj|	j|	jjt d¡|	jj|	jj|	jj|	jjd	�||< q2|||d
d� t||jd�}| t }
|
 ¡ rtj|
dd�ni }| |¡ |S )z½Loads a sharded checkpoint saved with the `torch.distributed.checkpoint` into a full state dict.

    The current implementation assumes that the entire checkpoint fits in CPU memory.

    z;Processing distributed checkpoints requires PyTorch >= 2.1.r   )ÚFileSystemReader)ÚBytesStorageMetadataÚTensorStorageMetadata)r‘   )rœ   z
<bytes_io>Úcpu)rC   rK   r†   Úmemory_formatr_   r;   Ú
pin_memoryT)Ú
state_dictZstorage_readerZno_dist)Úkey_map)Úmap_location)r   ÚImportErrorZtorch.distributed.checkpointr    Z%torch.distributed.checkpoint.metadatar¡   r¢   r   r‘   rœ   Úread_metadataZstate_dict_metadataÚitemsr6   r0   ÚemptyrC   Ú
propertiesrK   r†   r¤   r_   r;   r¥   Ú_unflatten_dictZplanner_dataÚ_METADATA_FILENAMEÚis_fileÚupdate)rŸ   r    r¡   r¢   r‘   ÚreaderrE   Ú
checkpointÚtensor_nameZsd_metadataZ
extra_fileÚextrar%   r%   r&   Ú_load_distributed_checkpointð   s<   



ù€

r¶   r³   r§   .c                 C   s>   |   ¡ |  ¡ ks
J ‚i }| D ]}|| }t||| | ƒ q|S )ak  Converts the flat dictionary with keys 'x.y.z...' to a nested dictionary using the provided key map.

    Args:
        checkpoint: The flat checkpoint dictionary.
        key_map: A dictionary that maps the keys in flattened format 'x.y.z...' to a tuple representing
            the index path into the nested dictonary that this function should construct.

    )r™   Ú_set_nested_dict_value)r³   r§   Ú	convertedZflat_keyÚkey_pathr%   r%   r&   r®     s   	r®   Únested_dictr¹   Úvaluec                 C   s>   | }|d d… D ]}||vri ||< || }q|||d < d S )Néÿÿÿÿr%   )rº   r¹   r»   Úresultr�   r%   r%   r&   r·   0  s   
r·   r#   )3rŒ   ÚpicklerS   Ú	functoolsr   Úior   Úpathlibr   Útypingr   r   r   r   r	   r
   r   r   r   r   r   r0   Ú#lightning_utilities.core.apply_funcr   r   Útorch._Cr   Útorch.nnr   Útyping_extensionsr   Ú"lightning.fabric.utilities.importsr   r   r   Ú lightning.fabric.utilities.typesr   r   r¯   rM   r   r   Ú	Unpicklerr   r“   r–   r~   rž   r¶   r®   r·   r%   r%   r%   r&   Ú<module>   sH   4 ÿ
ÿÿ
ÿ
þ6/.