o
    Ÿôj�>  ã                	   @   s  d dl Z d dlmZ d dlmZ d dlmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dl m!Z! d dl"m#Z# d dl$m%Z% d dl&m'Z' d dl(m)Z) d dl*m+Z+ d dl,m-Z- erŽd dl.m/Z/ ede	e0ef d�Z1dZ2G dd„ dƒZ3G dd„ de)ƒZ4G dd„ dƒZ5dedefdd „Z6d!eed"f deeeej7f ee	e0ef  f fd#d$„Z8d%ej7d&e	e0ef dd"fd'd(„Z9d!e:de;fd)d*„Z<d+edefd,d-„Z=e+�re=ej>ƒe_>dS dS ).é    N)Údeepcopy)Úwraps)ÚTYPE_CHECKINGÚAnyÚCallableÚDictÚ	GeneratorÚIteratorÚListÚMappingÚOptionalÚTupleÚTypeVarÚUnionÚoverload)Úapply_to_collection)ÚTensor)Únn)Ú_IncompatibleKeys)Ú	Optimizer)Ú
DataLoader)Úoverride)Ú	Precision)ÚStrategy)Úmove_data_to_device)Ú_set_sampler_epoch)Ú_DeviceDtypeModuleMixin)Ú_TORCH_GREATER_EQUAL_2_0)ÚOptimizable©ÚOptimizedModuleÚT_destination)Úbound)Ztraining_stepZvalidation_stepZ	test_stepZpredict_stepc                	   @   sœ   e Zd Zddededeee  ddfdd„Ze	defdd	„ƒZ
deeef fd
d„Zdeeef ddfdd„Zddee defdd„Zdedefdd„ZdS )Ú_FabricOptimizerNÚ	optimizerÚstrategyÚ	callbacksÚreturnc                 C   s8   || _ || _|p	g | _td|jj | j|jfi ƒ| _dS )aˆ  FabricOptimizer is a thin wrapper around the :class:`~torch.optim.Optimizer` that delegates the optimizer
        step calls to the strategy.

        The underlying wrapped optimizer object can be accessed via the property :attr:`optimizer`.

        Args:
            optimizer: The optimizer to wrap
            strategy: Reference to the strategy for handling the optimizer step

        ÚFabricN)Ú
_optimizerÚ	_strategyÚ
_callbacksÚtypeÚ	__class__Ú__name__)Úselfr$   r%   r&   © r0   úl/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/wrappers.pyÚ__init__:   s   
"z_FabricOptimizer.__init__c                 C   ó   | j S ©N)r)   ©r/   r0   r0   r1   r$   K   ó   z_FabricOptimizer.optimizerc                 C   s   | j  | j¡S r4   )r*   Úget_optimizer_stater$   r5   r0   r0   r1   Ú
state_dictO   s   z_FabricOptimizer.state_dictr8   c                 C   s   | j  |¡ d S r4   )r$   Úload_state_dict)r/   r8   r0   r0   r1   r9   R   s   z _FabricOptimizer.load_state_dictÚclosurec                 C   s„   |d urd|ini }t | jdƒrt| jjtƒr| jj}n| j}| jj|fi |¤Ž}| jD ]}t|dd ƒ}t	|ƒr?|| j|d� q,|S )Nr:   ÚmodelZon_after_optimizer_step)r%   r$   )
Úhasattrr*   Ú
isinstancer;   r   r$   Úoptimizer_stepr+   ÚgetattrÚcallable)r/   r:   Úkwargsr$   ÚoutputÚcallbackÚhookr0   r0   r1   ÚstepU   s   
ÿþ
€z_FabricOptimizer.stepÚitemc                 C   s   t | j|ƒS r4   )r?   r)   ©r/   rF   r0   r0   r1   Ú__getattr__f   s   z_FabricOptimizer.__getattr__r4   )r.   Ú
__module__Ú__qualname__r   r   r   r
   r   r2   Úpropertyr$   r   Ústrr   r8   r9   r   rE   rH   r0   r0   r0   r1   r#   9   s    $r#   c                       st  e Zd Z	d+dejdedeej ddf‡ fdd„Zedejfdd	„ƒZ	e
d
ededefdd„ƒZedddœdedededefdd„ƒZedddœdededeeef fdd„ƒZe
	d,dee dededeeeef  fdd„ƒZe
	d-deeef dededefdd„ƒZdedefdd „Zd!ed"edefd#d$„Ze
d%edef‡ fd&d'„ƒZe
d"ed(eddf‡ fd)d*„ƒZ‡  ZS ).Ú_FabricModuleNÚforward_moduleÚ	precisionÚoriginal_moduler'   c                    s*   t ƒ  ¡  || _|p|| _|| _d| _dS )a¬  The FabricModule is a thin wrapper around the :class:`torch.nn.Module` and handles precision / autocast
        automatically for the forward pass.

        The underlying wrapped module can be accessed via the property :attr:`module`.

        Args:
            forward_module: The module to wrap the ``forward`` method on.
            precision: Reference to the precision plugin for handling precision context
            original_module: The original, unmodified module as passed into the
                :meth:`lightning.fabric.fabric.Fabric.setup` method. This is needed when attribute lookup
                on this wrapper should pass through to the original module.

        TN)Úsuperr2   Ú_forward_moduleÚ_original_moduleÚ
_precisionÚ_fabric_module_initialized)r/   rN   rO   rP   ©r-   r0   r1   r2   k   s
   


z_FabricModule.__init__c                 C   s   | j p| jS r4   )rS   rR   r5   r0   r0   r1   Úmodule�   s   z_FabricModule.moduleÚargsrA   c                 O   s^   | j  ||f¡\}}| j  ¡ � | j|i |¤Ž}W d  ƒ n1 s"w   Y  | j  |¡}|S )ziCasts all inputs to the right precision and handles autocast for operations in the module forward method.N)rT   Úconvert_inputÚforward_contextrR   Úconvert_output)r/   rX   rA   rB   r0   r0   r1   Úforward…   s   ÿz_FabricModule.forward.)ÚprefixÚ	keep_varsÚdestinationr]   r^   c                C   ó   d S r4   r0   ©r/   r_   r]   r^   r0   r0   r1   r8   �   ó   z_FabricModule.state_dictc                C   r`   r4   r0   )r/   r]   r^   r0   r0   r1   r8   “   rb   Ú Fc                 C   s   | j j|||d�S )N)r_   r]   r^   )rS   r8   ra   r0   r0   r1   r8   –   s
   ýTr8   Ústrictc                 K   s   | j jd||dœ|¤ŽS )N)r8   rd   r0   )rS   r9   )r/   r8   rd   rA   r0   r0   r1   r9       s   z_FabricModule.load_state_dictÚmethod_namec                    sR   ˆ dksJ ‚ˆj j‰dtdtdtf‡ ‡‡fdd„‰dtdtdtf‡‡fdd„}|S )	Nr\   rX   rA   r'   c                     s"   ˆˆj _tˆj ˆ ƒ}|| i |¤ŽS r4   )rS   r\   r?   )rX   rA   Úmethod)re   Úoriginal_forwardr/   r0   r1   Úwrapped_forwardª   s   zC_FabricModule._redirection_through_forward.<locals>.wrapped_forwardc                     s   ˆˆ j _ˆ j| i |¤ŽS r4   )rS   r\   )rX   rA   )r/   rh   r0   r1   Úcall_forward_moduleµ   s   zG_FabricModule._redirection_through_forward.<locals>.call_forward_module)rS   r\   r   )r/   re   ri   r0   )re   rg   r/   rh   r1   Ú_redirection_through_forward¦   s
   z*_FabricModule._redirection_through_forwardrf   Únamec                    sL   d‰dt dt ddf‡fdd„‰ tˆƒdt d	t dt f‡ ‡‡‡‡fd
d„ƒ}|S )zŸTracks whether any submodule in ``self._original_module`` was called during the execution of ``method`` by
        registering forward hooks on all submodules.FÚ_Ú__r'   Nc                     s   d‰ d S )NTr0   )rl   rm   )Úmodule_calledr0   r1   rD   Á   s   zA_FabricModule._wrap_method_with_module_call_tracker.<locals>.hookrX   rA   c                     sj   g }ˆj  ¡ D ]
}| | ˆ ¡¡ qˆ| i |¤Ž}ˆr*tdtˆj ƒj› dˆ› d�ƒ‚|D ]}| ¡  q,|S )NzYou are calling the method `Ú.z®()` from outside the model. This will bypass the wrapper from the strategy and result in incorrect behavior in `.backward()`. You should pass your inputs through `forward()`.)rS   ÚmodulesÚappendÚregister_forward_hookÚRuntimeErrorr,   r.   Úremove)rX   rA   ÚhandlesrW   rB   Úhandle©rD   rf   rn   rk   r/   r0   r1   Ú_wrapped_methodÅ   s   ÿ
zL_FabricModule._wrap_method_with_module_call_tracker.<locals>._wrapped_method)r   r   )r/   rf   rk   rx   r0   rw   r1   Ú%_wrap_method_with_module_call_tracker¼   s
   $z3_FabricModule._wrap_method_with_module_call_trackerrF   c                    s|   |t v r| j| jkr|  |¡S ztƒ  |¡W S  ty=   tƒ  d¡}t||ƒ}t 	|¡r9| j| jkr9|  
||¡}| Y S w )NrS   )Ú_LIGHTNING_MODULE_STEP_METHODSrR   rS   rj   rQ   rH   ÚAttributeErrorr?   ÚinspectÚismethodry   )r/   rF   rP   ÚattrrV   r0   r1   rH   Ù   s   

ùz_FabricModule.__getattr__Úvaluec                    s|   t | ddƒstƒ  ||¡ d S | j}t||ƒ}|t| ƒv }|s)|s)t|||ƒ d S |r1t|||ƒ |r<tƒ  ||¡ d S d S )NrU   F)r?   rQ   Ú__setattr__rS   r<   ÚdirÚsetattr)r/   rk   r   rP   Zoriginal_has_attrZfabric_has_attrrV   r0   r1   r€   ì   s   
ÿz_FabricModule.__setattr__r4   )Nrc   F)T)r.   rI   rJ   r   ÚModuler   r   r2   rK   rW   r   r   r\   r   r!   rL   Úboolr8   r   r   r   r9   r   rj   ry   rH   r€   Ú__classcell__r0   r0   rV   r1   rM   j   s^    ÿÿÿÿþ
$(ÿÿÿÿþ	ÿ
ÿÿÿþ$rM   c                   @   st   e Zd Zddedeej ddfdd„Zedeej fdd„ƒZde	fd	d
„Z
deee eeddf f fdd„ZdS )Ú_FabricDataLoaderNÚ
dataloaderÚdevicer'   c                 C   s$   | j  |j ¡ || _|| _d| _dS )a¾  The FabricDataLoader is a wrapper for the :class:`~torch.utils.data.DataLoader`. It moves the data to the
        device automatically if the device is specified.

        Args:
            dataloader: The dataloader to wrap
            device: The device to which the data should be moved. By default the device is `None` and no data
                transfers will be made (identical behavior as :class:`~torch.utils.data.DataLoader`).

        r   N)Ú__dict__ÚupdateÚ_dataloaderÚ_deviceÚ_num_iter_calls)r/   r‡   rˆ   r0   r0   r1   r2     s   

z_FabricDataLoader.__init__c                 C   r3   r4   )rŒ   r5   r0   r0   r1   rˆ     r6   z_FabricDataLoader.devicec                 C   s
   t | jƒS r4   )Úlenr‹   r5   r0   r0   r1   Ú__len__  s   
z_FabricDataLoader.__len__c                 c   sZ   � t | j| jƒ |  jd7  _| jd u rt| jƒE d H  d S | jD ]	}t|| jƒV  q!d S )Né   )r   r‹   r�   rŒ   Úiterr   rG   r0   r0   r1   Ú__iter__  s   €

ÿz_FabricDataLoader.__iter__r4   )r.   rI   rJ   r   r   Útorchrˆ   r2   rK   Úintr�   r   r	   r   r   r’   r0   r0   r0   r1   r†     s    (r†   Ú
collectionr'   c                 C   s^   dt tttf dt tjttf fdd„}tttg}tr&ddl	m
} | |¡ t| t|ƒ|d�S )NÚobjr'   c                 S   sH   t t| ƒd  }tƒrt|jƒd S t | tƒr| jS t | tƒr"| jS | S ©Nr   )r=   Ú_unwrap_compiledrM   rR   r#   r$   r†   r‹   )r–   Ú	unwrappedr0   r0   r1   Ú_unwrap-  s   

z _unwrap_objects.<locals>._unwrapr   r   )ÚdtypeÚfunction)r   rM   r#   r†   r   rƒ   r   r   r   Útorch._dynamor    rq   r   Útuple)r•   rš   Útypesr    r0   r0   r1   Ú_unwrap_objects,  s   ÿ
þ

r    r–   r    c                 C   sP   t s| dfS ddlm} t| |ƒr$t| ddƒ }du rtdƒ‚| j|fS | dfS )z¯Removes the :class:`torch._dynamo.OptimizedModule` around the object if it is wrapped.

    Use this function before instance checks against e.g. :class:`_FabricModule`.

    Nr   r   Ú_compile_kwargsz…Failed to determine the arguments that were used to compile the module. Make sure to import lightning before `torch.compile` is used.)r   r�   r    r=   r?   rs   Ú	_orig_mod)r–   r    Úcompile_kwargsr0   r0   r1   r˜   A  s   
ÿ
r˜   rW   r£   c                 C   s   t stdƒ‚tj| fi |¤ŽS )NzEConverting to a compiled module is only supported in PyTorch >= 2.0.0)r   rs   r“   Úcompile)rW   r£   r0   r0   r1   Ú_to_compiledW  s   r¥   c                 C   s   t | ƒ\} }t| tttfƒS )aZ  Checks if an object was set up by Fabric.

    A :class:`~torch.nn.Module` may be wrapped by a :class:`_FabricModule`, a :class:`~torch.optim.Optimizer`
    may be wrapped by a :class:`_FabricOptimizer`, or a :class:`~torch.utils.data.DataLoader` may be wrapped by
    :class:`_FabricDataLoader`.

    Args:
        obj: The object to test.

    )r˜   r=   rM   r#   r†   )r–   rl   r0   r0   r1   Ú
is_wrapped^  s   r¦   Ú
compile_fnc                    s&   t ˆ ƒdtdtdtf‡ fdd„ƒ}|S )aZ  Wraps the ``torch.compile`` function and captures the compile arguments.

    We extract the compile arguments so that we can reapply ``torch.compile`` in ``Fabric.setup()`` with the
    same arguments as the user passed to the original call. The arguments get stored in a dictionary
    ``_compile_kwargs`` on the returned compiled module.

    rX   rA   r'   c                     sH   | r
t | d tjƒsˆ | i |¤ŽS | d }ˆ |fi |¤Ž}t|ƒ|_|S r—   )r=   r   rƒ   r   r¡   )rX   rA   r;   Zcompiled_model©r§   r0   r1   Ú_capturex  s   
z)_capture_compile_kwargs.<locals>._capture)r   r   )r§   r©   r0   r¨   r1   Ú_capture_compile_kwargsm  s   
rª   )?r|   Úcopyr   Ú	functoolsr   Útypingr   r   r   r   r   r	   r
   r   r   r   r   r   r   r“   Ú#lightning_utilities.core.apply_funcr   r   r   Ztorch.nn.modules.moduler   Útorch.optimr   Útorch.utils.datar   Útyping_extensionsr   Úlightning.fabric.pluginsr   Úlightning.fabric.strategiesr   Úlightning.fabric.utilitiesr   Úlightning.fabric.utilities.datar   Ú-lightning.fabric.utilities.device_dtype_mixinr   Ú"lightning.fabric.utilities.importsr   Ú lightning.fabric.utilities.typesr   r�   r    rL   r!   rz   r#   rM   r†   r    rƒ   r˜   r¥   Úobjectr„   r¦   rª   r¤   r0   r0   r0   r1   Ú<module>   sF   <1 %8 ÿ