o
    £ôjÝ  ã                   @   s¤   d dl Z d dlZd dlZd dlZd dlmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZ er:d dlmZ edejdƒZG dd	„ d	ƒZG d
d„ deƒZdS )é    N)ÚTYPE_CHECKINGÚAnyÚDictÚListÚLiteralÚOptionalÚUnion)Úcompare_version)Ú_PATH)ÚFabricÚtorchmetricsz1.0.0c                       s\  e Zd ZdZ							d/ded	 d
ededee dee dee de	f‡ fdd„Z
e ¡ dddejdededdf
dd„ƒZdejde	fdd„Zdddeddfdd„Zdeeejf deeejf de	fd d!„Zdeeejf deeejf de	fd"d#„Zd$ejde	fd%d&„Zd'ejddfd(d)„Zdeeef fd*d+„Zd,eeef ddfd-d.„Z‡  ZS )0ÚSpikeDetectionaÊ  Spike Detection Callback.

    Terminates training with a ``TrainingSpikeException`` when a loss-spike was detected and
    saves the batches to skip when resuming to a file.

    We skip the current and the previous batch since it is unclear whether the previous batch
    altered the weights in a way that it causes the spike or just the current batch is corrupted somehow.

    Args:
        mode: Whether to minimize or maximize the tracked metric
        window: A running mean of metrics with ``window`` size. Serves as reference value for spikes.
        warmup: After how many batches spike-tracking should start
        atol: An absolute tolerance.  Every diff between the running mean and the current value,
            that's not an improvement and above ``atol`` will be considered a spike
        rtol: A relative tolerance. Every diff between the running mean and the current value,
            that's higher than ``rtol * running_mean`` is considered a spike
        exclude_batches_path: Where to save the file that contains the batches to exclude.
            Will default to current directory.
        finite_only: If set to ``False``, consider non-finite values like NaN, inf and -inf a spike as well.

    Úminé
   é   Nç       @TÚmode)r   ÚmaxÚwindowÚwarmupÚatolÚrtolÚexclude_batches_pathÚfinite_onlyc           
         sŠ   t rddlm} ddlm}	 ntdƒ‚tƒ  ¡  d| _|	|ddd�|d�| _	d| j	_
d| j	_|| _|| _|| _|| _g | _|| _|| _d S )	Nr   )Ú
MeanMetric)ÚRunningzISpikeDetection requires torchmetrics>=1.0.0! Please upgrade your version!ç        F)Údist_sync_on_stepÚsync_on_compute)r   )Ú!_TORCHMETRICS_GREATER_EQUAL_1_0_0Útorchmetrics.aggregationr   Ztorchmetrics.wrappersr   ÚRuntimeErrorÚsuperÚ__init__Úlast_valÚrunning_meanr   r   r   r   r   r   Úbad_batchesr   r   )
Úselfr   r   r   r   r   r   r   r   r   ©Ú	__class__© ús/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/lightning/fabric/utilities/spike.pyr#   )   s    


zSpikeDetection.__init__Úfabricr   ÚlossÚbatchÚ	batch_idxÚreturnc                 C   sÎ   |dkr| j  |jj¡ | jdu rt ¡ | _t| jƒ d¡s'tj	 
| jd¡| _t|| jko1|  |¡ƒ}|j ¡  |jj|dd�}|rJ|  ||¡ dS | jp[|jjtt |¡ ¡ ƒdd�}|re|  |¡ dS dS )z)Checks if we currently have a loss-spike.r   Nz.jsonzskip_batches.jsonF)ÚallT)r%   ÚtoÚstrategyÚroot_devicer   ÚosÚgetcwdÚstrÚendswithÚpathÚjoinÚboolr   Ú	_is_spikeÚbarrierÚreduce_boolean_decisionÚ_handle_spiker   ÚtorchÚisfiniter1   Ú_update_stats)r'   r,   r-   r.   r/   Zis_spikeZis_spike_globalZis_finite_allr*   r*   r+   Úon_train_batch_endI   s"   


ÿÿz!SpikeDetection.on_train_batch_endc                 C   s€   t  ¡ � t  d¡ | j ¡ }W d   ƒ n1 sw   Y  || j }| jr-t |¡s-dS |  	|¡r4dS |  
||¡o?|  ||¡S )NÚignoreTF)ÚwarningsÚcatch_warningsÚsimplefilterr%   Úcomputer$   r   r@   rA   Ú
_is_betterÚ_check_atolÚ_check_rtol)r'   r-   Zrunning_valZ	curr_diffr*   r*   r+   r<   d   s   

þ

zSpikeDetection._is_spikec                 C   sŒ   | j  |d |g¡ |jdkrA| jd usJ ‚tjtj | j¡dd� t| jdƒ�}t	j
| j |dd� W d   ƒ n1 s<w   Y  t|d�‚)	Nr   r   T)Úexist_okÚwé   )Úindent)r/   )r&   ÚextendÚglobal_rankr   r5   Úmakedirsr9   ÚdirnameÚopenÚjsonÚdumpÚTrainingSpikeException)r'   r,   r/   Úfr*   r*   r+   r?   t   s   
ÿ
zSpikeDetection._handle_spikeÚval_aÚval_bc                 C   s$   | j d u ptt|| ƒt| j ƒkƒS ©N)r   r;   Úabs©r'   rY   rZ   r*   r*   r+   rJ   ƒ   ó   $zSpikeDetection._check_atolc                 C   s(   | j d u ptt|| ƒt| j | ƒkƒS r[   )r   r;   r\   r]   r*   r*   r+   rK   †   s   (zSpikeDetection._check_rtolÚdiff_valc                 C   sD   | j dkrt|dk ¡ ƒS | j dkrt|dk ¡ ƒS td| j › �ƒ‚)Nr   r   r   r   z*Invalid mode. Has to be min or max, found )r   r;   r1   Ú
ValueError)r'   r_   r*   r*   r+   rI   ‰   s
   

zSpikeDetection._is_betterÚvalc                 C   s   | j  |¡ || _d S r[   )r%   Úupdater$   )r'   ra   r*   r*   r+   rB   ‘   s   
zSpikeDetection._update_statsc              
   C   sL   t | jtjƒr| j ¡ n| j| j| j| j| j| j	| j
| j ¡ | jj ¡ dœ	S )N)	r$   r   r   r   r   r&   Úbad_batches_pathÚrunningÚmean)Ú
isinstancer$   r@   ÚTensorÚitemr   r   r   r   r&   r   r%   Ú
state_dictÚbase_metric)r'   r*   r*   r+   ri   –   s   
÷zSpikeDetection.state_dictri   c                 C   s~   |  d¡| _|  d¡| _|  d¡| _|  d¡| _|  d¡| _|  d¡| _|  d¡| _| j 	|  d¡¡ | j
j 	|  d	¡¡ d S )
Nr$   r   r   r   r   r&   rc   rd   re   )Úpopr$   r   r   r   r   r&   r   rd   Úload_state_dictr%   rj   )r'   ri   r*   r*   r+   rl   £   s   zSpikeDetection.load_state_dict)r   r   r   Nr   NT)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úintr   Úfloatr
   r;   r#   r@   Úno_gradrg   r   rC   r<   r?   r   rJ   rK   rI   rB   r   r7   ri   rl   Ú__classcell__r*   r*   r(   r+   r      sD    øþýüûúùø "**"r   c                       s.   e Zd ZdZdededef‡ fdd„Z‡  ZS )rW   z,Exception to be raised with Training Spikes.r/   ÚargsÚkwargsc                    s$   t ƒ jd|› �g|¢R i |¤Ž d S )Nz!Training spike detected in batch )r"   r#   )r'   r/   ru   rv   r(   r*   r+   r#   ²   r^   zTrainingSpikeException.__init__)rm   rn   ro   rp   rq   r   r#   rt   r*   r*   r(   r+   rW   ¯   s    "rW   )rU   Úoperatorr5   rE   Útypingr   r   r   r   r   r   r   r@   Ú lightning_utilities.core.importsr	   Ú lightning.fabric.utilities.typesr
   Úlightning.fabric.fabricr   Úger   r   r!   rW   r*   r*   r*   r+   Ú<module>   s    $ 