o
    ¼ôj|B  ã                   @   sÚ  d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZmZmZmZmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZm Z m!Z!m"Z" d
dl#m$Z$ d
dl%m&Z&m'Z' d
dl(m)Z) erzd
dl*m+Z+ e  ,e-¡Z.e /d¡Z0dZ1dddœZ2eG dd„ dƒƒZ3e"				dFdee3 dee4 de4de4dee4 dee4 deee4e4f  deee4  deee5 ee5 ee4 f fdd „ƒZ6G d!d"„ d"eƒZ7G d#d$„ d$eƒZ8			dGd%d&d'edee4 deee4e4f  dee4 ddfd(d)„Z9d*e5fd+d,„Z:d-e5fd.d/„Z;d0e5fd1d2„Z<d%d&d3e4ddfd4d5„Z=d%d&d6ed7e>d3e4ddf
d8d9„Z?d6ed:e3d7e>dee4 fd;d<„Z@d=ee d>e4de8fd?d@„ZAd%d&dAee4 d7e>dee fdBdC„ZBd%d&dAee4 d7e>dee fdDdE„ZCdS )Hz.Git LFS related type definitions and utilitiesé    N)Ú	dataclass)Úceil)Úgetsize)ÚPath)ÚTYPE_CHECKINGÚBinaryIOÚDictÚIterableÚListÚOptionalÚTupleÚ	TypedDict)Úunquote)Ú	constantsé   )Úbuild_hf_headersÚfix_hf_endpoint_in_urlÚget_sessionÚhf_raise_for_statusÚhttp_backoffÚloggingÚtqdmÚvalidate_hf_hub_args)ÚSliceFileObj)Úsha256Úsha_fileobj)Úis_tqdm_disabled)ÚCommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)ÚAcceptzContent-Typec                   @   s`   e Zd ZU dZeed< eed< eed< edefdd„ƒZ	edefd	d
„ƒZ
edefdd„ƒZdS )Ú
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   ÚsizeÚsampleÚpathc                 C   s\   t |ƒ}t |d¡�}| d¡d d… }t|ƒ}W d   ƒ n1 s"w   Y  | |||d�S )NÚrbé   ©r    r   r!   )r   ÚioÚopenÚpeekr   )Úclsr"   r    Úfiler!   Úsha© r,   úf/var/www/html/academy/tools/voices/clone/cosy-venv/lib/python3.10/site-packages/huggingface_hub/lfs.pyÚ	from_pathO   s   
þzUploadInfo.from_pathÚdatac                 C   s&   t |ƒ ¡ }| t|ƒ|d d… |d�S )Nr$   )r    r!   r   )r   ÚdigestÚlen)r)   r/   r+   r,   r,   r-   Ú
from_bytesW   s   zUploadInfo.from_bytesÚfileobjc                 C   sD   |  d¡}| dtj¡ t|ƒ}| ¡ }| dtj¡ | |||d�S )Nr$   r   r%   )ÚreadÚseekr&   ÚSEEK_SETr   Útell)r)   r3   r!   r+   r    r,   r,   r-   Úfrom_fileobj\   s   
zUploadInfo.from_fileobjN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚbytesÚ__annotations__ÚintÚclassmethodÚstrr.   r2   r   r8   r,   r,   r,   r-   r   <   s   
 r   Úupload_infosÚtokenÚ	repo_typeÚrepo_idÚrevisionÚendpointÚheadersÚ	transfersÚreturnc                 C   s  |dur|nt j}d}|t jv rt j| }|› d|› |› d�}	d|dur&|nddgdd	„ | D ƒd
dœ}
|dur?dt|ƒi|
d< i t¥t|d�¥|pJi ¥}tƒ j|	||
d�}t|ƒ | 	¡ }| 
dd¡}t|tƒsltdƒ‚| 
d¡}t|tƒrx|nd}dd	„ |D ƒdd	„ |D ƒ|fS )aÍ  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        revision (`str`, *optional*):
            The git revision to upload to.
        headers (`dict`, *optional*):
            Additional headers to include in the request
        transfers (`list`, *optional*):
            List of transfer methods to use. Defaults to ["basic", "multipart"].

    Returns:
        `LfsBatchInfo`: 3-tuple:
            - First element is the list of upload instructions from the server
            - Second element is a list of errors, if any
            - Third element is the chosen transfer adapter if provided by the server (e.g. "basic", "multipart", "xet")

    Raises:
        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If an argument is invalid or the server response is malformed.
        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            If the server returned an error.
    NÚ ú/z.git/info/lfs/objects/batchÚuploadÚbasicÚ	multipartc                 S   s   g | ]}|j  ¡ |jd œ‘qS )©Úoidr    )r   Úhexr    )Ú.0rM   r,   r,   r-   Ú
<listcomp>š   s    ýþÿz'post_lfs_batch_info.<locals>.<listcomp>r   )Ú	operationrI   ÚobjectsZ	hash_algoÚnameÚref)rC   ©rH   ÚjsonrV   zMalformed response from serverZtransferc                 S   s   g | ]
}d |vrt |ƒ‘qS ©Úerror)Ú_validate_batch_actions©rS   Úobjr,   r,   r-   rT   ·   ó    c                 S   s   g | ]
}d |v rt |ƒ‘qS r[   )Ú_validate_batch_errorr^   r,   r,   r-   rT   ¸   r`   )r   ÚENDPOINTÚREPO_TYPES_URL_PREFIXESr   ÚLFS_HEADERSr   r   Úpostr   rZ   ÚgetÚ
isinstanceÚlistÚ
ValueErrorrA   )rB   rC   rD   rE   rF   rG   rH   rI   Ú
url_prefixZ	batch_urlÚpayloadÚrespZ
batch_inforV   Úchosen_transferr,   r,   r-   Úpost_lfs_batch_infof   sB   ,

ûöÿþý

ýrn   c                   @   s   e Zd ZU eed< eed< dS )ÚPayloadPartTÚ
partNumberÚetagN)r9   r:   r;   r?   r>   rA   r,   r,   r,   r-   ro   ½   s   
 ro   c                   @   s&   e Zd ZU dZeed< ee ed< dS )ÚCompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rQ   ÚpartsN)r9   r:   r;   r<   rA   r>   r
   ro   r,   r,   r,   r-   rr   Â   s   
 rr   rU   r   Úlfs_batch_actionc              	   C   sH  t |ƒ | d¡}|du rt d| j› d�¡ dS |d d }t|ƒ |d  d¡}|dur2t|ƒ | di ¡}| d¡}	t|d	 |d
�}
|	durjzt|	ƒ}	W n tt	fy`   td|	› d�ƒ‚w t
| ||	|
d� nt| |
d� |dur™t|ƒ t|d	 |ƒ}tƒ j|t||d�| jj ¡ | jjdœd�}t|ƒ t | j› d�¡ dS )aw  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        headers (`dict`, *optional*):
            Headers to include in the request, including authentication and user agent headers.

    Raises:
        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If `lfs_batch_action` is improperly formatted
        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            If the upload resulted in an error
    ÚactionsNzContent of file z. is already present upstream - skipping uploadrM   ÚverifyÚheaderÚ
chunk_sizeÚhref)rG   zTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.)rU   rw   rx   Ú
upload_url)rU   rz   )rC   rH   rP   rY   z: Upload successful)r]   rf   ÚloggerÚdebugÚpath_in_repoÚ_validate_lfs_actionr   r?   ri   Ú	TypeErrorÚ_upload_multi_partÚ_upload_single_partr   re   r   Úupload_infor   rR   r    r   )rU   rt   rC   rH   rG   ru   Úupload_actionÚverify_actionrw   rx   rz   Z
verify_urlZverify_respr,   r,   r-   Ú
lfs_uploadÉ   sB   


ÿÿ
ýr…   Ú
lfs_actionc                 C   s:   t |  d¡tƒr|  d¡du st |  d¡tƒstdƒ‚| S )ú.validates response from the LFS batch endpointry   rw   Nz"lfs_action is improperly formatted)rg   rf   rA   Údictri   )r†   r,   r,   r-   r~     s
   ÿr~   Úlfs_batch_actionsc                 C   sp   t |  d¡tƒrt |  d¡tƒstdƒ‚|  di ¡ d¡}|  di ¡ d¡}|dur.t|ƒ |dur6t|ƒ | S )r‡   rQ   r    z)lfs_batch_actions is improperly formattedru   rM   rv   N)rg   rf   rA   r?   ri   r~   )r‰   rƒ   r„   r,   r,   r-   r]     s    r]   Úlfs_batch_errorc                 C   sh   t |  d¡tƒrt |  d¡tƒstdƒ‚|  d¡}t |tƒr.t | d¡tƒr.t | d¡tƒs2tdƒ‚| S )r‡   rQ   r    z'lfs_batch_error is improperly formattedr\   ÚmessageÚcode)rg   rf   rA   r?   ri   rˆ   )rŠ   Z
error_infor,   r,   r-   ra   '  s    
ÿþýra   rz   c                 C   sH   | j dd��}td||d�}t|ƒ W d  ƒ dS 1 sw   Y  dS )a¤  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises:
     [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
        If the upload resulted in an error.
    T©Ú	with_tqdmÚPUT©r/   N)Úas_filer   r   )rU   rz   r3   Úresponser,   r,   r-   r�   5  s   
"ýr�   rw   rx   c                 C   sŽ   t || j|d�}tj}tjr!t| jtƒs!t| jtƒs!t 	d¡ d}|r*t
| ||d�nt| ||d�}tƒ j|t|| jj ¡ ƒtd�}t|ƒ dS )z@
    Uploads file using HF multipart LFS transfer protocol.
    )rw   r‚   rx   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rU   Úsorted_parts_urlsrx   )rZ   rH   N)Ú_get_sorted_parts_urlsr‚   r   ÚHF_HUB_ENABLE_HF_TRANSFERrg   Úpath_or_fileobjrA   r   ÚwarningsÚwarnÚ_upload_parts_hf_transferÚ_upload_parts_iterativelyr   re   Ú_get_completion_payloadr   rR   rd   r   )rU   rw   rx   rz   r“   Zuse_hf_transferÚresponse_headersZcompletion_resr,   r,   r-   r€   K  s,   ÿ
þ
ýÿÿýýr€   r‚   c                 C   sN   dd„ t dd„ |  ¡ D ƒdd„ d�D ƒ}t|ƒ}|t|j| ƒkr%tdƒ‚|S )Nc                 S   s   g | ]\}}|‘qS r,   r,   )rS   Ú_rz   r,   r,   r-   rT   o  s    ÿÿz*_get_sorted_parts_urls.<locals>.<listcomp>c                 S   s2   g | ]\}}|  ¡ rt|ƒd krt|dƒ|f‘qS )r   é
   )Úisdigitr1   r?   )rS   Zpart_numrz   r,   r,   r-   rT   r  s    ýþc                 S   s   | d S )Nr   r,   )Útr,   r,   r-   Ú<lambda>w  s    z(_get_sorted_parts_urls.<locals>.<lambda>)Úkeyz0Invalid server response to upload large LFS file)ÚsortedÚitemsr1   r   r    ri   )rw   r‚   rx   Zsorted_part_upload_urlsZ	num_partsr,   r,   r-   r”   n  s   þúþr”   rœ   rQ   c                 C   sf   g }t | ƒD ]'\}}| d¡}|d u s|dkr#td|› d|d › �ƒ‚| |d |dœ¡ q||dœS )Nrq   rK   zInvalid etag (`z`) returned for part r   )rp   rq   )rQ   rs   )Ú	enumeraterf   ri   Úappend)rœ   rQ   rs   Zpart_numberrw   rq   r,   r,   r-   r›   €  s   
þÿ
r›   r“   c           	   
   C   sœ   g }| j dd��<}t|ƒD ].\}}t||| |d��}td||d�}t|ƒ | |j¡ W d   ƒ n1 s6w   Y  qW d   ƒ |S 1 sGw   Y  |S )NTr�   )Ú	seek_fromÚ
read_limitr�   r�   )r‘   r¥   r   r   r   r¦   rH   )	rU   r“   rx   rH   r3   Zpart_idxZpart_upload_urlZfileobj_sliceZpart_upload_resr,   r,   r-   rš   �  s&   ýø€ÿ
ÿõrš   c           
      C   s  zddl m} W n ty   tdƒ‚w dt |¡jv }|s#t d¡ | j	j
}| j}t|ƒdkr9d|dd … › �}td	d
|d|tt ¡ ƒdd��;}z|d| j||ddddœ|r\d|jini ¤Ž}W n tyr }	 ztdƒ|	‚d }	~	ww |sz| |¡ |W  d   ƒ S 1 s†w   Y  d S )Nr   )Úmultipart_uploadzªFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.Úcallbackz”You are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`.é(   u   (â€¦)iØÿÿÿÚBTzhuggingface_hub.lfs_upload)ÚunitÚ
unit_scaleÚtotalÚinitialÚdescÚdisablerW   é€   é   é   )Ú	file_pathZ
parts_urlsrx   Ú	max_filesÚparallel_failuresÚmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.r,   )Úhf_transferr©   ÚImportErrorri   ÚinspectÚ	signatureÚ
parametersr—   r˜   r‚   r    r}   r1   r   r   r{   ÚgetEffectiveLevelr–   ÚupdateÚ	ExceptionÚRuntimeError)
rU   r“   rx   r©   Úsupports_callbackr¯   r±   ÚprogressÚoutputÚer,   r,   r-   r™   ¡  sb   ÿÿÿ
ùú
ù	ÿý€ÿ
$ær™   )NNNN)NNN)Dr<   r¼   r&   Úrer—   Údataclassesr   Úmathr   Zos.pathr   Úpathlibr   Útypingr   r   r   r	   r
   r   r   r   Úurllib.parser   Úhuggingface_hubr   Úutilsr   r   r   r   r   r   r   r   Z
utils._lfsr   Z	utils.shar   r   Z
utils.tqdmr   Ú_commit_apir   Ú
get_loggerr9   r{   ÚcompileZ	OID_REGEXZLFS_MULTIPART_UPLOAD_COMMANDrd   r   rA   rˆ   rn   ro   rr   r…   r~   r]   ra   r�   r?   r€   r”   r›   rš   r™   r,   r,   r,   r-   Ú<module>   sº   ((


þ)øÿþýüûúù
ø	÷V
ûÿþýüû
úF
#ÿÿÿ
þÿÿÿþ