U
    4AfA                     @   s   d Z ddlZddlZddlZddlZddlZddlmZ ddlm	Z	 ddl
mZmZmZ ddlZddlmZ dd	lmZ eeZd
ZG dd deZG dd dZedddZee dddZG dd de	ZG dd dZdS )a  
Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs.

Inspired by: github.com/cbartz/git-lfs-swift-transfer-agent/blob/master/git_lfs_swift_transfer.py

Spec is: github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md


To launch debugger while developing:

``` [lfs "customtransfer.multipart"]
path = /path/to/transformers/.env/bin/python args = -m debugpy --listen 5678 --wait-for-client
/path/to/transformers/src/transformers/commands/transformers_cli.py lfs-multipart-upload ```    N)ArgumentParser)AbstractContextManager)DictListOptional   )logging   )BaseTransformersCLICommandzlfs-multipart-uploadc                   @   s"   e Zd ZdZeedddZdS )LfsCommandsu  
    Implementation of a custom transfer agent for the transfer type "multipart" for git-lfs. This lets users upload
    large files >5GB 🔥. Spec for LFS custom transfer agent is:
    https://github.com/git-lfs/git-lfs/blob/master/docs/custom-transfers.md

    This introduces two commands to the CLI:

    1. $ transformers-cli lfs-enable-largefiles

    This should be executed once for each model repo that contains a model file >5GB. It's documented in the error
    message you get if you just try to git push a 5GB file without having enabled it before.

    2. $ transformers-cli lfs-multipart-upload

    This command is called by lfs directly and is not meant to be called by the user.
    )parserc                 C   sP   | j ddd}|jdtdd |jdd d	 | j td
d}|jdd d	 d S )Nzlfs-enable-largefileszeDeprecated: use `huggingface-cli` instead. Configure your repository to enable upload of files > 5GB.)helppathz/Local path to repository you want to configure.)typer   c                 S   s   t | S N)LfsEnableCommandargs r   =/tmp/pip-unpacked-wheel-zw5xktn0/transformers/commands/lfs.py<lambda>?       z1LfsCommands.register_subcommand.<locals>.<lambda>)funczgDeprecated: use `huggingface-cli` instead. Command will get called by git-lfs, do not call it directly.c                 S   s   t | S r   )LfsUploadCommandr   r   r   r   r   H   r   )
add_parseradd_argumentstrset_defaultsLFS_MULTIPART_UPLOAD_COMMAND)r   Zenable_parserZupload_parserr   r   r   register_subcommand6   s    zLfsCommands.register_subcommandN)__name__
__module____qualname____doc__staticmethodr   r   r   r   r   r   r   $   s   r   c                   @   s   e Zd Zdd Zdd ZdS )r   c                 C   s
   || _ d S r   r   selfr   r   r   r   __init__L   s    zLfsEnableCommand.__init__c                 C   sp   t d tj| jj}tj|s6td td t	j
d d|d t	j
dt  d|d td d S )	NzcManaging repositories through transformers-cli is deprecated. Please use `huggingface-cli` instead.z)This does not look like a valid git repo.r	   z=git config lfs.customtransfer.multipart.path transformers-cliT)checkcwdz-git config lfs.customtransfer.multipart.args z Local repo set up for largefiles)warningswarnosr   abspathr   isdirprintexit
subprocessrunsplitr   )r&   
local_pathr   r   r   r2   O   s$      zLfsEnableCommand.runNr    r!   r"   r'   r2   r   r   r   r   r   K   s   r   msgc                 C   s(   t | d } tj|  tj  dS )z-Write out the message in Line delimited JSON.
N)jsondumpssysstdoutwriteflushr6   r   r   r   	write_msgb   s    r?   )returnc                  C   sV   t tj  } d| d| dfkr0dS | ddkrRtd t	d | S )z$Read Line delimited JSON from stdin.	terminater   eventN)downloaduploadzReceived unexpected messager	   )
r9   loadsr;   stdinreadlinestripgetloggercriticalr0   r6   r   r   r   read_msgi   s    

rL   c                   @   sL   e Zd ZdZeeedddZdd Zdd Zdd
dZ	dd Z
dd ZdS )	FileSlicezq
    File-like object that only reads a slice of a file

    Inspired by stackoverflow.com/a/29838711/593036
    )filepath	seek_from
read_limitc                 C   s   || _ || _|| _d| _d S )Nr   )rN   rO   rP   n_seen)r&   rN   rO   rP   r   r   r   r'      s    zFileSlice.__init__c                 C   s    t | jd| _| j| j | S )Nrb)openrN   fseekrO   r&   r   r   r   	__enter__   s    zFileSlice.__enter__c                 C   s$   t | j j}t| j|| j S r   )r,   fstatrT   filenost_sizeminrP   rO   )r&   total_lengthr   r   r   __len__   s    zFileSlice.__len__c                 C   sP   | j | jkrdS | j| j  }| j|dk r.|nt||}|  j t|7  _ |S )Nr   r   )rQ   rP   rT   readr[   len)r&   nZremaining_amountdatar   r   r   r_      s    zFileSlice.readc                 c   s   | j ddV  d S )Ni  @ )ra   )r_   rV   r   r   r   __iter__   s    zFileSlice.__iter__c                 G   s   | j   d S r   )rT   closer%   r   r   r   __exit__   s    zFileSlice.__exit__N)r^   )r    r!   r"   r#   r   intr'   rW   r]   r_   rc   re   r   r   r   r   rM   x   s   
rM   c                   @   s   e Zd Zdd Zdd ZdS )r   c                 C   s
   || _ d S r   r   r%   r   r   r   r'      s    zLfsUploadCommand.__init__c              
   C   s^  t tj  }|ddkr0|ddksLtddddi td	 ti  t	 }|d krltd
 |d }|d }|d d }|d d }t
|d}t| }g }	t|D ]r\}
}t||
| |dR}tj||d}|  |	|jd|
d	 d td||
d	 | |d W 5 Q R X qtj|||	dd}|  td|d qTd S )NrB   initZ	operationrD   error    zWrong lfs init operation)codemessager	   r   oidr   actionhrefheader
chunk_size)rO   rP   )rb   etag)rq   Z
partNumberprogress)rB   rl   Z
bytesSoFarZbytesSinceLast)rl   parts)r9   Zcomplete)rB   rl   )r9   rE   r;   rF   rG   rH   rI   r?   r0   rL   rf   poplistvalues	enumeraterM   requestsputraise_for_statusappendheaderspost)r&   Zinit_msgr7   rl   rN   Zcompletion_urlro   rp   Zpresigned_urlsrs   iZpresigned_urlrb   rr   r   r   r2      sN    




zLfsUploadCommand.runNr5   r   r   r   r   r      s   r   )r#   r9   r,   r1   r;   r*   argparser   
contextlibr   typingr   r   r   rx   utilsr    r
   Z
get_loggerr    rJ   r   r   r   r?   rL   rM   r   r   r   r   r   <module>   s&   
'%