U
    5A·fΐ,  γ                   @   s¦  d dl mZmZ G dd dedZG dd dedZG dd dedZG d	d
 d
edZG dd dedZG dd dedZG dd dedZ	G dd dedZ
G dd dedZG dd dedZG dd dedZG dd dedZG dd dedZG dd dedZG dd  d edZG d!d" d"edZG d#d$ d$edZG d%d& d&edZG d'd( d(edZG d)d* d*edZG d+d, d,edZG d-d. d.edZG d/d0 d0edZG d1d2 d2edZG d3d4 d4edZG d5d6 d6edZG d7d8 d8edZG d9d: d:edZG d;d< d<edZG d=d> d>edZ G d?d@ d@edZ!G dAdB dBedZ"G dCdD dDedZ#G dEdF dFedZ$G dGdH dHedZ%G dIdJ dJedZ&G dKdL dLedZ'G dMdN dNedZ(G dOdP dPedZ)G dQdR dRedZ*G dSdT dTedZ+G dUdV dVedZ,G dWdX dXedZ-G dYdZ dZedZ.G d[d\ d\edZ/G d]d^ d^edZ0G d_d` d`edZ1G dadb dbedZ2G dcdd ddedZ3G dedf dfedZ4G dgdh dhedZ5G didj djedZ6G dkdl dledZ7G dmdn dnedZ8G dodp dpedZ9G dqdr dredZ:G dsdt dtedZ;G dudv dvedZ<G dwdx dxedZ=G dydz dzedZ>G d{d| d|edZ?G d}d~ d~edZ@G dd dedZAG dd dedZBG dd dedZCdS )ι   )ΪDummyObjectΪrequires_backendsc                   @   s   e Zd ZdgZdd ZdS )ΪAlbertTokenizerFastΪ
tokenizersc                 O   s   t | dg d S ©Nr   ©r   ©ΪselfΪargsΪkwargs© r   ϊO/tmp/pip-unpacked-wheel-zw5xktn0/transformers/utils/dummy_tokenizers_objects.pyΪ__init__   s    zAlbertTokenizerFast.__init__N©Ϊ__name__Ϊ
__module__Ϊ__qualname__Z	_backendsr   r   r   r   r   r      s   r   )Ϊ	metaclassc                   @   s   e Zd ZdgZdd ZdS )ΪBartTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    zBartTokenizerFast.__init__Nr   r   r   r   r   r      s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪBarthezTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    zBarthezTokenizerFast.__init__Nr   r   r   r   r   r      s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    zBertTokenizerFast.__init__Nr   r   r   r   r   r      s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪBigBirdTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   $   s    zBigBirdTokenizerFast.__init__Nr   r   r   r   r   r   !   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪBlenderbotTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   +   s    z BlenderbotTokenizerFast.__init__Nr   r   r   r   r   r   (   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪBlenderbotSmallTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   2   s    z%BlenderbotSmallTokenizerFast.__init__Nr   r   r   r   r   r   /   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪBloomTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   9   s    zBloomTokenizerFast.__init__Nr   r   r   r   r   r   6   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪCamembertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   @   s    zCamembertTokenizerFast.__init__Nr   r   r   r   r   r   =   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪCLIPTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   G   s    zCLIPTokenizerFast.__init__Nr   r   r   r   r   r   D   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪCodeLlamaTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   N   s    zCodeLlamaTokenizerFast.__init__Nr   r   r   r   r   r   K   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪCodeGenTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   U   s    zCodeGenTokenizerFast.__init__Nr   r   r   r   r   r   R   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪCohereTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   \   s    zCohereTokenizerFast.__init__Nr   r   r   r   r   r   Y   s   r   c                   @   s   e Zd ZdgZdd ZdS )ΪConvBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   c   s    zConvBertTokenizerFast.__init__Nr   r   r   r   r   r    `   s   r    c                   @   s   e Zd ZdgZdd ZdS )ΪCpmTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   j   s    zCpmTokenizerFast.__init__Nr   r   r   r   r   r!   g   s   r!   c                   @   s   e Zd ZdgZdd ZdS )ΪDebertaTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   q   s    zDebertaTokenizerFast.__init__Nr   r   r   r   r   r"   n   s   r"   c                   @   s   e Zd ZdgZdd ZdS )ΪDebertaV2TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   x   s    zDebertaV2TokenizerFast.__init__Nr   r   r   r   r   r#   u   s   r#   c                   @   s   e Zd ZdgZdd ZdS )ΪRealmTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    zRealmTokenizerFast.__init__Nr   r   r   r   r   r$   |   s   r$   c                   @   s   e Zd ZdgZdd ZdS )ΪRetriBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    zRetriBertTokenizerFast.__init__Nr   r   r   r   r   r%      s   r%   c                   @   s   e Zd ZdgZdd ZdS )ΪDistilBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    z DistilBertTokenizerFast.__init__Nr   r   r   r   r   r&      s   r&   c                   @   s   e Zd ZdgZdd ZdS )ΪDPRContextEncoderTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    z'DPRContextEncoderTokenizerFast.__init__Nr   r   r   r   r   r'      s   r'   c                   @   s   e Zd ZdgZdd ZdS )ΪDPRQuestionEncoderTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    z(DPRQuestionEncoderTokenizerFast.__init__Nr   r   r   r   r   r(      s   r(   c                   @   s   e Zd ZdgZdd ZdS )ΪDPRReaderTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ’   s    zDPRReaderTokenizerFast.__init__Nr   r   r   r   r   r)      s   r)   c                   @   s   e Zd ZdgZdd ZdS )ΪElectraTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ©   s    zElectraTokenizerFast.__init__Nr   r   r   r   r   r*   ¦   s   r*   c                   @   s   e Zd ZdgZdd ZdS )ΪFNetTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   °   s    zFNetTokenizerFast.__init__Nr   r   r   r   r   r+   ­   s   r+   c                   @   s   e Zd ZdgZdd ZdS )ΪFunnelTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ·   s    zFunnelTokenizerFast.__init__Nr   r   r   r   r   r,   ΄   s   r,   c                   @   s   e Zd ZdgZdd ZdS )ΪGemmaTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Ύ   s    zGemmaTokenizerFast.__init__Nr   r   r   r   r   r-   »   s   r-   c                   @   s   e Zd ZdgZdd ZdS )ΪGPT2TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Ε   s    zGPT2TokenizerFast.__init__Nr   r   r   r   r   r.   Β   s   r.   c                   @   s   e Zd ZdgZdd ZdS )ΪGPTNeoXTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Μ   s    zGPTNeoXTokenizerFast.__init__Nr   r   r   r   r   r/   Ι   s   r/   c                   @   s   e Zd ZdgZdd ZdS )ΪGPTNeoXJapaneseTokenizerr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Σ   s    z!GPTNeoXJapaneseTokenizer.__init__Nr   r   r   r   r   r0   Π   s   r0   c                   @   s   e Zd ZdgZdd ZdS )ΪHerbertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Ϊ   s    zHerbertTokenizerFast.__init__Nr   r   r   r   r   r1   Χ   s   r1   c                   @   s   e Zd ZdgZdd ZdS )ΪLayoutLMTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   α   s    zLayoutLMTokenizerFast.__init__Nr   r   r   r   r   r2   ή   s   r2   c                   @   s   e Zd ZdgZdd ZdS )ΪLayoutLMv2TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   θ   s    z LayoutLMv2TokenizerFast.__init__Nr   r   r   r   r   r3   ε   s   r3   c                   @   s   e Zd ZdgZdd ZdS )ΪLayoutLMv3TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ο   s    z LayoutLMv3TokenizerFast.__init__Nr   r   r   r   r   r4   μ   s   r4   c                   @   s   e Zd ZdgZdd ZdS )ΪLayoutXLMTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   φ   s    zLayoutXLMTokenizerFast.__init__Nr   r   r   r   r   r5   σ   s   r5   c                   @   s   e Zd ZdgZdd ZdS )ΪLEDTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ύ   s    zLEDTokenizerFast.__init__Nr   r   r   r   r   r6   ϊ   s   r6   c                   @   s   e Zd ZdgZdd ZdS )ΪLlamaTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    zLlamaTokenizerFast.__init__Nr   r   r   r   r   r7     s   r7   c                   @   s   e Zd ZdgZdd ZdS )ΪLongformerTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    z LongformerTokenizerFast.__init__Nr   r   r   r   r   r8     s   r8   c                   @   s   e Zd ZdgZdd ZdS )ΪLxmertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    zLxmertTokenizerFast.__init__Nr   r   r   r   r   r9     s   r9   c                   @   s   e Zd ZdgZdd ZdS )ΪMarkupLMTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    zMarkupLMTokenizerFast.__init__Nr   r   r   r   r   r:     s   r:   c                   @   s   e Zd ZdgZdd ZdS )ΪMBartTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r      s    zMBartTokenizerFast.__init__Nr   r   r   r   r   r;     s   r;   c                   @   s   e Zd ZdgZdd ZdS )ΪMBart50TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   '  s    zMBart50TokenizerFast.__init__Nr   r   r   r   r   r<   $  s   r<   c                   @   s   e Zd ZdgZdd ZdS )ΪMobileBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   .  s    z MobileBertTokenizerFast.__init__Nr   r   r   r   r   r=   +  s   r=   c                   @   s   e Zd ZdgZdd ZdS )ΪMPNetTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   5  s    zMPNetTokenizerFast.__init__Nr   r   r   r   r   r>   2  s   r>   c                   @   s   e Zd ZdgZdd ZdS )ΪMT5TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   <  s    zMT5TokenizerFast.__init__Nr   r   r   r   r   r?   9  s   r?   c                   @   s   e Zd ZdgZdd ZdS )ΪMvpTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   C  s    zMvpTokenizerFast.__init__Nr   r   r   r   r   r@   @  s   r@   c                   @   s   e Zd ZdgZdd ZdS )ΪNllbTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   J  s    zNllbTokenizerFast.__init__Nr   r   r   r   r   rA   G  s   rA   c                   @   s   e Zd ZdgZdd ZdS )ΪNougatTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Q  s    zNougatTokenizerFast.__init__Nr   r   r   r   r   rB   N  s   rB   c                   @   s   e Zd ZdgZdd ZdS )ΪOpenAIGPTTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   X  s    zOpenAIGPTTokenizerFast.__init__Nr   r   r   r   r   rC   U  s   rC   c                   @   s   e Zd ZdgZdd ZdS )ΪPegasusTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   _  s    zPegasusTokenizerFast.__init__Nr   r   r   r   r   rD   \  s   rD   c                   @   s   e Zd ZdgZdd ZdS )ΪQwen2TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   f  s    zQwen2TokenizerFast.__init__Nr   r   r   r   r   rE   c  s   rE   c                   @   s   e Zd ZdgZdd ZdS )ΪReformerTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   m  s    zReformerTokenizerFast.__init__Nr   r   r   r   r   rF   j  s   rF   c                   @   s   e Zd ZdgZdd ZdS )ΪRemBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   t  s    zRemBertTokenizerFast.__init__Nr   r   r   r   r   rG   q  s   rG   c                   @   s   e Zd ZdgZdd ZdS )ΪRobertaTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   {  s    zRobertaTokenizerFast.__init__Nr   r   r   r   r   rH   x  s   rH   c                   @   s   e Zd ZdgZdd ZdS )ΪRoFormerTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    zRoFormerTokenizerFast.__init__Nr   r   r   r   r   rI     s   rI   c                   @   s   e Zd ZdgZdd ZdS )ΪSeamlessM4TTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    z!SeamlessM4TTokenizerFast.__init__Nr   r   r   r   r   rJ     s   rJ   c                   @   s   e Zd ZdgZdd ZdS )ΪSplinterTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    zSplinterTokenizerFast.__init__Nr   r   r   r   r   rK     s   rK   c                   @   s   e Zd ZdgZdd ZdS )ΪSqueezeBertTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    z!SqueezeBertTokenizerFast.__init__Nr   r   r   r   r   rL     s   rL   c                   @   s   e Zd ZdgZdd ZdS )ΪT5TokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r     s    zT5TokenizerFast.__init__Nr   r   r   r   r   rM     s   rM   c                   @   s   e Zd ZdgZdd ZdS )ΪUdopTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ₯  s    zUdopTokenizerFast.__init__Nr   r   r   r   r   rN   ’  s   rN   c                   @   s   e Zd ZdgZdd ZdS )ΪWhisperTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ¬  s    zWhisperTokenizerFast.__init__Nr   r   r   r   r   rO   ©  s   rO   c                   @   s   e Zd ZdgZdd ZdS )ΪXGLMTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   ³  s    zXGLMTokenizerFast.__init__Nr   r   r   r   r   rP   °  s   rP   c                   @   s   e Zd ZdgZdd ZdS )ΪXLMRobertaTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Ί  s    z XLMRobertaTokenizerFast.__init__Nr   r   r   r   r   rQ   ·  s   rQ   c                   @   s   e Zd ZdgZdd ZdS )ΪXLNetTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Α  s    zXLNetTokenizerFast.__init__Nr   r   r   r   r   rR   Ύ  s   rR   c                   @   s   e Zd ZdgZdd ZdS )ΪPreTrainedTokenizerFastr   c                 O   s   t | dg d S r   r   r   r   r   r   r   Θ  s    z PreTrainedTokenizerFast.__init__Nr   r   r   r   r   rS   Ε  s   rS   N)DΪutilsr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   r   r   r   r   Ϊ<module>   s   