U 5Af@sdZddlZddlZddlmZmZmZmZddlZ ddl m Z m Z m Z ddlmZmZmZmZmZmZmZmZddlmZmZmZddlmZmZmZd d lm Z e!e"Z#d Z$d/d d Z%d0ddZ&Gdddej'j(Z)Gdddej'j(Z*ddZ+d1ddZ,Gdddej'j(Z-e j.e/e j.dddZ0Gdddej'j(Z1Gdd d ej'j(Z2eGd!d"d"ej'j(Z3d#Z4ed$e4Gd%d&d&eZ5d'Z6ed$e4Gd(d)d)e5Z7Gd*d+d+e5eZ8ed,e4Gd-d.d.e5eZ9dS)2zTF 2.0 Mistral model.N)ListOptionalTupleUnion)TFBaseModelOutputWithPastTFCausalLMOutputWithPast"TFSequenceClassifierOutputWithPast)TFCausalLanguageModelingLossTFPreTrainedModelTFSequenceClassificationLossget_initializerget_tf_activationkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)add_start_docstrings%add_start_docstrings_to_model_forwardlogging) MistralConfigrcCs|\}}t||ftj|j}t|}t|dddf|dddfkd|}|dkr~tjtj||f|d|gdd}|dkrt |d}t |d}t ||dddg}n,t |ddddddf|d|||f}|S)zm Make causal mask used for bi-directional self-attention, supporting both static and dynamic shapes. Ngrdtypeaxisr) tffillZdtypesZas_dtypeminrangewhereconcatzeros expand_dimsZtile broadcast_to)Zinput_ids_shaperpast_key_values_lengthbsztgt_lenmaskZ mask_condr,S/tmp/pip-unpacked-wheel-zw5xktn0/transformers/models/mistral/modeling_tf_mistral.py_make_causal_mask9s *   ,r.cCs|t|\}}|dk r|n|}tt|dd}t||d||g}dt||}tt|ttjt|tjj d|S)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. Nr?)Zdimsvalue) rrr&r'castr#boolr float32r!)r+rr*r)Zsrc_lenZ expanded_maskZ inverted_maskr,r,r- _expand_maskTs  r4cs0eZdZd fdd Zd ddZddZZS) TFMistralRMSNormư>c stjf|||_||_dS)z? TFMistralRMSNorm is equivalent to T5LayerNorm N)super__init__ hidden_sizevariance_epsilon)selfr9epskwargs __class__r,r-r8fszTFMistralRMSNorm.__init__NcCs(|jd|jdd|_|jrdSd|_dS)Nweightones)nameshapeZ initializerT)Z add_weightr9r@builtr; input_shaper,r,r-buildnszTFMistralRMSNorm.buildcCsT|j}t|tj}tjt|ddd}t|t||j}|j t||S)NrT)rZkeepdims) rrr1r3Z reduce_meanZsquaredividesqrtr:r@)r; hidden_statesZ input_dtypeZvariancer,r,r-callxs zTFMistralRMSNorm.call)r6)N)__name__ __module__ __qualname__r8rGrK __classcell__r,r,r>r-r5es r5cs(eZdZdfdd Zd ddZZS) TFMistralRotaryEmbedding'c sLtjf|||_||_||_d|jtjd|jdtjd|j|_dS)Nr/r)startlimitdeltar) r7r8dimmax_position_embeddingsbaserr"r3inv_freq)r;rWrXrYr=r>r,r-r8s z!TFMistralRotaryEmbedding.__init__NcCsttj|tjd|jj}td||j}tj||gdd}tt||j}tt ||j}|d|}tj||jd}|d|}tj||jd}||fS)Nrzi,j->ijrr) rr1r"int64rZrZeinsumr$cossin)r;xseq_lentZfreqsZembZ cos_valuesZ sin_valuesr,r,r-rKs  zTFMistralRotaryEmbedding.call)rQrR)N)rLrMrNr8rKrOr,r,r>r-rPsrPcCsDt|dd}|dd|f}|d|df}tj| |gddS)z*Rotates half the hidden dims of the input.rrS.Nr)rrr$)r^Z mid_lengthx1Zx2r,r,r- rotate_halfsrbcCsXtt|||}tt|||}||t||}||t||}||fS)a\Applies Rotary Position Embedding to the query and key tensors. Args: q (`tf.Tensor`): The query tensor. k (`tf.Tensor`): The key tensor. cos (`tf.Tensor`): The cosine part of the rotary embedding. sin (`tf.Tensor`): The sine part of the rotary embedding. position_ids (`tf.Tensor`): The position indices of the tokens corresponding to the query and key tensors. For example, this can be used to pass offsetted position ids when working with a KV-cache. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(tf.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. )rr&gatherrb)qkr\r] position_idsZ unsqueeze_dimZq_embedZk_embedr,r,r-apply_rotary_pos_embs rgcs.eZdZfddZddZdddZZS) TFMistralMLPc svtjf|||_|j|_|j|_tjj|jddd|_tjj|jddd|_ tjj|jddd|_ t |j |_ dS)NF gate_projuse_biasrBup_proj down_proj)r7r8configr9intermediate_sizerlayersDenserirlrmrZ hidden_actact_fnr;rnr=r>r,r-r8szTFMistralMLP.__init__cCs ||||||SN)rmrrrirl)r;r^r,r,r-rKszTFMistralMLP.callNc Cs|jr dSd|_t|dddk rJt|jj|j|jfW5QRXt|dddk rt|jj|j|jfW5QRXt|dddk rt|j j|j |j fW5QRXdS)NTrirlrm) rDgetattrr name_scoperirBrGr9rlrmrorEr,r,r-rGszTFMistralMLP.build)N)rLrMrNr8rKrGrOr,r,r>r-rhs rh)rJn_repreturncCsPt|\}}}}|dkr|St|d}tj||dd}t||||||fS)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rrS)Zrepeatsr)rrr&repeatreshape)rJrwbatchnum_key_value_headsslenhead_dimr,r,r- repeat_kvs  rc seZdZdZdeeedfdd Zej eedddZ dej eej eej ee ej ee ee e ej eej ee ej fd d d Z dd dZZS)TFMistralAttentionz Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer and "Generating Long Sequences with Sparse Transformers". Nrn layer_idxc sNtjf|||_||_|dkr8td|jjd|j|_|j |_ |j|j |_ |j |_ |j |j |_ |j|_|j|_d|_|j|_|j |j |jkrtd|jd|j dtjj|j |j ddd |_tjj|j |j dd d |_tjj|j |j dd d |_tjj|jdd d |_t|j |j|jd d|_tjj|jd|_dS)NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.Tz?hidden_size must be divisible by num_heads (got `hidden_size`: z and `num_heads`: z).Fq_projrjk_projv_projo_proj rotary_emb)rXrYrB)Zrate)r7r8rnrlogger warning_oncer?rLr9Znum_attention_heads num_headsr~r|num_key_value_groupsrXZ rope_thetaZ is_causalZattention_dropout ValueErrorrrprqrrrrrPrZDropoutdropoutr;rnrr=r>r,r-r8s>zTFMistralAttention.__init__)tensorr_r)cCs*t||||j|jf}tj|dd}|S)NrrSrrperm)rrzrr~ transpose)r;rr_r)r,r,r-_shapeszTFMistralAttention._shapeFrJattention_maskrfpast_key_valueoutput_attentions use_cacherxcKsd|krtdt|\} } } ||} ||} ||}tjt| | | |j |j fdd} tjt| | | |j |j fdd} tjt|| | |j |j fdd}t| d}|dk r||dj d7}|j ||d\}}t| | |||d\} } |dk r(tj|d| gd d } tj|d |gd d }|r6| |fnd}t| |j} t||j}tj| | d d t|j }|dk r||}t|dd }t|| j}|j||d}t||}tj|dd}t|| | |jf}||}|sd}|||fS)N padding_maskrPassing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`rrr)r^r_)rdrer\r]rfrSrrT)Z transpose_br)training)warningswarnrrrrrrrzrr~r|rCrrgr$rrmatmulmathrIrr1rrr9r)r;rJrrfrrrrr=r)Zq_len_Z query_statesZ key_statesZ value_statesZ kv_seq_lenr\r]Z attn_weightsZ attn_outputr,r,r-rKsp              zTFMistralAttention.callc Cs|jr dSd|_t|dddk rJt|jj|j|jfW5QRXt|dddk rt|jj|j|jfW5QRXt|dddk rt|j j|j |jfW5QRXt|dddk rt|j j|j |j |j fW5QRXdS)NTrrrr) rDrurrvrrBrGr9rrrrr~rEr,r,r-rGkszTFMistralAttention.build)N)NNNFFN)N)rLrMrN__doc__rrintr8rTensorrrr2rKrGrOr,r,r>r-rs&'  Qrc seZdZeedfdd Zd ejeejeejee ejee ee e ejee ejejffdddZ d d d Z Z S) TFMistralDecoderLayerrc s`tjf||j|_t||dd|_t|dd|_t|j|jdd|_ t|j|jdd|_ dS)N self_attnrBmlpinput_layernormr<rBpost_attention_layernorm) r7r8r9rrrhrr5 rms_norm_epsrrrr>r,r-r8~szTFMistralDecoderLayer.__init__NFrc Ksd|krtd|}||}|j||||||d\}} } ||}|}||}||}||}|f} |rx| | f7} |r| | f7} | S)a( Args: hidden_states (`tf.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`tf.Tensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_value (`Tuple(tf.Tensor)`, *optional*): cached past key and value projection states rr)rJrrfrrr)rrrrrr) r;rJrrfrrrr=ZresidualZself_attn_weightsZpresent_key_valueoutputsr,r,r-rKs2      zTFMistralDecoderLayer.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXt|dddk rt|jj|jdW5QRXt|dddk rt|j j|j dW5QRXdS)NTrrrr) rDrurrvrrBrGrrrrEr,r,r-rGszTFMistralDecoderLayer.build)NNNFF)N)rLrMrNrrr8rrrrr2rKrGrOr,r,r>r-r}s   ;rcseZdZdZeZedfdd ZddZddZd d Z e de j e e j e e j e ee j e e j e ee ee ee eeeefd d dZdddZZS)TFMistralMainLayerz Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MistralDecoderLayer`] Args: config: MistralConfig rnc s~tjf|j|_j|_j|_tjjjjdd|_ fddt j D|_j |_ t jjdd|_|_dS)N embed_tokens)Z input_dimZ output_dimrBcs g|]}t|d|dqS)zlayers.r)r).0rrr,r- sz/TFMistralMainLayer.__init__..normr)r7r8 pad_token_idZ padding_idx vocab_sizer9rrpZ Embeddingrr"Znum_hidden_layersZ_attn_implementationr5rrrnrsr>rr-r8s zTFMistralMainLayer.__init__cCs|jSrtrr;r,r,r-get_input_embeddingssz'TFMistralMainLayer.get_input_embeddingscCs ||_dSrtrr;r0r,r,r-set_input_embeddingssz'TFMistralMainLayer.set_input_embeddingscCsHd}t||j|d}|dk rDt||j|dd}|dkr<|n||}|S)N)r(r)r*)r.rr4)r;rrF inputs_embedsr(Zcombined_attention_maskZexpanded_attn_maskr,r,r-_prepare_decoder_attention_masksz2TFMistralMainLayer._prepare_decoder_attention_maskN input_idsrrfpast_key_valuesrrroutput_hidden_states return_dictrxc  Cs0|dk r|dk rtdn6|dk r0t|\} } n |dk rHt|\} } } ntd| } d}|dk r|t|ddd}| |} |dkrtj|| |tjd}tt|dd| f}ntt|d| ftj}|dkrt||j j | |}|dkr tj | | ftj d}||| | f||}|}|r.dnd}|rgsz*TFMistralMainLayer.call..)Zlast_hidden_staterrJ attentions)rrrr"r[rzr&r1rrnrrrAr2r enumeraterprtupler)r;rrrfrrrrrrZ batch_sizeZ seq_lengthrZseq_length_with_pastr(rJZall_hidden_statesZall_self_attnsZnext_decoder_cacheidxZ decoder_layerrZ layer_outputsZ next_cacher,r,r-rKs       zTFMistralMainLayer.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk r|t|jj|jdW5QRXt|dddk r|jD]&}t|j|dW5QRXqdS)NTrrrp) rDrurrvrrBrGrrp)r;rFZlayerr,r,r-rGos zTFMistralMainLayer.build) NNNNNNNNN)N)rLrMrNrr config_classr8rrrrrrrrr2rrrrKrGrOr,r,r>r-rs:  ^rax This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. TensorFlow models and layers in `model` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`MistralConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. zUThe bare Mistral Model outputting raw hidden-states without any specific head on top.c@seZdZeZdZdS)TFMistralPreTrainedModelmodelN)rLrMrNrrZbase_model_prefixr,r,r,r-rsra1 Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) past_key_values (`Cache` or `tuple(tuple(tf.Tensor))`, *optional*): Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. Two formats are allowed: - a [`~cache_utils.Cache`] instance; - Tuple of `tuple(tf.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy cache format. The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the legacy cache format will be returned. If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cseZdZedfdd Zeeed ej e ej e ej e e ej e ej e e e e e e e e e eefd ddZd dd ZZS) TFMistralModelrcs&tj|f||t|dd|_dS)Nrr)r7r8rrr;rninputsr=r>r,r-r8szTFMistralModel.__init__Nrc Cs |j||||||||| d } | S)N rrrfrrrrrrr) r;rrrfrrrrrrrr,r,r-rKs zTFMistralModel.callc CsJ|jr dSd|_t|dddk rFt|jj|jdW5QRXdS)NTr)rDrurrvrrBrGrEr,r,r-rGs zTFMistralModel.build) NNNNNNNNN)N)rLrMrNrr8rrMISTRAL_INPUTS_DOCSTRINGrrrrr2rrrrKrGrOr,r,r>r-rs2  rcseZdZfddZddZddZddZd d Zd d Zd dZ e e e ddejeejeejeeejeejeejeeeeeeeeeeefd ddZdddZdddZZS)TFMistralForCausalLMcsRtj|f||t|dd|_|j|_tjj|jdt|j dd|_ ||_ dS)NrrFlm_headrkZkernel_initializerrB) r7r8rrrrrprqr initializer_rangerrnrr>r,r-r8'szTFMistralForCausalLM.__init__cCs|jjSrtrrrr,r,r-r3sz)TFMistralForCausalLM.get_input_embeddingscCs ||j_dSrtrrr,r,r-r6sz)TFMistralForCausalLM.set_input_embeddingscCs|jSrtrrr,r,r-get_output_embeddings9sz*TFMistralForCausalLM.get_output_embeddingscCs ||_dSrtr)r;Znew_embeddingsr,r,r-set_output_embeddings<sz*TFMistralForCausalLM.set_output_embeddingscCs ||_dSrtr)r;decoderr,r,r- set_decoder?sz TFMistralForCausalLM.set_decodercCs|jSrtrrr,r,r- get_decoderBsz TFMistralForCausalLM.get_decoderbatch_size, sequence_lengthN rrrfrrlabelsrrrrrxc  Cs|j|||||||| | d } | d} || } t| tj} d}|dk r|| ddddf}|ddddf}|||}| s| f| dd}|dk r|f|S|St|| | j| j| j dS)a Args: labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. rrNrrlosslogitsrrJr) rrrr1r3hf_compute_lossrrrJr)r;rrrfrrrrrrrrrJrrZshifted_logitsoutputr,r,r-rKEs:   zTFMistralForCausalLM.callcKs||rt|dddfd}|dd}|dk rf|dkrftjj|ddd}|rft|dddfd}|||||ddS)NrrfT)rZ exclusiver)rrrfrr)rr&getrZcumsum)r;rrrrr=rfr,r,r-prepare_inputs_for_generations z2TFMistralForCausalLM.prepare_inputs_for_generationc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk rt|jj|j|jj fW5QRXdS)NTrr) rDrurrvrrBrGrrnr9rEr,r,r-rGszTFMistralForCausalLM.build) NNNNNNNNNN)NNN)N)rLrMrNr8rrrrrrrrrformatrrrrr2rrrrKrrGrOr,r,r>r-r&sJ    : ra The Mistral Model transformer with a sequence classification head on top (linear layer). [`MistralForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). cseZdZfddZddZddZeee dde j e e j e e j e e e j e e j e e j e ee ee ee eeeefd d d Zdd d ZZS)"TFMistralForSequenceClassificationcsRtj|f|||j|_t|dd|_tjj|jdt|j dd|_ ||_ dS)NrrFscorer) r7r8 num_labelsrrrrprqr rrrnrr>r,r-r8sz+TFMistralForSequenceClassification.__init__cCs|jjSrtrrr,r,r-rsz7TFMistralForSequenceClassification.get_input_embeddingscCs ||j_dSrtrrr,r,r-rsz7TFMistralForSequenceClassification.set_input_embeddingsrNrc  Cs|j|||||||| | d } | d} || } t| }d}|jjdkrLd}n|dk rtjttj ||jj|j ddd}t |dk|tt|d|j d}tj | |ddd}nd}t |jjdd}|dk rH|jjdkr|ddkrtd t|s$| d|d|f}|t|dgt|d|jg}|dk rV|n| }| s|f| dd}|dk r|f|S|St||| j| j| jd S) a Args: labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. rrNrrr)Z batch_dimsrz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`z=Cannot handle batch sizes > 1 if no padding token is defined.r)rrrrnrrZargmaxr1requalrr#rcrrr?rLrZ is_tensorrrzrr rrJr)r;rrrfrrrrrrrZtransformer_outputsrJrZ logits_shapeZ in_logitsZsequence_lengthsrZ pooled_logitsrr,r,r-rKsd   $   $z'TFMistralForSequenceClassification.callc Cs|jr dSd|_t|dddk rFt|jj|jdW5QRXt|dddk rt|jj|j|jj fW5QRXdS)NTrr) rDrurrvrrBrGrrnr9rEr,r,r-rGsz(TFMistralForSequenceClassification.build) NNNNNNNNNN)N)rLrMrNr8rrrrrrrrrrr2rrr rKrGrOr,r,r>r-rs:    Qr)r)N)r):rrrtypingrrrrZ tensorflowrZmodeling_tf_outputsrrr Zmodeling_tf_utilsr r r r rrrrZtf_utilsrrrutilsrrrZconfiguration_mistralrZ get_loggerrLrZ_CONFIG_FOR_DOCr.r4rpZLayerr5rPrbrgrhrrrrrrZMISTRAL_START_DOCSTRINGrrrrrr,r,r,r-sV(       Z(+F){