U 5AfT@sdZddlZddlmZmZmZmZddlZddlZddlm Z ddl m Z m Z m Z ddlmZmZddlmZmZmZmZmZmZmZmZdd lmZdd lmZmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%d d l&m'Z'e$(e)Z*dZ+dZ,Gddde j-Z.Gddde j-Z/Gddde j-Z0de/iZ1Gddde j-Z2Gddde j-Z3Gddde j-Z4Gddde j-Z5Gdd d e j-Z6Gd!d"d"e j-Z7Gd#d$d$eZ8d%Z9d&Z:e"d'e9Gd(d)d)e8Z;e"d*e9Gd+d,d,e8Ze"d2e9Gd3d4d4e8Z?e"d5e9Gd6d7d7e8Z@e"d8e9Gd9d:d:e8ZAGd;d<dd?d?e8ZCdBd@dAZDdS)CzPyTorch RoBERTa model.N)ListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FNgelu))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) RobertaConfigzFacebookAI/roberta-basercs2eZdZdZfddZd ddZdd ZZS) RobertaEmbeddingszV Same as BertEmbeddings with a tiny tweak for positional embeddings indexing. csttj|j|j|jd|_t|j|j|_ t|j |j|_ tj |j|j d|_ t|j|_t|dd|_|jdt|jddd|jd tj|jtjd dd|j|_tj|j|j|jd|_ dS) N) padding_idxZepsposition_embedding_typeabsolute position_ids)rF) persistenttoken_type_idsdtype)super__init__r Embedding vocab_size hidden_sizeZ pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsZtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutgetattrr#Zregister_buffertorcharangeexpandzerosr%sizelongr!selfconfig __class__P/tmp/pip-unpacked-wheel-zw5xktn0/transformers/models/roberta/modeling_roberta.pyr,=s. zRobertaEmbeddings.__init__Nrc Cs|dkr*|dk r t||j|}n ||}|dk r<|}n|dd}|d}|dkrt|dr|jddd|f}||d|} | }ntj|tj |j j d}|dkr| |}| |} || } |jdkr||} | | 7} || } || } | S)Nr&rr(rr*devicer$)"create_position_ids_from_input_idsr!&create_position_ids_from_inputs_embedsr>hasattrr(r<r:r=r?r%rHr0r3r#r2r4r8) rA input_idsr(r% inputs_embedspast_key_values_length input_shape seq_lengthbuffered_token_type_ids buffered_token_type_ids_expandedr3 embeddingsr2rErErFforwardVs0         zRobertaEmbeddings.forwardcCsN|dd}|d}tj|jd||jdtj|jd}|d|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr&rrGr)r>r:r;r!r?rHZ unsqueezer<)rArMrOZsequence_lengthr%rErErFrJ~s  z8RobertaEmbeddings.create_position_ids_from_inputs_embeds)NNNNr)__name__ __module__ __qualname____doc__r,rTrJ __classcell__rErErCrFr 7s  (r c seZdZd fdd ZejejdddZd ejeejeejeejeejee e ejee e ejdd d Z Z S) RobertaSelfAttentionNcst|j|jdkr>t|ds>td|jd|jd|j|_t|j|j|_|j|j|_t |j|j|_ t |j|j|_ t |j|j|_ t |j|_|pt|dd|_|jdks|jd kr|j|_t d |jd |j|_|j|_dS) NrZembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r#r$ relative_keyrelative_key_queryr)r+r,r/num_attention_headsrK ValueErrorintattention_head_size all_head_sizerLinearquerykeyvaluer6Zattention_probs_dropout_probr8r9r#r1r-distance_embedding is_decoderrArBr#rCrErFr,s* zRobertaSelfAttention.__init__)xreturncCs6|dd|j|jf}||}|ddddS)Nr&rr^rr )r>r_rbviewpermute)rArkZ new_x_shaperErErFtranspose_for_scoress z)RobertaSelfAttention.transpose_for_scoresF hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsrlcCs||}|dk } | r4|dk r4|d} |d} |}n| r^|||} |||} |}nv|dk r|||} |||} tj|d| gdd} tj|d| gdd} n |||} |||} ||} |dk } |jr| | f}t| | dd}|j dks |j dkr| j d| j d}}| r^tj |dtj |j d dd}ntj|tj |j d dd}tj|tj |j d dd}||}|||jd}|j| jd }|j dkrtd | |}||}n4|j dkrtd | |}td | |}|||}|t|j}|dk r:||}tjj|dd}||}|dk rf||}t|| }|dddd }|dd|jf}||}|r||fn|f}|jr||f}|S)Nrrr^Zdimr&r\r]rGr)zbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rerorfrgr:catrimatmulZ transposer#shapeZtensorr?rHrmr;rhr1tor*ZeinsummathsqrtrbrZ functionalZsoftmaxr8rn contiguousr>rc)rArqrrrsrtrurvrwZmixed_query_layerZis_cross_attentionZ key_layerZ value_layerZ query_layer use_cacheZattention_scoresZ query_lengthZ key_lengthZposition_ids_lZposition_ids_rZdistanceZpositional_embeddingZrelative_position_scoresZrelative_position_scores_queryZrelative_position_scores_keyZattention_probsZ context_layerZnew_context_layer_shapeoutputsrErErFrTsp             zRobertaSelfAttention.forward)N)NNNNNF) rUrVrWr,r:Tensorror FloatTensorrboolrTrYrErErCrFrZs$rZcs4eZdZfddZejejejdddZZS)RobertaSelfOutputcsBtt|j|j|_tj|j|jd|_t|j |_ dSNr") r+r,rrdr/denser4r5r6r7r8r@rCrErFr,s zRobertaSelfOutput.__init__rq input_tensorrlcCs&||}||}|||}|SNrr8r4rArqrrErErFrTs  zRobertaSelfOutput.forwardrUrVrWr,r:rrTrYrErErCrFrs reagerc sveZdZd fdd ZddZd ejeejeejeejeejee e ejee e ejddd Z Z S) RobertaAttentionNcs4tt|j||d|_t||_t|_dS)Nr#) r+r,ROBERTA_SELF_ATTENTION_CLASSESZ_attn_implementationrAroutputset pruned_headsrjrCrErFr,-s  zRobertaAttention.__init__cCst|dkrdSt||jj|jj|j\}}t|jj||j_t|jj||j_t|jj ||j_ t|j j |dd|j _ |jjt||j_|jj|jj|j_ |j ||_dS)Nrrrx)lenrrAr_rbrrrerfrgrrrcunion)rAheadsindexrErErF prune_heads5s zRobertaAttention.prune_headsFrpc Cs<||||||||}||d|} | f|dd} | S)Nrr)rAr) rArqrrrsrtrurvrwZ self_outputsattention_outputrrErErFrTGs  zRobertaAttention.forward)N)NNNNNF) rUrVrWr,rr:rrrrrrTrYrErErCrFr,s$rcs0eZdZfddZejejdddZZS)RobertaIntermediatecsBtt|j|j|_t|jt r6t |j|_ n|j|_ dSr) r+r,rrdr/intermediate_sizer isinstanceZ hidden_actstrr intermediate_act_fnr@rCrErFr,as   zRobertaIntermediate.__init__rqrlcCs||}||}|Sr)rr)rArqrErErFrTis  zRobertaIntermediate.forwardrrErErCrFr`s rcs4eZdZfddZejejejdddZZS) RobertaOutputcsBtt|j|j|_tj|j|jd|_t |j |_ dSr) r+r,rrdrr/rr4r5r6r7r8r@rCrErFr,qs zRobertaOutput.__init__rcCs&||}||}|||}|SrrrrErErFrTws  zRobertaOutput.forwardrrErErCrFrps rc steZdZfddZd ejeejeejeejeejeeeejee eejdddZ dd Z Z S) RobertaLayercsrt|j|_d|_t||_|j|_|j|_|jrZ|jsLt|dt|dd|_ t ||_ t ||_ dS)Nrz> should be used as a decoder model if cross attention is addedr$r)r+r,chunk_size_feed_forward seq_len_dimr attentionriadd_cross_attentionr`crossattentionr intermediaterrr@rCrErFr,s   zRobertaLayer.__init__NFrpc Cs|dk r|ddnd}|j|||||d} | d} |jrP| dd} | d} n | dd} d} |jr|dk rt|dstd|d|dk r|d dnd}|| ||||||}|d} | |dd} |d} | | } t|j|j|j| }|f| } |jr| | f} | S) Nr^)rwrvrrr&rz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`ry) rrirKr`rrfeed_forward_chunkrr)rArqrrrsrtrurvrwZself_attn_past_key_valueZself_attention_outputsrrZpresent_key_valueZcross_attn_present_key_valueZcross_attn_past_key_valueZcross_attention_outputs layer_outputrErErFrTsV         zRobertaLayer.forwardcCs||}|||}|Sr)rr)rArZintermediate_outputrrErErFrs  zRobertaLayer.feed_forward_chunk)NNNNNF) rUrVrWr,r:rrrrrrTrrYrErErCrFrs$  ArcseZdZfddZd ejeejeejeejeejeeeejee ee ee ee e eeje fd ddZ Z S) RobertaEncodercs:t|_tfddtjD|_d|_dS)Ncsg|] }tqSrE)r).0_rBrErF sz+RobertaEncoder.__init__..F) r+r,rBrZ ModuleListrangenum_hidden_layerslayergradient_checkpointingr@rCrrFr,s  zRobertaEncoder.__init__NFT) rqrrrsrtrupast_key_valuesrrwoutput_hidden_states return_dictrlc  Csb| rdnd} |rdnd} |r(|jjr(dnd} |jrJ|jrJ|rJtdd}|rRdnd}t|jD]\}}| rv| |f} |dk r||nd}|dk r||nd}|jr|jr||j |||||||}n||||||||}|d}|r||df7}|r`| |df} |jjr`| |df} q`| r.| |f} | sPt dd ||| | | fDSt ||| | | d S) NrEzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frr&rr^css|]}|dk r|VqdSrrE)rvrErErF sz)RobertaEncoder.forward..)last_hidden_staterrq attentionscross_attentions) rBrrZtrainingloggerZ warning_once enumeraterZ_gradient_checkpointing_func__call__tupler )rArqrrrsrtrurrrwrrZall_hidden_statesZall_self_attentionsZall_cross_attentionsZnext_decoder_cacheiZ layer_moduleZlayer_head_maskrvZ layer_outputsrErErFrTsx          zRobertaEncoder.forward) NNNNNNFFT)rUrVrWr,r:rrrrrrr rTrYrErErCrFrs. rcs0eZdZfddZejejdddZZS) RobertaPoolercs*tt|j|j|_t|_dSr)r+r,rrdr/rZTanh activationr@rCrErFr,5s zRobertaPooler.__init__rcCs(|dddf}||}||}|SNr)rr)rArqZfirst_token_tensor pooled_outputrErErFrT:s  zRobertaPooler.forwardrrErErCrFr4s rc@s,eZdZdZeZdZdZddgZddZ dS) RobertaPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. robertaTr rZcCst|tjr:|jjjd|jjd|jdk r|jj nft|tj rz|jjjd|jjd|j dk r|jj|j  n&t|tj r|jj |jj ddS)zInitialize the weightsg)ZmeanZstdNg?)rrrdZweightdataZnormal_rBZinitializer_rangebiasZzero_r-r!r4Zfill_)rAmodulerErErF _init_weightsOs      z$RobertaPreTrainedModel._init_weightsN) rUrVrWrXr config_classZbase_model_prefixZsupports_gradient_checkpointingZ_no_split_modulesrrErErErFrCs raA This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`RobertaConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. This parameter can only be used when the model is initialized with `type_vocab_size` parameter with value >= 2. All the value in this tensor should be always < type_vocab_size. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zaThe bare RoBERTa Model transformer outputting raw hidden-states without any specific head on top.cseZdZdZdfdd ZddZddZd d Zee d e e e ed deejeejeejeejeejeejeejeejeeejeeeeeeeeeeeje fdddZZS) RobertaModela* The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in *Attention is all you need*_ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass. .. _*Attention is all you need*: https://arxiv.org/abs/1706.03762 TcsDt|||_t||_t||_|r2t|nd|_| dSr) r+r,rBr rSrencoderrpooler post_init)rArBadd_pooling_layerrCrErFr,s    zRobertaModel.__init__cCs|jjSrrSr0rArErErFget_input_embeddingssz!RobertaModel.get_input_embeddingscCs ||j_dSrr)rArgrErErFset_input_embeddingssz!RobertaModel.set_input_embeddingscCs*|D]\}}|jj|j|qdS)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrrrr)rAZheads_to_prunerrrErErF _prune_headsszRobertaModel._prune_headsbatch_size, sequence_length checkpoint output_typerN)rLrrr(r%rsrMrtrurrrwrrrlc Cs^| dk r | n|jj} | dk r | n|jj} | dk r4| n|jj} |jjrZ| dk rP| n|jj} nd} |dk rx|dk rxtdn@|dk r||||}n"|dk r|dd}ntd|\}}|dk r|j n|j }| dk r| ddj dnd}|dkrt j |||f|d}|dkrft |jd rT|jjddd|f}|||}|}nt j|t j|d }|||}|jjr|dk r|\}}}||f}|dkrt j ||d}||}nd}|||jj}|j|||||d }|j|||||| | | | | d }|d}|jdk r$||nd}| sB||f|d dSt|||j|j|j|jdS)a  encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). NFzDYou cannot specify both input_ids and inputs_embeds at the same timer&z5You have to specify either input_ids or inputs_embedsrr^)rHr(rG)rLr%r(rMrN) rrrsrtrurrrwrrr)rZ pooler_outputrrqrr)rBrwruse_return_dictrirr`Z%warn_if_padding_and_no_attention_maskr>rHr|r:ZonesrKrSr(r<r=r?Zget_extended_attention_maskZinvert_attention_maskZ get_head_maskrrrrrrqrr)rArLrrr(r%rsrMrtrurrrwrrrOZ batch_sizerPrHrNrQrRZextended_attention_maskZencoder_batch_sizeZencoder_sequence_lengthrZencoder_hidden_shapeZencoder_extended_attention_maskZembedding_outputZencoder_outputssequence_outputrrErErFrTs+          zRobertaModel.forward)T) NNNNNNNNNNNNN)rUrVrWrXr,rrrrROBERTA_INPUTS_DOCSTRINGformatr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOCrr:rrrrrrrTrYrErErCrFrsP   rzIRoBERTa Model with a `language modeling` head on top for CLM fine-tuning.cseZdZddgZfddZddZddZee d e e e d de eje eje eje eje eje eje eje eje ejeeeje ee ee ee eeeeje fd d dZdddZddZZS)RobertaForCausalLMlm_head.decoder.weightlm_head.decoder.biascs@t||jstdt|dd|_t||_| dS)NzOIf you want to use `RobertaLMHeadModel` as a standalone, add `is_decoder=True.`Fr r+r,rirwarningrr RobertaLMHeadlm_headrr@rCrErFr,bs    zRobertaForCausalLM.__init__cCs|jjSrrdecoderrrErErFget_output_embeddingsnsz(RobertaForCausalLM.get_output_embeddingscCs ||j_dSrrrAZnew_embeddingsrErErFset_output_embeddingsqsz(RobertaForCausalLM.set_output_embeddingsr)rrN)rLrrr(r%rsrMrtrulabelsrrrwrrrlcCs|dk r |n|jj}| dk r d} |j||||||||| | | | |d }|d}||}d}| dk r| |j} |ddddddf}| ddddf} t}||d|jj | d}|s|f|dd}|dk r|f|S|St |||j |j |j |jdS) a2 encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is configured as a decoder. encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). Returns: Example: ```python >>> from transformers import AutoTokenizer, RobertaForCausalLM, AutoConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("FacebookAI/roberta-base") >>> config = AutoConfig.from_pretrained("FacebookAI/roberta-base") >>> config.is_decoder = True >>> model = RobertaForCausalLM.from_pretrained("FacebookAI/roberta-base", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NF) rrr(r%rsrMrtrurrrwrrrr&rr^)losslogitsrrqrr)rBrrrr}rHrrrmr.rrrqrr)rArLrrr(r%rsrMrtrurrrrwrrrrprediction_scoresZlm_lossZshifted_prediction_scoresloss_fctrrErErFrTtsL>  zRobertaForCausalLM.forwardcKst|j}|dkr||}|dk rh|ddjd}|jd|krF|}n|jdd}|dd|df}|||dS)Nrr^r)rLrrr)r|Znew_ones)rArLrrrZ model_kwargsrOZ past_lengthZremove_prefix_lengthrErErFprepare_inputs_for_generations z0RobertaForCausalLM.prepare_inputs_for_generationcs.d}|D] }|tfdd|Df7}q|S)NrEc3s"|]}|d|jVqdS)rN)Z index_selectr}rH)rZ past_statebeam_idxrErFrsz4RobertaForCausalLM._reorder_cache..)r)rArrZreordered_pastZ layer_pastrErrF_reorder_caches z!RobertaForCausalLM._reorder_cache)NNNNNNNNNNNNNN)NN)rUrVrW_tied_weights_keysr,rrrrrrrrrr: LongTensorrrrrrrTrrrYrErErCrFr\sN    j rz5RoBERTa Model with a `language modeling` head on top.cseZdZddgZfddZddZddZee d e e e e d d d d deejeejeejeejeejeejeejeejeejeeeeeeeeeje fd ddZZS)RobertaForMaskedLMrrcs@t||jrtdt|dd|_t||_| dS)NznIf you want to use `RobertaForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Frrr@rCrErFr,s  zRobertaForMaskedLM.__init__cCs|jjSrrrrErErFrsz(RobertaForMaskedLM.get_output_embeddingscCs ||j_dSrrrrErErFrsz(RobertaForMaskedLM.set_output_embeddingsrzz' Paris'g?)rrrmaskexpected_output expected_lossN) rLrrr(r%rsrMrtrurrwrrrlc  Cs| dk r | n|jj} |j||||||||| | | d } | d}||}d}| dk r| |j} t}||d|jj| d}| s|f| dd}|dk r|f|S|St ||| j | j dS)a( labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` kwargs (`Dict[str, any]`, *optional*, defaults to `{}`): Used to hide legacy arguments that have been deprecated. N) rrr(r%rsrMrtrurwrrrr&r^rrrqr) rBrrrr}rHrrmr.rrqr)rArLrrr(r%rsrMrtrurrwrrrrrZmasked_lm_lossrrrErErFrTs<    zRobertaForMaskedLM.forward) NNNNNNNNNNNN)rUrVrWrr,rrrrrrrrrrr:rrrrrrrTrYrErErCrFrsP   rcs0eZdZdZfddZddZddZZS)rz*Roberta Head for masked language modeling.csdtt|j|j|_tj|j|jd|_t|j|j |_ t t |j |_|j|j _dSr)r+r,rrdr/rr4r5 layer_normr.r Parameterr:r=rr@rCrErFr,_s  zRobertaLMHead.__init__cKs*||}t|}||}||}|Sr)rr rrrAfeatureskwargsrkrErErFrThs    zRobertaLMHead.forwardcCs*|jjjjdkr|j|j_n |jj|_dS)Nmeta)rrrHtyperrErErF _tie_weightsrs zRobertaLMHead._tie_weights)rUrVrWrXr,rTrrYrErErCrFr\s  rz RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. cseZdZfddZeededee dddd e e j e e j e e j e e j e e j e e j e e j e ee ee eeee jefd d d ZZS) RobertaForSequenceClassificationcs>t||j|_||_t|dd|_t||_|dSNFr) r+r, num_labelsrBrrRobertaClassificationHead classifierrr@rCrErFr,s   z)RobertaForSequenceClassification.__init__rz'cardiffnlp/twitter-roberta-base-emotionz 'optimism'g{Gz?rrrrrN rLrrr(r%rsrMrrwrrrlc  Cs~| dk r | n|jj} |j|||||||| | d } | d} || } d}|dk r:|| j}|jjdkr|jdkr|d|j_n4|jdkr|jt j ks|jt j krd|j_nd|j_|jjdkrt }|jdkr|| | }n || |}nN|jjdkrt}|| d|j|d}n|jjdkr:t}|| |}| sj| f| d d}|dk rf|f|S|St|| | j| jd S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nrrr(r%rsrMrwrrrrZ regressionZsingle_label_classificationZmulti_label_classificationr&r^r)rBrrr r}rHZ problem_typer r*r:r?rar squeezerrmrrrqrrArLrrr(r%rsrMrrwrrrrrrrrrErErFrTsV       "     z(RobertaForSequenceClassification.forward) NNNNNNNNNN)rUrVrWr,rrrrrrrr:rrrrrrrTrYrErErCrFr{s@  rz Roberta Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RocStories/SWAG tasks. cseZdZfddZeedeee e dd e e j e e j e e je e j e e j e e je e je ee ee eeee je fd ddZZS) RobertaForMultipleChoicecs@t|t||_t|j|_t|j d|_ | dS)Nr) r+r,rrrr6r7r8rdr/r rr@rCrErFr,s   z!RobertaForMultipleChoice.__init__z(batch_size, num_choices, sequence_lengthrN) rLr(rrrr%rsrMrwrrrlc  Cs| dk r | n|jj} |dk r&|jdn|jd} |dk rJ|d|dnd} |dk rh|d|dnd} |dk r|d|dnd}|dk r|d|dnd}|dk r|d|d|dnd}|j| | |||||| | d }|d}||}||}|d| }d}|dk r<||j }t }|||}| sl|f|dd}|dk rh|f|S|St |||j |j dS)aJ labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nrr&ry)r%r(rrrsrMrwrrr^r)rBrr|rmr>rr8r r}rHrrrqr)rArLr(rrrr%rsrMrwrrZ num_choicesZflat_input_idsZflat_position_idsZflat_token_type_idsZflat_attention_maskZflat_inputs_embedsrrrZreshaped_logitsrrrrErErFrTsN       z RobertaForMultipleChoice.forward) NNNNNNNNNN)rUrVrWr,rrrrrrrrr:rrrrrrrTrYrErErCrFrs< rz Roberta Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. cseZdZfddZeededee dddd e e j e e j e e j e e j e e j e e j e e j e ee ee eeee jefd d d ZZS) RobertaForTokenClassificationcsbt||j|_t|dd|_|jdk r2|jn|j}t||_ t |j |j|_ | dSr)r+r,r rrclassifier_dropoutr7rr6r8rdr/r rrArBrrCrErFr,Bs  z&RobertaForTokenClassification.__init__rz'Jean-Baptiste/roberta-large-ner-englishzF['O', 'ORG', 'ORG', 'O', 'O', 'O', 'O', 'O', 'LOC', 'O', 'LOC', 'LOC']g{Gz?r Nr c  Cs| dk r | n|jj} |j|||||||| | d } | d} || } || } d}|dk r|| j}t}|| d|j |d}| s| f| dd}|dk r|f|S|St || | j | j dS)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Nrrr&r^r) rBrrr8r r}rHrrmr rrqrrrErErFrTPs:    z%RobertaForTokenClassification.forward) NNNNNNNNNN)rUrVrWr,rrrrrrrr:rrrrrrrTrYrErErCrFr:s@   rcs(eZdZdZfddZddZZS)r z-Head for sentence-level classification tasks.csTtt|j|j|_|jdk r,|jn|j}t||_ t|j|j |_ dSr) r+r,rrdr/rrr7r6r8r out_projrrCrErFr,s   z"RobertaClassificationHead.__init__cKsL|dddddf}||}||}t|}||}||}|Sr)r8rr:tanhrrrErErFrTs     z!RobertaClassificationHead.forward)rUrVrWrXr,rTrYrErErCrFr s r z Roberta Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). cseZdZfddZeededee dddd e e j e e j e e j e e j e e j e e j e e j e e j e ee ee eeee jefd d d ZZS) RobertaForQuestionAnsweringcs@t||j|_t|dd|_t|j|j|_| dSr) r+r,r rrrrdr/ qa_outputsrr@rCrErFr,s  z$RobertaForQuestionAnswering.__init__rzdeepset/roberta-base-squad2z ' puppet'gQ?r N) rLrrr(r%rsrMstart_positions end_positionsrwrrrlc  CsP| dk r | n|jj} |j||||||| | | d } | d} || }|jddd\}}|d}|d}d}|dk r|dk rt|dkr|d}t|dkr|d}|d}| d|}| d|}t |d}|||}|||}||d}| s:||f| dd}|dk r6|f|S|St |||| j | j d S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Nrrrr&rx)Z ignore_indexr^)r start_logits end_logitsrqr)rBrrrsplitrrrr>clamprrrqr)rArLrrr(r%rsrMrrrwrrrrrrrZ total_lossZ ignored_indexrZ start_lossZend_lossrrErErFrTsP            z#RobertaForQuestionAnswering.forward) NNNNNNNNNNN)rUrVrWr,rrrrrrrr:rrrrrrrTrYrErErCrFrsD  rcCs6||}tj|dd|||}||S)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rrx)nerar:ZcumsumZtype_asr?)rLr!rNrZincremental_indicesrErErFrI s rI)r)ErXr~typingrrrrr:Ztorch.utils.checkpointrZtorch.nnrrr Z activationsr r Zmodeling_outputsr rrrrrrrZmodeling_utilsrZ pytorch_utilsrrrutilsrrrrrZconfiguration_robertarZ get_loggerrUrrrModuler rZrrrrrrrrrZROBERTA_START_DOCSTRINGrrrrrrrrr rrIrErErErFs (   Z4W^36 ]\UN^