U
    5AfF                  V   @   s  U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZmZmZmZmZmZmZmZmZ d dlZd dlm  mZ d dlmZ d dlmZmZmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z& dd	l'm(Z( dd
l)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z?m@Z@mAZAmBZBmCZCmDZDmEZEmFZFmGZG ddlHmIZI ddlJmKZKmLZLmMZMmNZNmOZO eN rd dlPmQZQ e(ReSZTejUVddW eKkZXdee/ eeeYeeY f  eeY dddZZdddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGg1Z[d,d8gZ\g Z]e[D ]2Z^e_e^e`re]aeZf e^ ne]aeZe^ qdHdIdJdKdLdMdNdOdPdQdRdSgZbecedeee]eb ZfdagdTdU ZhddXdYZidZd[ Zjd\d] Zkd^d_ Zld`da Zmdbdc ZnddddeZodfdg ZpddhdidjZqdkdl Zrdmdn ZsdddhdodpZtdddhdqdrZudddsdtduZvddhdvdwZwdxdy Zxddhdzd{Zyddhd|d}Zzdddd~ddZ{dddd~ddZ|dd Z}dd Z~dddddZddhddZdd ZdWddddZdd ZdddZdd Zdd Zdd Zdd ZdddZdddZdd Zdd Zdd ZdddZdddZdd Zdd Zdd Zdd Zejjehejjjeiejjejejjekejjelejemejjjeoejjenejepejeqejerejesejetejeuejevejewejjexejeyejezeje{ejje|eje}ejje~ejeejeejeejjeejeejjeejeejjeejjeejjeejeejjeejeejjeejeejjjeejjeejjeejjeejei+Zeeef ed< eIreeejjj< G dd deZG dd deZG dd deZG dd deZdeeed ed ed f eeegef  edddÄZG ddń deZeeeef dƜddȄZddʄ Zee* eegef d˜dd̈́Zede*fi ee*dύZede+fi ee+dύZede,fi ee,dύZede-fi ee-dύZdeeeee  d՜ddׄZG ddل de ZejeeY dڜdd܄ZddޜddZddޜddZddWefdeeeY  eee edddZdS )    N)	AnyCallableDictListLiteralOptionalTupleTypeUnion)nn)GraphGraphModuleNodeProxyTracer)compatibility)is_fx_tracingParameterProxy   )logging)CacheDynamicCache	SinkCacheStaticCache)PretrainedConfigPreTrainedModel)
get_values),MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES MODEL_FOR_BACKBONE_MAPPING_NAMES!MODEL_FOR_CAUSAL_LM_MAPPING_NAMESMODEL_FOR_CTC_MAPPING_NAMES3MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMESMODEL_FOR_IMAGE_MAPPING_NAMES-MODEL_FOR_MASKED_IMAGE_MODELING_MAPPING_NAMES!MODEL_FOR_MASKED_LM_MAPPING_NAMES'MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES0MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES#MODEL_FOR_PRETRAINING_MAPPING_NAMES*MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES-MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES/MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES(MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES,MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES6MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMESMODEL_MAPPING_NAMES)"is_torch_greater_or_equal_than_2_0   )ENV_VARS_TRUE_VALUESTORCH_FX_REQUIRED_VERSIONget_torch_versionis_peft_availableis_torch_fx_available)	PeftModelZFX_DEBUG_MODE )
model_namesupported_tasksreturnc                 C   s~   t ttttttttt	t
tttttttttd}|d kr>| }t|trN|g}g }|D ]"}|| | d }|rV|| qV|S )N)defaultZpretrainingznext-sentence-predictionz	masked-lmz	causal-lmz
seq2seq-lmzspeech-seq2seqzmultiple-choicezdocument-question-answeringzquestion-answeringzsequence-classificationztoken-classificationzmasked-image-modelingzimage-classificationzzero-shot-image-classificationZctczaudio-classificationzsemantic-segmentationZbackbonezimage-feature-extraction)r1   r)   r(   r&   r    r,   r.   r'   r"   r*   r-   r/   r%   r#   r0   r!   r   r+   r   r$   keys
isinstancestrgetappend)r;   r<   Ztask_mappingZmodel_class_namesZtask
class_name rE   9/tmp/pip-unpacked-wheel-zw5xktn0/transformers/utils/fx.py%_generate_supported_model_class_namesQ   s>    
rG   ZaltclipZalbertZbartZbertZ
blenderbotzblenderbot-smallZbloomZclipZconvnextZdebertaz
deberta-v2Zdinov2Z
distilbertz
donut-swinZelectraZgpt2Zgpt_neoZgptjZhieraZhubertZlayoutlmZllamaZcohereZlxmertZm2m_100ZmarianZmbartmegatron-bertZmistralZmixtralZ
mobilebertZmt5ZnezhaoptZpegasusZplbartZqwen2Z	qwen2_moeZresnetZrobertaZ	segformerZspeech_to_textZspeech_to_text_2ZswinZt5ZtrocrZvitZxglmZwav2vec2ZCLIPTextModelZCLIPTextModelWithProjectionZCLIPVisionModelZCLIPVisionModelWithProjectionZAltCLIPTextModelZAltCLIPVisionModelZGitVisionModelGPT2DoubleHeadsModelZSpeech2Text2DecoderZTrOCRDecoderPeftModelForCausalLMPeftModelForSeq2SeqLMc                 C   s&   t j|j| jjd fd| jjdS Nmeta)devicedtype)torchemptyshapeweightrQ   selfinputrE   rE   rF   torch_nn_embedding   s    rY          @Fc                 C   s"   t j| j|jd fd|jdS rM   )rR   rS   rT   rQ   )rX   rU   Zpadding_idxZmax_normZ	norm_typeZscale_grad_by_freqsparserE   rE   rF   torch_nn_functional_embedding   s    r\   c                 C   s   |S NrE   rV   rE   rE   rF   torch_nn_layernorm   s    r^   c                 C   s   |S r]   rE   rV   rE   rE   rF   torch_nn_groupnorm   s    r_   c                 C   s    t j|jd d | jf ddS )NrN   rO   rP   )rR   rS   rT   Zout_featuresrV   rE   rE   rF   torch_nn_linear   s    ra   c                 C   s   | S r]   rE   xrE   rE   rF   
torch_relu   s    rd   c                 C   s   |S r]   rE   )rW   rc   rE   rE   rF   torch_nn_relu   s    re   c                 C   s   |st d| S )Nz>Don't support in-place functional.relu for MetaTensor analysis
ValueError)rc   ZinplacerE   rE   rF   torch_nn_functional_relu   s    rh   c                 C   s$   | j dd|j dd |j dd S NrO   r`   to)	conditionrc   yrE   rE   rF   torch_where   s    rn   outc                C   s   |d k	rt d| S )Nz2Don't support in-place abs for MetaTensor analysisrf   )rX   rp   rE   rE   rF   	torch_abs   s    rq   c                  O   s   t | }d}|dkr"d}| d }n|dkr4| \}}n
| \}}}t|trPt|}t|trbt|}t|trtt|}|d|}|d}tj|| | |ddS )Nr3   r   r   steprQ   rO   rQ   rP   )lenr@   floatintrB   rR   rS   )argskwargsnrr   startendrQ   rE   rE   rF   torch_arange   s"    






r|   c                  O   sN   t | } t| dkrd| d< nd|d< t|}|dd  tj| |ddiS )Nr3   Z
fill_valuerP   rO   )listrt   dictpoprR   full)rw   rx   Zkwargs_without_devicerE   rE   rF   
torch_full  s    
r   c                   s    d kr|d krd  d kr(|d k	r(|  dk r@| d      dd | D }t|d }t fdd|D }|d   |g | d d   }tj|ddS )	Nr   c                 S   s   g | ]
}|j qS rE   )rT   ).0trE   rE   rF   
<listcomp>%  s     ztorch_cat.<locals>.<listcomp>c                 3   s   | ]}|  V  qd S r]   rE   )r   rT   dimrE   rF   	<genexpr>'  s     ztorch_cat.<locals>.<genexpr>r3   rO   r`   )r   r}   sumrR   rS   )tensorsr   axisrp   ZshapesrT   Zconcatenated_dimZfinal_shaperE   r   rF   	torch_cat  s    "r   c                C   sp   |d kr|d krd}|d kr(|d k	r(|}|dk rD| d   d | }t| d j}||t|  tj|ddS Nr   r3   rO   r`   )r   r}   rT   insertrt   rR   rS   )r   r   r   rp   rT   rE   rE   rF   torch_stack,  s    r   )alpharp   c          	      C   s   t | tjstj|ddS t |tjs4tj| ddS t|  | }t| jdg||     }t|jdg||    }g }t|D ]}|	t|| ||  qtj
|ddS )NrO   r`   r3   )r@   rR   Tensor
empty_likemaxr   r}   rT   rangerC   rS   )	rX   otherr   rp   
max_lengthZinput_shapeZother_shaperT   irE   rE   rF   	torch_add8  s    r   c                C   s   t | ||dS )Nro   )r   )rX   r   rp   rE   rE   rF   	torch_mulF  s    r   c                 C   s
   t | |S r]   )r   )rW   r   rE   rE   rF   torch_tensor_mulJ  s    r   c          
      C   s  |   }|  }d }|dkr,|dkr,d }nL|dkrT|dkrT| d|df}n$|dkrt|dkrt|df}n|dkr|dkr| df}nt|   |  }t| j}t|j}|dkrdg| }|dkr|d dg||  t| j }dg||  t|j }g }t|D ]}	|t||	 ||	  q|d |d< |d |d< |dkrd|d |dkrx|d |d krtj	dddS tj
|d	diS )
Nr3   r   r   rN           rO   r`   rP   )r   sizer   r}   rT   rC   r   r   rR   tensorrS   )
rX   r   rp   d1Zd2rT   r   Zshape1Zshape2r   rE   rE   rF   torch_matmulN  s@    








r   c                C   s:   |d k	rt d| j\}}}|j\}}}tj|||ddS )Nz2Don't support in-place bmm for MetaTensor analysisrO   r`   )rg   rT   rR   rS   )rX   Zmat2rp   
batch_sizery   m_prE   rE   rF   	torch_bmmr  s
    r   betar   rp   c                C   s   |d k	rt dt||S )Nz6Don't support in-place baddbmm for MetaTensor analysis)rg   r   )rX   batch1batch2r   r   rp   rE   rE   rF   torch_baddbmmz  s    r   c                C   s   t | |||||dS )Nr   )r   )rW   r   r   r   r   rp   rE   rE   rF   torch_tensor_baddbmm  s    r   c                 G   s$   dd |D }t j| f| dS )Nc                 s   s   | ]}t j|d dV  qdS )cpur`   N)rR   r   )r   ZoperandrE   rE   rF   r     s     ztorch_einsum.<locals>.<genexpr>rO   )rR   einsumrk   )ZequationZoperandsZconcrete_operandsrE   rE   rF   torch_einsum  s    r   c                 G   s:   t | j}t|D ]\}}||  |9  < qtj|ddS ri   )r}   rT   	enumeraterR   rS   )rW   ZsizesrT   r   rc   rE   rE   rF   torch_tensor_repeat  s    
r   )r   output_sizec                 G   s   t |}|dkr,|d k	r|n
|d  g}nt|d j}| d krb|dkrT|d } nt|g}d} |d }t|tst|dkr||   t|9  < n|d k	r|n| || < tj|ddiS )Nr3   r   r   rP   rO   )	rt   r   r}   rT   r@   rv   rR   ZnumelrS   )r   r   rw   num_argsrT   ZrepeatsrE   rE   rF   torch_repeat_interleave  s    

r   c                C   s&   t | j}t|||< tj|ddiS NrP   rO   )r}   rT   rt   rR   rS   )rX   r   indexrp   rT   rE   rE   rF   torch_index_select  s    
r   c                 C   s   t | ||S r]   )r   rW   r   r   rE   rE   rF   torch_tensor_index_select  s    r   )sparse_gradrp   c                C   s(   t | j}|j| ||< tj|ddiS r   )r}   rT   rR   rS   )rX   r   r   r   rp   rT   rE   rE   rF   torch_gather  s    
r   c                 C   s   t | ||S r]   )r   r   rE   rE   rF   torch_tensor_gather  s    r   c                 C   s   | S r]   rE   )rX   ZshiftsdimsrE   rE   rF   
torch_roll  s    r   c                 C   s   | S r]   rE   )rX   r   rE   rE   rF   
torch_flip  s    r   c                 C   s   | S r]   rE   )rW   r   rE   rE   rF   torch_tensor_flip  s    r   c                 C   s   |j d }d }| j}|dkr d}|dkr2t|j }|d krt|j }t|d|d   | jd | jd d   d | jd  d }||d< | j|d< t	j
|d	d
S )NrN   validr   r   samer   r   r3   r   rO   r`   rT   paddingr}   mathfloorZdilationZkernel_sizeZstrideZout_channelsrR   rS   )rW   rX   Zl_inrT   r   Zl_outrE   rE   rF   torch_nn_conv1d  s    


8
r   c                 C   s   |j dd  \}}d }| j}|dkr(d}|dkr:t|j }|d krt|j }t|d|d   | jd | jd d   d | jd  d }t|d|d   | jd | jd d   d | jd  d }||g|dd < | j|d< t	j
|d	d
S )Nr   r   r   r   r   r   r3   rO   r`   r   )rW   rX   Zh_inZw_inrT   r   Zh_outZw_outrE   rE   rF   torch_nn_conv2d  s$    

88
r   c                 C   sr   t | j}|d k	r>|dk r&|  | }|| dkrd|| n&g }|D ]}|dkrTqF|| qF|}tj|ddS r   )r}   rT   r   r   rC   rR   rS   )rX   r   rT   Z	new_shapeZ	dim_valuerE   rE   rF   torch_squeeze  s    
r   c                 C   s
   t | |S r]   )r   rW   r   rE   rE   rF   torch_tensor_squeeze  s    r   c                 C   s<   t | j}|dk r"|  d | }||d tj|ddS r   )r}   rT   r   r   rR   rS   )rX   r   rT   rE   rE   rF   torch_unsqueeze  s
    
r   c                 C   s
   t | |S r]   )r   r   rE   rE   rF   torch_tensor_unsqueeze	  s    r   c                 K   sD   t jt j| ddf|}t|t jr.|dS tt|dd S d S )Nr   r`   rO   c                 S   s
   |  dS NrO   rj   rb   rE   rE   rF   <lambda>      z*torch_unique_consecutive.<locals>.<lambda>)rR   unique_consecutiveZ
zeros_liker@   r   rk   tuplemap)rX   rx   outputrE   rE   rF   torch_unique_consecutive  s    
r   rN   c                 C   s.   |dk rt dt| j|g }tj|ddS )Nr   zEDon't support automatic num_classes inference for MetaTensor analysisrO   r`   )rg   r}   rT   rR   rS   )r   Znum_classesrT   rE   rE   rF   torch_nn_functional_one_hot  s    r   r   c           	      C   s4   | j d }|j d }tj| j d d ||fddS )Nr   rN   rO   r`   )rT   rR   rS   )	querykeyvalueZ	attn_maskZ	dropout_pZ	is_causalZscaleZtarget_lengthhead_dimrE   rE   rF   0torch_nn_functional_scaled_dot_product_attention  s    

r   c                 C   s$   | j dkr|j}nd}tj|ddS Nnone)r3   rO   r`   Z	reductionrT   rR   rS   rW   rX   targetrT   rE   rE   rF   torch_nn_mseloss$  s    
r   c                 C   s$   | j dkr|j}nd}tj|ddS r   r   r   rE   rE   rF   torch_nn_crossentropyloss,  s    
r   c                 C   s$   | j dkr|j}nd}tj|ddS r   r   r   rE   rE   rF   torch_nn_bcewithlogitsloss4  s    
r   c                 C   s^   dd }t | tjrRt |tr.tt||}n||}ttj| dd|dS t| |S )Nc                 S   sH   t | tjrDtj| dd}|jtjtjtjtjfkr@|	tj
}|S | S )Nr   r`   )r@   rR   r   Z	ones_likerQ   Zfloat16float32Zfloat64Zint32rk   Zint64)r   ZconcreterE   rE   rF   to_concrete=  s    z%operator_getitem.<locals>.to_concreter   r`   rO   )	r@   rR   r   r   r   operatorgetitemr   rk   )abr   rE   rE   rF   operator_getitem<  s    
r   _MANUAL_META_OVERRIDESc                       sh   e Zd ZdZdd Zedd Zedd Z fdd	Z fd
dZ	dd Z
dd Z fddZ  ZS )HFProxyzI
    Proxy that uses metadata to handle data-dependent control-flow.
    c                 C   s
   || _ d S r]   )	_metadata)rW   metadatarE   rE   rF   install_metadata  s    zHFProxy.install_metadatac                 C   s   | j dd| fi S )Ncall_methodr   )tracercreate_proxyrW   rE   rE   rF   rT     s    zHFProxy.shapec                 C   s
   t | dS )NrP   )MetaDeviceAttributer   rE   rE   rF   rP     s    zHFProxy.devicec                    s(   t | dr| jd k	rt| jS t  S Nr   )hasattrr   rt   super__len__r   	__class__rE   rF   r     s    
zHFProxy.__len__c                    s$   t | dr| jd k	r| jS t  S r   )r   r   r   __bool__r   r   rE   rF   r     s    zHFProxy.__bool__c                 C   s   |dkr|  |S t| |S r   )__getattribute__HFAttribute)rW   krE   rE   rF   __getattr__  s    
zHFProxy.__getattr__c                 C   s   | j dtj| ||fi S Ncall_function)r   r   r   setitem)rW   indicesvaluesrE   rE   rF   __setitem__  s    zHFProxy.__setitem__c                    s*   t | dr| jd k	r|| jkS t |S r   )r   r   r   __contains__)rW   r   r   rE   rF   r    s    
zHFProxy.__contains__)__name__
__module____qualname____doc__r   propertyrT   rP   r   r   r   r  r  __classcell__rE   rE   r   rF   r     s   

r   c                   @   s.   e Zd ZedddZedd Zdd ZdS )	r   )attrc                 C   s>   || _ || _|j| _d | _t| j dr:| t| j j| d S r   )rootr  r   _noder   r   getattrr   )rW   r  r  rE   rE   rF   __init__  s    zHFAttribute.__init__c                 C   s0   | j d kr*| jdtj| j| jfi j| _ | j S r   )r  r   r   builtinsr  r  r  noder   rE   rE   rF   r    s    
 zHFAttribute.nodec                 O   s   | j d| j| jf| |S )Nr   )r   r   r  r  )rW   rw   rx   rE   rE   rF   __call__  s    zHFAttribute.__call__N)r  r  r	  rA   r  r  r  r  rE   rE   rE   rF   r     s   	
r   c                   @   s   e Zd ZdS )r   N)r  r  r	  rE   rE   rE   rF   r     s   r   c                   @   s.   e Zd ZdZee dddZedd ZdS )HFCacheProxyzP
    Proxy that represents an instance of `transformers.cache_utils.Cache`.
    orig_cache_clsc                 C   s
   || _ d S r]   )_orig_cache_cls)rW   r  rE   rE   rF   install_orig_cache_cls  s    z#HFCacheProxy.install_orig_cache_clsc                 C   s    t | dstd| jj| j S )Nr  z?The original Cache class must be installed to the HFCacheProxy.)r   RuntimeErrorr   _CLASSES_TO_PATCHr  r   rE   rE   rF   r     s    
zHFCacheProxy.__class__N)	r  r  r	  r
  r	   r   r  r  r   rE   rE   rE   rF   r    s   r  r  r   get_attr)functionop_typeproxy_factory_fnr=   c                    s   t   fdd}|S )Nc                     s   t  s| |S g   fdd}tjj| | tjj|| t dkr d j}dkrd}n0dkrtj}n dkrj}ntd d|j	|| |d	S | |S d S )
Nc                    s   t | tr |  d S r]   )r@   r   rC   )r   Zfound_proxiesrE   rF   check_proxy  s    
z4create_wrapper.<locals>.wrapper.<locals>.check_proxyr   r  r   r  zop_type z not supported.r  )
r   rR   fxr  map_aggregatert   r   r  rg   r   )rw   rx   r!  r   r   r  r  r  r   rF   wrapper  s"    

zcreate_wrapper.<locals>.wrapper)	functoolswraps)r  r  r  r&  rE   r%  rF   create_wrapper  s    r)  c                       sL   e Zd ZdZdeeedf eeef e	e
egef  d fddZ  ZS )HFProxyableClassMetazW
    Metaclass that creates a class with its main methods wrapped to be proxyable.
    N.)namebasesattrsr  c              	      s   t  | |||} t| D ]z}t| |d }|d kr4q|dkrBd}n4|drRd }n$t|rbd}nt|rrd}nd }|d k	rt| |t	|||d q| S )Nr  r  __r   r"  )
r   __new__dirr  
startswithinspectismethod
isfunctionsetattrr)  )clsr+  r,  r-  r  	attr_namer  r  r   rE   rF   r/    s"    


zHFProxyableClassMeta.__new__)N)r  r  r	  r
  rA   r   r	   r   r   r   r   r   r   r/  r  rE   rE   r   rF   r*    s   	 

r*  )r   r=   c                 C   s   t | d}|| fS )zo
    Wraps `target` to be proxyable. Used for tensor creators like `torch.ones`, `torch.arange` and so on.
    r  )r)  )r   r&  rE   rE   rF   gen_constructor_wrapper  s    
r8  c                 C   sH   t | trdS t | tjjrDt | tr0t| ds>td|  | jS | S )z\Returns the underlying metadata for HFProxies, and behaves like the identity for the others.rO   r   zNo metadata was found for )	r@   r   rR   r#  r   r   r   r  r   )vrE   rE   rF   _proxies_to_metas$  s    
r:  )r  r=   c                    s   t td fdd}|S )N)ry   r=   c                    s*   t ttstdt| t}|  |S )NzJCannot create HFCacheProxy because there is no HFTracer currently tracing.)r@   _CURRENT_TRACERHFTracerr  r  r  )ry   Zcache_proxyr  rE   rF   cache_proxy_factory_fn0  s
    


z=create_cache_proxy_factory_fn.<locals>.cache_proxy_factory_fn)r   r  )r  r=  rE   r  rF   create_cache_proxy_factory_fn/  s    r>  ProxyableCacher"  ProxyableDynamicCacheProxyableSinkCacheProxyableStaticCache
      )lowhighforbidden_valuesc                 C   s2   |d krg }t | |}||kr.t | |}q|S r]   )randomrandint)rE  rF  rG  r   rE   rE   rF   _generate_random_intS  s    rJ  c                       s  e Zd ZU dZdZeed< dZeed< ddddd	d
dddddgZe	e
eeeeeeiZe s`efneefZefdf fdd	Zdeee ee eeejf dddZd9 fdd	Zdd Z ee!eee!f dddZ" fdd Z#d!d" Z$e%j&e'ej(j)e*d#e!f f d$d%d&Z+d:e'ej(j)e*d#e!f f e,eee!f  e,eee!f  ee-d' fd(d)Z.e(j)ed*d+d,Z/e(j)ed*d-d.Z0e(j)ed* fd/d0Z1ej(j)eed1 fd2d3Z2e3dd4d5e!d6d7d8Z4  Z5S );r<  z
    Tracer that is able to symbolically trace models from the library. To do that, it uses the HFProxy instead of the
    regular PyTorch torch.fx.Proxy.
    Tproxy_buffer_attributesallow_insert_stateless_modsarangezerosZonesr   Z	full_likeZeyerS   r   clampZfinfoZtrilrE   c                    s2   t  j||d t s.tdt  dt dd S )N)autowrap_modulesautowrap_functionsz6Found an incompatible version of torch. Found version z, but only version z is supported.)r   r  r8   ImportErrorr6   r5   )rW   rP  rQ  r   rE   rF   r  {  s
    zHFTracer.__init__r   )model
input_namerT   input_namesr=   c              
      sh  t |d|jj}|ji }d}|dkr|d }|tttttttttt	krpt
j|t
jd|d< qd|ttttd4krt
j|t
jd|d< t
j|t
jd|d	< qd|ttkrpt|jd
r|jjdkrtd|jjdkr||jjf}	t
j}
nR|jjdkr"|f}	t
j}
n6|jjdkrD||jjf}	t
j}
ntd|jj dt
j|	|
d|d< n|ttttttttttttd5krt
j|t
jd|d< n>|ttkrt
j|t
jd|d< ntd| d| dnjd|kr|d }t |jdd}|dkrdt|jdr>|jjj}n&t|jdrX|jj j}nt! t! f}t |jdd}t"|t#j$j%s||f}|\}}t
j||||t
jd||< nd|krt
j|d6t
j&d||< nd |krt
j||jj'ft
j&d||< nXd!|kr|d }t |jd"ddk	rJ|jj(d#krJ|jj)}n|jj*}t+|dkrv||d$ |d% |f}n||d$ |f}t
j|t
j&d||< nd&|krt
j||jj,g t
j&d||< nd'|krt
j||jj-g t
j&d||< njd(|kr t
j|t
j&d||< nDd)|krZ|\}}t!d*d+d,}t
j||t
j&d||< n
d-|krd.|kr|d |d$ | g}n|}t
j|t
jd||< nd/|krt
j|t
jd||< nd.|kr@|jj(t.krtd0|jj( d1|jj/}|jj*|jj/ }|d |||f t0 fd2d3t1|jj2D }|||< n$||jj*g }t
j|t
j&d||< |S )7z4Generates dummy input for model inference recording.class_for_deserialization   )labelsstart_positionsend_positionsr   rs   rX  XLNetForQuestionAnsweringrY  rZ  problem_typeNzCould not retrieve the problem type for the sequence classification task, please set model.config.problem_type to one of the following values: "regression", "single_label_classification", or "multi_label_classification".Z
regressionZsingle_label_classificationZmulti_label_classificationzExpected model.config.problem_type to be either: "regression", "single_label_classification", or "multi_label_classification", but "z" was provided.rJ   rK   rL   z!Generating the dummy input named z for z is not supported yet.Zpixel_values
image_sizevision_configencodernum_channels   Zbbox   Zinput_featuresZinputs_embedsembedding_sizerH   r3   r   Zvisual_featsZ
visual_posinputsZinput_valuesi'  i N  rE  rF  maskpast_key_valuesidszMSymbolic trace with past_key_values input is not supported yet for the model ze. Please open an issue or a PR in Transformers repository if you would like to see the support added.c                 3   s2   | ]*}t j t jd t j t jd fV  qdS )rs   N)rR   Zrandru   )r   r   Zcache_shaperP   rE   rF   r   %  s   z1HFTracer._generate_dummy_input.<locals>.<genexpr>)r[  )rJ   rK   rL   )rb  )3r  r   r  rP   r   r(   r'   r#   r   r   rR   rN  longr*   r"   r-   r   configr\  rg   Z
num_labelsr   r)   r/   r    r&   r,   r+   r!   NotImplementedErrorr^  r]  r_  rJ  r@   collectionsabcIterableru   Zinput_feat_per_channelZ
model_typerc  Zhidden_sizert   Zvisual_feat_dimZvisual_pos_dim"_FX_SUPPORTED_MODELS_WITH_KV_CACHEZnum_attention_headsr   r   Znum_hidden_layers)rW   rS  rT  rT   rU  Zmodel_class_nameZinputs_dictZkv_cache_lengthr   Zlabels_shapeZlabels_dtyper]  r`  heightwidthrc  Zembedding_shaper   Z
seq_lengthZ
mask_shapeZ	num_headsr   ZpkvZshape_with_hidden_sizerE   ri  rF   _generate_dummy_input  s"   
    


     
 
  












zHFTracer._generate_dummy_inputNc                    s.  t  |||||||}|dkr>|| jkr>|| j|  |S || jkrXd|krXd|d< z|tjj|t	}	tjj|t	}
d}d| _
d| _|dkrt||}||	|
}t|tjr|jdd}n|dkrt|	d j|}t||}||	|
}n|d	krZt| d
st|  d| j|}t|}|tkrLt| |f|	|
}n| j|	|
}nV|dkr| j}|d}|D ]}t||}qxt|tjr|jdd}n|}nd}|rt|tstd|| W nF tk
r } z&trtd| d| d|  W 5 d }~X Y nX d| _
d| _|S )NplaceholderrP   rO   Tr  r`   r   r   call_moduleorig_forwardz/ does not have an attribute called orig_forwardr  .Fz"Don't support composite output yetzCould not compute metadata for z target z: ) r   r   	meta_argsr   orig_fnsrR   r#  r  r$  r:  _disable_module_getattr_disable_call_moduler   rB   r@   r   rk   r  r   r   AttributeErrorr  Zget_submoduletyperv  splitr   rg   	Exception_IS_IN_DEBUG_MODEwarningswarn)rW   kindr   rw   rx   r+  Z	type_exprr  rvZ
args_metasZkwargs_metasZshould_install_metadataZmeta_targetZmeta_outmethodmodmod_typeZattr_itrZatomsZatomer   rE   rF   r   3  sb    





.zHFTracer.create_proxyc                    s   t  ddr|S  fdd}t|tjjrH|| j |}|d k	rH|S  jrxt|tjrx|| j	 |}|d k	rx|S |S d S )Nrz  Fc                    s~   |D ]t\} |kr|krli }dt jjkrPjs<d n fdd|d< jddi f|}||< |   S qd S )Nr  c                    s   t |  S r]   r   )r  )attr_valry   rW   rE   rF   r     r   zLHFTracer._module_getattr.<locals>.maybe_get_proxy_for_attr.<locals>.<lambda>r  rE   )r2  	signaturer   
parametersZparam_shapes_constant)r  Zcollection_to_searchparameter_proxy_cacher   rx   Z	val_proxyr   )r  ry   rF   maybe_get_proxy_for_attr~  s    z:HFTracer._module_getattr.<locals>.maybe_get_proxy_for_attr)
r  r@   rR   r   	Parameterr  Znamed_parametersrK  r   Znamed_buffers)rW   r  r  r  r  Zmaybe_parameter_proxyZmaybe_buffer_proxyrE   r   rF   _module_getattry  s(        zHFTracer._module_getattr)r  r  r  c                 C   s   |  |||S r]   )r  )rW   r  r  r  rE   rE   rF   r    s    zHFTracer.getattrc                    s.   t | ddr|||S || _t ||||S )Nr{  F)r  rv  r   ru  )rW   r   forwardrw   rx   r   rE   rF   ru    s    
zHFTracer.call_modulec                 C   s
   t || S r]   )r   )rW   r  rE   rE   rF   proxy  s    zHFTracer.proxy.)r  c                 c   s0  dd | j D | _t | _| j D ]$\}\}}tt|| | j| q$g }t	|}t
j D ]r\}}|d k	r|||k	r|qb|dsqb| j D ]@\}}	|j D ],\}
}||kr|||
|f t||
|	 qqqbd V  | j D ]\}\}}tt|| qi | _t | _|D ]\}}
}t||
| qd S )Nc                 S   s   i | ]}|t tt|qS rE   )r8  r  rR   )r   r   rE   rE   rF   
<dictcomp>  s     z.HFTracer.patch_for_tracing.<locals>.<dictcomp>Ztransformers)_TORCH_METHODS_TO_PATCHZpatched_torch_methodssetry  itemsr5  rR   addr2  	getmodulesysmodulesr1  r  __dict__rC   )rW   r  r+  r&  origZpatchedZmodule_of_modelr  Zorig_clsZpatched_clsr7  r  r   rE   rE   rF   patch_for_tracing  s4    

zHFTracer.patch_for_tracing)r  concrete_argsdummy_inputs6complete_concrete_args_with_inputs_not_in_dummy_inputsr=   c              	      s  t t|tjjr|jn|} dkr*i  dk	r|r|j D ]0}|j	krPq@|j
t jjkr@td|j	 dq@  fdd|j D  |j    }t }t }	||	g}
|jjttkrtddd}|
d	| dk	rtni }|D ]Z}||krqt|| js(t|jd
rB|| j|||
|d qtd| dqdd }t||}|j D ]2}|jt jj krr|j	|krri |d|j	 < qr|| _!| a"| #|$ zt$ j%| d| _&W 5 da"X W 5 Q R X | j&j'D ]}|j(dkrt|j)|krd|_*tj+|_n\|g}t,- }|rT|.d}d||< |t/|j0 7 }q&t1| D ]}| j&2| q`|j(dkrd|_q| j&S )a  
        Traces `root` and returns the corresponding FX `torch.fx.Graph` representation. `root` can either be a
        `torch.nn.Module` instance or a Python callable. Note that after this call, `self.root` may be different from
        the `root` passed in here. For example, when a free function is passed to `trace()`, we will create a
        `torch.nn.Module` instance to use as the root and add embedded constants to.

        Args:
            root (`torch.nn.Module` or  `Callable`):
                Either a `torch.nn.Module`` or a function to be traced through. If root is not a
                [`~transformers.PreTrainedModel`], then `dummy_inputs` must be passed, otherwise tracing will fail.
            concrete_args (`Dict[str, Any], *optional*):
                Concrete arguments that should not be treated as Proxies
            dummy_inputs (`Dict[str, Any]`, *optional*):
                The dummy inputs needed to handle data-dependent control-flow if `root` is not a
                [`~transformers.PreTrainedModel`]. It can also be used when `root` is a
                [`~transformers.PreTrainedModel`] to specify custom dummy inputs for a subset or all the model inputs.
            complete_concrete_args_with_inputs_not_in_dummy_inputs (`bool`, *optional*, defaults to `True`):
                If `True`, and `dummy_inputs` is specified, every argument that `root` can take that is not in
                `dummy_inputs` and not in `concrete_args` will be added to `concrete_args`, otherwise does nothing.

        Returns:
            `torch.fx.Graph`:
                A FX `torch.fx.Graph` representing the semantics of the passed-in `root`.

        Nz6You need to specify a default value for the parameter rw  c                    s*   i | ]"}|j kr|j  kr|j |jqS rE   r+  r>   r   r   r  r  rE   rF   r    s
   
 
 z"HFTracer.trace.<locals>.<dictcomp>r   rW  re  r3   )Z_deserialize_graph_moduleZ_CodeOnlyModulerU  zCould not generate input named z8 for because root is not a transformers.PreTrainedModel.c                 S   s   t | tjr| dS | S r   )r@   rR   r   rk   )r   rE   rE   rF   to_meta  s    
zHFTracer.trace.<locals>.to_metaz**r  rt  rE   r   r   )3r2  r  r@   rR   r   Moduler  r  r  r+  r>   r  rS   rg   updater?   rJ  r   r  r   r'   r   r~   supported_archsr}  r	  r1  rs  r  pytreeZtree_mapr  VAR_KEYWORDrx  r;  r  r   tracegraphZnodesopr   rw   r   rm  OrderedDictr   r}   ZusersreversedZ
erase_node)rW   r  r  r  r  sigparamrU  r   Zsequence_lengthrT   Znum_choicesrd  rT  r  Zconcrete_metasr  Zto_visitZ	to_deletery   userr   r  rF   r    sv     





zHFTracer.trace)r  r=   c                 C   s   t dd |j D S )z
        Whether the module was instantiated with Proxies. If that is the case, such module cannot be a leaf module
        because its attributes are input-dependent.
        c                 s   s   | ]}t |tV  qd S r]   )r@   r   )r   r  rE   rE   rF   r   T  s     zKHFTracer._stateless_mod_instanciation_depends_on_proxies.<locals>.<genexpr>)anyr  r  )rW   r  rE   rE   rF   /_stateless_mod_instanciation_depends_on_proxiesO  s    z8HFTracer._stateless_mod_instanciation_depends_on_proxiesc                 C   s   |  |rdS d}|jj }| d| }d}t| j|rjt| j||krRd}qj| d| }|d7 }q0|s|| j|| |S )zb
        Helper method which tries to insert a module that was not declared as submodule.
        r:   r   r   FTr3   )r  r   r  lowerr   r  r  Z
add_module)rW   r  idxmod_namepathZalready_insertedrE   rE   rF   _insert_module_as_submoduleV  s    

z$HFTracer._insert_module_as_submodulec              
      s~   zt  |W S  tk
rx } zL| jrdtt| dkrdtt| dkrd| |}| W Y 
S |W 5 d}~X Y nX dS )ag  
        Helper method to find the qualified name of `mod` in the Module hierarchy of `root`. For example, if `root` has
        a submodule named `foo`, which has a submodule named `bar`, passing `bar` into this function will return the
        string "foo.bar".

        Args:
            mod (str): The `Module` to retrieve the qualified name for.
        r   N)	r   path_of_module	NameErrorrL  rt   r}   r  buffersr  )rW   r  r  r  r   rE   rF   r  n  s    	.
zHFTracer.path_of_module)r   module_qualified_namer=   c                    s   |  | ot ||S r]   )r  r   is_leaf_module)rW   r   r  r   rE   rF   r    s     zHFTracer.is_leaf_module)Zis_backward_compatibler   )objr=   c                 C   s"   t |d }|jjdkr|jS |S )zCalled when a proxy object is has the keys() method called.
        This is what happens when ** is called on a proxy. This should return an iterator if ** is supposed to work in
        your custom tracer.
        r?   z**kwargs)r   r  r   r   )rW   r  	attributerE   rE   rF   r?     s    zHFTracer.keys)NNN)NNT)6r  r  r	  r
  rK  bool__annotations__rL  r  r   r?  r   r@  r   rA  r   rB  r  r7   r   r9   r  r   r  rA   r   rv   r   rR   r   rs  r   r  r   r  ru  r  
contextlibcontextmanagerr
   r   r  r   r  r   r   r  r  r  r  r  r   r?   r  rE   rE   r   rF   r<  \  sp   
    
    0F&$'   r<  )rS  rU  c                    s|   t | j}t t|j ksdt dkr6 d nd }d|j }td| d|  fdd|j	 D S )Nr3   r   , z(The model does not have input(s) named: z&, expected a subset of the following: c                    s    i | ]}|j  kr|j |jqS rE   r  r  r  rE   rF   r    s     
  z%get_concrete_args.<locals>.<dictcomp>)
r2  r  r  r  r  r?   rt   joinrg   r  )rS  rU  r  Zformatted_input_namesZformatted_allowed_input_namesrE   r  rF   get_concrete_args  s    r  r   rS  c                 C   s   | j jtkS r]   )r   r  _SUPPORTED_MODELSr  rE   rE   rF   is_model_supported  s    r  c                 C   s.   t | s*dt}td| jj d| d S )Nr  zModel z) is not supported yet, supported models: )r  r  r  rl  r   r  )rS  Zsupported_model_namesrE   rE   rF   check_if_model_is_supported  s
    
r  )rS  rU  disable_check
tracer_clsr=   c                 C   s   |dkr| j  }t|}t| |}|s0t|  d|krPt| jddsPtd d|krxt| jddrxtd d| j_	| }|j
| |d}tj| |}| j|_| j|_| j|_|S )a  
    Performs symbolic tracing on the model.

    Args:
        model ([`PretrainedModel`]):
            The model to trace.
        input_names (`List[str]`, *optional*):
            The names of the inputs of the traced model. If unset, model.dummy_inputs.keys() are used instead.
        disable_check (`bool`, *optional*, defaults to `False`):
            If `True`, no check is done before trying to trace the model, this is mostly usesul for debugging purposes.
        tracer_cls (`Type[HFTracer]`, *optional*, defaults to `HFTracer`):
            The tracer class to use for instantiating the tracer. If unset, `HFTracer` is used instead.

    Returns:
        `torch.fx.GraphModule`: A GraphModule constructed by recording operations seen while tracing the model.

    Example:

        ```python
        from transformers.utils.fx import symbolic_trace

        traced_model = symbolic_trace(model, input_names=["input_ids", "attention_mask", "token_type_ids"])
        ```
    Nrg  	use_cacheFz|`past_key_values` were specified as input names, but model.config.use_cache = False, this might lead to unexpected behavior.z`past_key_values` were not specified as input names, but model.config.use_cache = True. Setting model.config.use_cache = False.r  )r  r?   r}   r  r  r  rk  loggerwarningr  r  rR   r#  r   r   rV  rP   )rS  rU  r  r  r  r   Ztraced_graphZtracedrE   rE   rF   symbolic_trace  s,    

r  )N)NNrZ   FF)F)NN)NN)N)N)N)rN   )Nr   FN)N)rC  rD  N)r  rm  r  r'  r2  r   r   osrH  r  r  typingr   r   r   r   r   r   r   r	   r
   rR   Ztorch.utils._pytreeutilsZ_pytreer  r   Ztorch.fxr   r   r   r   r   Ztorch.fx._compatibilityr   Ztorch.fx._symbolic_tracer   Ztorch.fx.proxyr   r:   r   Zcache_utilsr   r   r   r   Zmodeling_utilsr   r   Zmodels.autor   Zmodels.auto.modeling_autor   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   Zpytorch_utilsr2   Zimport_utilsr4   r5   r6   r7   r8   Zpeftr9   Z
get_loggerr  r  environrB   upperr  rA   rG   Z(_REGULAR_SUPPORTED_MODEL_NAMES_AND_TASKSrp  Z_REGULAR_SUPPORTED_MODELSitemr@   r~   extendZ_SPECIAL_SUPPORTED_MODELSr   sortedr  r  r;  rY   r\   r^   r_   ra   rd   re   rh   rn   rq   r|   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   Z	EmbeddingZ
functionalZ	embeddingZ	LayerNormZ	GroupNormZLinearZreluZReLUwhereabsrM  r   catstackr  mulr   matmulZbmmZbaddbmmr   repeatZrepeat_interleaveZrollZflipZindex_selectZgatherZConv1dZConv2dZsqueezeZ	unsqueezer   Zone_hotZMSELossZCrossEntropyLossZBCEWithLogitsLossr   r   r  Zscaled_dot_product_attentionr   r   r   r  r)  r}  r*  r8  r:  r>  r?  r@  rA  rB  rv   rJ  r<  r  r  r  r  r  r  rE   rE   rE   rF   <module>   sh   ,X	
 *5         

$



       
                                           ., $    	    8

