Transformers: AttributeError: 'Tensor' object has no attribute 'transpose'

[mobassir94]

❓ Questions & Help

i get this error :

AttributeError                            Traceback (most recent call last)

<ipython-input-80-01c16e13fe9a> in <module>()
----> 1 get_ipython().run_cell_magic('time', '', "gkf = GroupKFold(n_splits=5).split(X=df_train.question_body, groups=df_train.question_body)\n\nvalid_preds = []\ntest_preds = []\nfor fold, (train_idx, valid_idx) in enumerate(gkf):\n    \n    # will actually only do 2 folds (out of 5) to manage < 2h\n    if fold in [0, 2]:\n\n        train_inputs = [inputs[i][train_idx] for i in range(len(inputs))]\n        train_outputs = outputs[train_idx]\n\n        valid_inputs = [inputs[i][valid_idx] for i in range(len(inputs))]\n        valid_outputs = outputs[valid_idx]\n        \n        K.clear_session()\n        model = create_model()\n        optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5)\n        #optimizer = AdamW(lr=1e-4)\n        model.compile(loss=bce_dice_loss, optimizer=optimizer)\n        model.fit(train_inputs, train_outputs, epochs=6, batch_size=6)\n        # model.save_weights(f'bert-{fold}.h5')\n        valid_preds.append(model.predict(valid_inputs))\n        test_preds.append(model.predict(test_inputs))\n        \n        rho_val = compute_spearmanr_ignore_nan(valid_outputs, valid_preds[-1])\n        print('validation score = ', rho_val)\n        model.save_weights(f'/content/drive/My Drive/quest/validation-{rho_val}-fold-{fold}.hdf5')")

5 frames

</usr/local/lib/python3.6/dist-packages/decorator.py:decorator-gen-60> in time(self, line, cell, local_ns)

<timed exec> in <module>()

/usr/local/lib/python3.6/dist-packages/transformers/modeling_xlnet.py in forward(self, input_ids, attention_mask, mems, perm_mask, target_mapping, token_type_ids, input_mask, head_mask, inputs_embeds)
    726             raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
    727         elif input_ids is not None:
--> 728             input_ids = input_ids.transpose(0, 1).contiguous()
    729             qlen, bsz = input_ids.shape[0], input_ids.shape[1]
    730         elif inputs_embeds is not None:

AttributeError: 'Tensor' object has no attribute 'transpose'

when i try xlnet but i don't get error when i try bert

code i am using :

from transformers import XLNetConfig, XLNetModel,XLNetTokenizer
tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased')

def compute_spearmanr_ignore_nan(trues, preds):
    rhos = []
    for tcol, pcol in zip(np.transpose(trues), np.transpose(preds)):
        rhos.append(spearmanr(tcol, pcol).correlation)
    return np.nanmean(rhos)

def create_model():
    q_id = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
    a_id = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)

    q_mask = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
    a_mask = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)

    q_atn = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)
    a_atn = tf.keras.layers.Input((MAX_SEQUENCE_LENGTH,), dtype=tf.int32)

    #config = BertConfig() # print(config) to see settings
    config = XLNetConfig()
    config.output_hidden_states = False # Set to True to obtain hidden states
    # caution: when using e.g. XLNet, XLNetConfig() will automatically use xlnet-large config

    # normally ".from_pretrained('bert-base-uncased')", but because of no internet, the 
    # pretrained model has been downloaded manually and uploaded to kaggle. 
    #bert_model = TFBertModel.from_pretrained(BERT_PATH+'bert-base-uncased-tf_model.h5', config=config)
    #bert_model = TFBertModel.from_pretrained('xlnet-base-cased')
    #bert_model = XLNetModel(config)
    bert_model = XLNetModel.from_pretrained('xlnet-large-cased')

    # if config.output_hidden_states = True, obtain hidden states via bert_model(...)[-1]
    q_embedding = bert_model(q_id, attention_mask=q_mask, token_type_ids=q_atn)[0]
    a_embedding = bert_model(a_id, attention_mask=a_mask, token_type_ids=a_atn)[0]

    q = tf.keras.layers.GlobalAveragePooling1D()(q_embedding)
    a = tf.keras.layers.GlobalAveragePooling1D()(a_embedding)

    x = tf.keras.layers.Concatenate()([q, a])

    x = tf.keras.layers.Dropout(0.2)(x)

    x = tf.keras.layers.Dense(30, activation='sigmoid')(x)

    model = tf.keras.models.Model(inputs=[q_id, q_mask, q_atn, a_id, a_mask, a_atn,], outputs=x)

    return model

[LysandreJik]

It seems you're passing TensorFlow variables to a PyTorch model. The TensorFlow equivalent of XLNetModel is TFXLNetModel.

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.