【发布时间】：2022-09-27 18:00:42
【问题描述】：
我想使用小队数据集微调 LabSE 以进行问答。我得到了这个错误：
ValueError: The model did not return a loss from the inputs, only the following keys: last_hidden_state,pooler_output. For reference, the inputs it received are input_ids,token_type_ids,attention_mask.
我正在尝试使用 pytorch 微调模型。我尝试使用较小的批量大小，并且只使用了 10% 的训练数据集，因为我遇到了内存分配问题。 如果内存分配问题消失了，则会发生此错误。 老实说，我坚持下去。你有什么提示吗？
我正在尝试使用拥抱脸教程，但我想使用其他评估（我想自己做）所以我跳过了使用数据集的评估部分。
from datasets import load_dataset
raw_datasets = load_dataset(\"squad\", split=\'train\')
from transformers import BertTokenizerFast, BertModel
from transformers import AutoTokenizer
model_checkpoint = \"setu4993/LaBSE\"
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
model = BertModel.from_pretrained(model_checkpoint)
max_length = 384
stride = 128
def preprocess_training_examples(examples):
questions = [q.strip() for q in examples[\"question\"]]
inputs = tokenizer(
questions,
examples[\"context\"],
max_length=max_length,
truncation=\"only_second\",
stride=stride,
return_overflowing_tokens=True,
return_offsets_mapping=True,
padding=\"max_length\",
)
offset_mapping = inputs.pop(\"offset_mapping\")
sample_map = inputs.pop(\"overflow_to_sample_mapping\")
answers = examples[\"answers\"]
start_positions = []
end_positions = []
for i, offset in enumerate(offset_mapping):
sample_idx = sample_map[i]
answer = answers[sample_idx]
start_char = answer[\"answer_start\"][0]
end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])
sequence_ids = inputs.sequence_ids(i)
# Find the start and end of the context
idx = 0
while sequence_ids[idx] != 1:
idx += 1
context_start = idx
while sequence_ids[idx] == 1:
idx += 1
context_end = idx - 1
# If the answer is not fully inside the context, label is (0, 0)
if offset[context_start][0] > start_char or offset[context_end][1] < end_char:
start_positions.append(0)
end_positions.append(0)
else:
# Otherwise it\'s the start and end token positions
idx = context_start
while idx <= context_end and offset[idx][0] <= start_char:
idx += 1
start_positions.append(idx - 1)
idx = context_end
while idx >= context_start and offset[idx][1] >= end_char:
idx -= 1
end_positions.append(idx + 1)
inputs[\"start_positions\"] = start_positions
inputs[\"end_positions\"] = end_positions
return inputs
train_dataset = raw_datasets.map(
preprocess_training_examples,
batched=True,
remove_columns=raw_datasets.column_names,
)
len(raw_datasets), len(train_dataset)
from transformers import TrainingArguments
args = TrainingArguments(
\"bert-finetuned-squad\",
save_strategy=\"epoch\",
learning_rate=2e-5,
num_train_epochs=3,
weight_decay=0.01,
)
from transformers import Trainer
trainer = Trainer(
model=model,
args=args,
train_dataset=train_dataset,
tokenizer=tokenizer,
)
trainer.train()
-
代替
BertModel，使用BertForQuestionAnswering。对于未来，请始终发布完整的错误堆栈跟踪。
标签： nlp pytorch huggingface-transformers bert-language-model