PyTorch 中的损失维度问题（标签学习的序列）

Question

我在 PyTorch 中做一个序列来标记学习模型。我有两个句子，我正在分类它们是否包含（SNLI 数据集）。我将两个 50 个单词的句子连接在一起（有时被填充）成一个长度为 100 的向量。然后我将小批量发送到单词嵌入 -> LSTM -> 线性层。我正在做交叉熵损失，但我需要一个 [mini_batch, C] 的向量才能进入 CrossEntropyLoss 函数。相反，我的向量中仍然有 100 个单词为 [mini_batch, 100, C]

这是我的模型：

class myLSTM(nn.Module):
    def __init__(self, h_size=128, v_size=10, embed_d=300, mlp_d=256):
        super(myLSTM, self).__init__()
        self.embedding = nn.Embedding(v_size, embed_d)
        self.lstm = nn.LSTM(embed_d, h_size, num_layers=1, bidirectional=True, batch_first=True)
        self.mlp = nn.Linear(mlp_d, 1024)

        # Set static embedding vectors
        self.embedding.weight.requires_grad = False

        #self.sm = nn.CrossEntropyLoss()

    def display(self):
        for param in self.parameters():
            print(param.data.size())

    def filter_params(self):
        # Might not be compatible with python 3
        #self.parameters = filter(lambda p: p.requires_grad, self.parameters())
        pass

    def init_hidden(self):
        # Need to init hidden weights in LSTM
        pass

    def forward(self, sentence):
        print(sentence.size())
        embeds = self.embedding(sentence)
        print(embeds.size())
        out, _ = self.lstm(embeds)
        print(out.size())
        out = self.mlp(out)
        return out

我的输出训练序列：

batch_size = 3
SGD_optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0.01, weight_decay=1e-4)
ADM_optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=0.01)

criterion = nn.CrossEntropyLoss()
num_epochs = 50

from torch.autograd import Variable
from torch import optim

for epoch in range(num_epochs):
    print("Epoch {0}/{1}: {2}%".format(epoch, num_epochs, float(epoch)/num_epochs))
    for start, end in tqdm(batch_index_gen(batch_size, len(n_data))):

        # Convert minibatch to numpy
        s1, s2, y = convert_to_numpy(n_data[start:end])

        # Convert numpy to Tensor
        res = np.concatenate((s1,s2), axis=1) # Attach two sentences into 1 input vector
        input_tensor = torch.from_numpy(res).type(torch.LongTensor)
        target_tensor = torch.from_numpy(y).type(torch.FloatTensor)
        data, target = Variable(input_tensor), Variable(target_tensor)

        # Zero gradients
        SGD_optimizer.zero_grad()

        # Forward Pass
        output = model.forward(data) 
        print("Output size: ")
        print(output.size())
        print("Target size: ")
        print(target.size())
        # Calculate loss with respect to training labels
        loss = criterion(output, target)

        # Backprogogate and update optimizer
        loss.backward()
        SGD_optimizer.step()
        #ADAM_optimizer.step()    

输出：

Epoch 0/50: 0.0%
torch.Size([3, 100])
torch.Size([3, 100, 300])
torch.Size([3, 100, 256])
Output size: 
torch.Size([3, 100, 1024])
Target size: 
torch.Size([3])

错误：

ValueError: Expected 2 or 4 dimensions (got 3)

已编辑 ------------------------------------------- ------------------------

我现在已经接受了模型训练，但我的准确率很低。我的 LSTM 输出被连接然后压缩到一个较小的张量以通过我的线性层是否存在问题？

新模型：

class myLSTM(nn.Module):
    def __init__(self, h_size=128, v_size=10, embed_d=300, mlp_d=256, num_classes=3, lstm_layers=1):
        super(myLSTM, self).__init__()
        self.num_layers = lstm_layers
        self.hidden_size = h_size
        self.embedding = nn.Embedding(v_size, embed_d)
        self.lstm = nn.LSTM(embed_d, h_size, num_layers=lstm_layers, bidirectional=True, batch_first=True)
        self.mlp = nn.Linear(2 * h_size * 2, num_classes)

        # Set static embedding vectors
        self.embedding.weight.requires_grad = False

    def forward(self, s1, s2):
        # Set initial states
        #h0 = Variable(torch.zeros(self.num_layers*2, s1.size(0), self.hidden_size)).cuda() # 2 for bidirection 
        #c0 = Variable(torch.zeros(self.num_layers*2, s1.size(0), self.hidden_size)).cuda()

        batch_size = s1.size()[0]
        embeds_1 = self.embedding(s1)
        embeds_2 = self.embedding(s2)
        _, (h_1_last, _) = self.lstm(embeds_1)#, (h0, c0)) #note the change here. Last hidden state is taken
        _, (h_2_last, _) = self.lstm(embeds_2)#, (h0, c0))
        concat = torch.cat( (h_1_last, h_2_last), dim=2) #double check the dimension
        concat = concat.view(batch_size, -1)
        scores = self.mlp(concat)
        return scores

新培训

batch_size = 64
SGD_optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=0.001, weight_decay=1e-4)

criterion = nn.CrossEntropyLoss()
num_epochs = 10
model.train()

if cuda:
    model = model.cuda()
    criterion = criterion.cuda()

from torch.autograd import Variable
from torch import optim

epoch_losses = []

for epoch in range(num_epochs):
    print("Epoch {0}/{1}: {2}%".format(epoch, num_epochs, 100*float(epoch)/num_epochs))

    # Batch loss aggregator
    losses = []

    for start, end in tqdm(batch_index_gen(batch_size, len(n_data))):
        # Convert minibatch to numpy
        s1, s2, y = convert_to_numpy(n_data[start:end])

        # Convert numpy to Tensor
        s1_tensor = torch.from_numpy(s1).type(torch.LongTensor)
        s2_tensor = torch.from_numpy(s2).type(torch.LongTensor)
        target_tensor = torch.from_numpy(y).type(torch.LongTensor)

        s1 = Variable(s1_tensor)
        s2 = Variable(s2_tensor)
        target = Variable(target_tensor)

        if cuda:
            s1 = s1.cuda()
            s2 = s2.cuda()
            target = target.cuda()

        # Zero gradients
        SGD_optimizer.zero_grad()

        # Forward Pass
        output = model.forward(s1,s2) 

        # Calculate loss with respect to training labels
        loss = criterion(output, target)
        losses.append(loss.data[0])

        # Backprogogate and update optimizer
        loss.backward()
        SGD_optimizer.step()

    # concat losses to epoch losses
    epoch_losses += losses

打印张量大小的训练：

Epoch 0/10: 0.0%
Batch size: 64
Sentences
torch.Size([64, 50])
torch.Size([64, 50])
torch.Size([64, 50, 300])
torch.Size([64, 50, 300])
Hidden states
torch.Size([2, 64, 128])
torch.Size([2, 64, 128])
Concatenated hidden states
torch.Size([2, 64, 256])
Reshaped tensors for linear layer
torch.Size([64, 512])
Linear propogation
torch.Size([64, 3])

评估

def eval_model(model, mode='dev'):
    file_name = 'snli_1.0/snli_1.0_dev.jsonl' if mode == 'dev' else 'snli_1.0/snli_1.0_test.jsonl'

    dev_data, _ = obtain_data(file_name)
    dev_n_data = vocab.process_data(dev_data)

    print("Length of data: {}".format(len(dev_n_data)))

    eval_batch_size = 1024
    model.eval()

    total = len(dev_n_data)
    hit = 0
    correct = 0

    # Batch dev eval
    for start, end in batch_index_gen(eval_batch_size, len(dev_n_data)):

        s1, s2, y = convert_to_numpy(dev_n_data[start:end])

        s1_tensor = torch.from_numpy(s1).type(torch.LongTensor)
        s2_tensor = torch.from_numpy(s2).type(torch.LongTensor)
        target_tensor = torch.from_numpy(y).type(torch.LongTensor)

        s1 = Variable(s1_tensor, volatile=True)
        s2 = Variable(s2_tensor, volatile=True)
        target = Variable(target_tensor, volatile=True)

        if cuda:
            s1 = s1.cuda()
            s2 = s2.cuda()
            target = target.cuda()

        output = model.forward(s1,s2)
        loss = criterion(output, target)

        #print("output size: {}".format(output.size()))
        #print("target size: {}".format(target.size()))
        pred = output.data.max(1)[1] # get the index of the max log-probability
        #print(pred[:5])
        #print(output[:])
        correct += pred.eq(target.data).cpu().sum()

    return correct / float(total)

eval_model(model)

Answer 1

我认为您尝试解决蕴涵问题的方式存在问题。

也许你可以这样做：

1. 设计您的模块以接受两个句子作为输入
2. 将它们都嵌入到您的嵌入中
3. 使用 LSTM 模块对其进行编码。
4. 现在您有两个句子的两个固定长度向量表示。最简单的事情就是将它们连接起来 在一起。
5. 在顶部添加衬垫层以评估每个蕴涵类的分数（我想是 3 个）
6. 应用 softmax 得到合适的概率分布

所以你的模型看起来像这样（仔细检查尺寸）：

class myLSTM(nn.Module):
    def __init__(self, h_size=128, v_size=10, embed_d=300, num_classes = 3):
        super(myLSTM, self).__init__()
        self.embedding = nn.Embedding(v_size, embed_d)
        self.lstm = nn.LSTM(embed_d, h_size, num_layers=1, bidirectional=True, batch_first=True)
        self.mlp = nn.Linear(2*h_size*2, num_classes) #<- change here

    def forward(self, sentence1, sentence2):
        embeds_1 = self.embedding(sentence1)
        embeds_2 = self.embedding(sentence2)
        _, (h_1_last, _) = self.lstm(embeds_1) #note the change here. Last hidden state is taken
        _, (h_2_last, _) = self.lstm(embeds_2)
        concat = torch.concat([h_1_last, h_2_last], dim=1) #double check the dimension
        scores = self.mlp(concat)
        probas = F.softmax(scores) #from torch.functional ...

然后您可以尝试添加更多隐藏层或思考如何以更智能的方式（注意力等）组合两个句子。 仔细检查 CrossEntropyLoss 接受什么作为输入和目标并进行调整（是非标准化的类分数还是概率分布）。查看http://pytorch.org/docs/master/nn.html#lstm 获取 LSTM 模块文档以阐明 LSTM 返回的内容（您需要每个单词的隐藏状态还是仅在最后一个单词之后的表示）。