如何导航 Nonetype 错误训练数据

Question

错误 TypeError: 'NoneType' 类型的参数不可迭代 上线

TypeError                                 Traceback (most recent call last)
<ipython-input-59-04c754604fb6> in <module>
    122             # Update the model
    123             nlp.update([example], losses=losses, drop=0.3)
--> 124         with textcat.model.use_params(optimizer.averages):
    125             # evaluate on the dev data split off in load_data()
    126             scores = evaluate(nlp.tokenizer, textcat, dev_texts, dev_cats)

import pandas as pd
import numpy as np
import spacy
from spacy import displacy
from spacy.util import minibatch, compounding
from spacy.training.example import Example

import matplotlib.pyplot as plt
%matplotlib inline


df=pd.read_csv('305.csv')
df.shape

df.head().T

first_party = df[df.label=="First Party Collection/Use"][:50000]
data_security = df[df.label=="Data Security"][:50000]
policy_change = df[df.label=="Policy Change"][:50000]
third_party = df[df.label=="Third Party Sharing/Collection"][:50000]
user_choice = df[df.label=="User Choice/Control"][:50000]
user_access = df[df.label=="User Access, Edit and Deletion"][:50000]
international = df[df.label=="International and Specific Audiences"][:50000] 
data_r = df[df.label=="Data Retention"][:50000] 

train_df=first_party.append(data_security)
train_df=first_party.append(policy_change)
train_df=first_party.append(third_party)
train_df=first_party.append(user_choice)
train_df=first_party.append(user_access)
train_df=first_party.append(international)
train_df=first_party.append(data_r)
train_df.shape


train_df['tuples'] = train_df.apply(
    lambda row: (row['data'],row['label']), axis=1)
train = train_df['tuples'].tolist()
train[:1]

train[-2:]

def load_data(limit=0, split=0.8):
    train_data = train
    np.random.shuffle(train_data)
    train_data = train_data[-limit:]
    texts, labels = zip(*train_data)
    cats = [{'POSITIVE': bool(y)} for y in labels]
    split = int(len(train_data) * split)
    return (texts[:split], cats[:split]), (texts[split:], cats[split:])

def evaluate(tokenizer, textcat, texts, cats):
    docs = (tokenizer(text) for text in texts)
    tp = 1e-8  # True positives
    fp = 1e-8  # False positives
    fn = 1e-8  # False negatives
    tn = 1e-8  # True negatives
    for i, doc in enumerate(textcat.pipe(docs)):
        gold = cats[i]
        for label, score in doc.cats.items():
            if label not in gold:
                continue
            if score >= 0.5 and gold[label] >= 0.5:
                tp += 1.
            elif score >= 0.5 and gold[label] < 0.5:
                fp += 1.
            elif score < 0.5 and gold[label] < 0.5:
                tn += 1
            elif score < 0.5 and gold[label] >= 0.5:
                fn += 1
    precision = tp / (tp + fp)
    recall = tp / (tp + fn)
    f_score = 2 * (precision * recall) / (precision + recall)
    return {'textcat_p': precision, 'textcat_r': recall, 'textcat_f': f_score}

#("Number of texts to train from","t" , int)
n_texts=30000
#You can increase texts count if you have more computational power.

#("Number of training iterations", "n", int))
n_iter=10

nlp = spacy.blank('en')


# add the text classifier to the pipeline if it doesn't exist
# nlp.create_pipe works for built-ins that are registered with spaCy
if 'textcat' not in nlp.pipe_names:
    textcat = nlp.add_pipe('textcat', last=True)
# otherwise, get it, so we can add labels to it
else:
    textcat = nlp.get_pipe('textcat')

# add label to text classifier
textcat.add_label('POSITIVE')

# load the dataset
print("Loading food reviews data...")
(train_texts, train_cats), (dev_texts, dev_cats) = load_data(limit=n_texts)
print("Using {} examples ({} training, {} evaluation)"
      .format(n_texts, len(train_texts), len(dev_texts)))
train_data = list(zip(train_texts,
                      [{'cats': cats} for cats in train_cats]))



# get names of other pipes to disable them during training
other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'textcat']
with nlp.disable_pipes(*other_pipes):  # only train textcat
    optimizer = nlp.begin_training()
    print("Training the model...")
    print('{:^5}\t{:^5}\t{:^5}\t{:^5}'.format('LOSS', 'P', 'R', 'F'))
    for i in range(n_iter):
        losses = {}
        # batch up the examples using spaCy's minibatch
        batches = minibatch(train_data, size=compounding(4., 32., 1.001))
    for batch in spacy.util.minibatch(train_data, size=2):
        for text, annotations in batch:
            # create Example
            doc = nlp.make_doc(text)
            example = Example.from_dict(doc, annotations)
            # Update the model
            nlp.update([example], losses=losses, drop=0.3)
        with textcat.model.use_params(optimizer.averages):
            # evaluate on the dev data split off in load_data()
            scores = evaluate(nlp.tokenizer, textcat, dev_texts, dev_cats)
        print('{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}'  # print a simple table
              .format(losses['textcat'], scores['textcat_p'],
                      scores['textcat_r'], scores['textcat_f']))

我最初的问题是由于 Spacy v2 更改了 Spacy v3 中的更新功能，但是 我现在收到此 Nonetype 错误，我不知道为什么。

我正在尝试训练我的数据。错误是否出在 optimizer.averages 中？

请告诉我如何解决这个问题！

Answer 1

我认为您的问题是您正在使用带有单个标签的 textcat 架构。如果您有一个二元分类任务（看起来您想将文档标记为“正面”或不“正面”），您应该将其构建为具有两个标签的任务。请参阅 this thread 了解相关讨论。