如何从 Python 列表中删除常用词？

Question

我有大量的单词“组”。如果一组中的任何单词同时出现在 A 列和 B 列中，我想从两列中删除该组中的单词。如何循环遍历所有组（即遍历列表中的子列表）？

下面有缺陷的代码只删除了最后一组中的常用词，而不是全部三个组（列表）。 [如果组中的一个单词在字符串中，我首先创建一个指示符，然后如果两个字符串都包含该组中的一个单词，则创建另一个指示符。仅对于 A 和 B 对，其中都具有组中的单词，我删除了特定的组单词。]

如何正确指定循环？

编辑： 在我建议的代码中，每个循环都使用原始列重新开始，而不是循环遍历从前一组中删除的单词的列。

解决方案建议更加优雅和整洁，但如果它们是另一个单词的一部分，则删除这些单词（例如，单词 'foo' 正确地从 'foo hello' 中删除，但也错误地从 'foobar' 中删除。

# Input data:

data = {'A': ['summer time third grey abc', 'yellow sky hello table', 'fourth autumnwind'],
        'B': ['defg autumn times fourth table', 'not red skies second garnet', 'first blue chair winter']
}
df = pd.DataFrame (data, columns = ['A', 'B'])  

                            A                               B
0  summer time third grey abc  defg autumn times fourth table
1      yellow sky hello table     not red skies second garnet
2           fourth autumnwind         first blue chair winter

# Groups of words to be removed:

colors = ['red skies', 'red sky', 'yellow sky', 'yellow skies', 'red', 'blue', 'black', 'yellow', 'green', 'grey']
seasons = ['summer times', 'summer time', 'autumn times', 'autumn time', 'spring', 'summer', 'winter', 'autumn']
numbers = ['first', 'second', 'third', 'fourth']

stuff = [colors, seasons, numbers]



# Code below only removes the last list in stuff (numbers):

def fA(S,y):
    for word in listed:
        if re.search(r'\b' + re.escape(word) + r'\b', S):
            y = 1
    return y


def fB(T,y):
    for word in listed:
        if re.search(r'\b' + re.escape(word) + r'\b', T):
            y = 1
    return y



def fARemove(S):
    for word in listed:
        if re.search(r'\b' + re.escape(word) + r'\b', S):
            S=re.sub(r'\b{}\b'.format(re.escape(word)), ' ', S)
    return S



def fBRemove(T):
    for word in listed:
        if re.search(r'\b' + re.escape(word) + r'\b', T):
            T=re.sub(r'\b{}\b'.format(re.escape(word)), ' ', T)
    return T

for listed in stuff:

    df['A_Ind'] = 0
    df['B_Ind'] = 0

    df['A_Ind'] = df.apply(lambda x: fA(x.A, x.A_Ind), axis=1)
    df['B_Ind'] = df.apply(lambda x: fB(x.B, x.B_Ind), axis=1)

    df['inboth'] = 0
    df.loc[((df.A_Ind == 1) & (df.B_Ind == 1)), 'inboth'] = 1

    df['A_new'] = df['A']
    df['B_new'] = df['B']

    df.loc[df.inboth == 1, 'A_new'] = df.apply(lambda x: fARemove(x.A), axis=1)
    df.loc[df.inboth == 1, 'B_new'] = df.apply(lambda x: fBRemove(x.B), axis=1)


    del df['inboth']
    del df['A_Ind']
    del df['B_Ind']
    
    df['A_new'] = df['A_new'].str.replace('\s{2,}', ' ')
    df['A_new'] = df['A_new'].str.strip()
    df['B_new'] = df['B_new'].str.replace('\s{2,}', ' ')
    df['B_new'] = df['B_new'].str.strip()

预期输出是：

         A_new              B_new
0     grey abc         defg table
1  hello table   no second garnet
2   autumnwind  blue chair winter

Answer 1

以下是使用正则表达式 r'\b{}\b' 的原始问题的代码，已针对循环最新字符串而不是原始字符串进行了更正。

# Groups of words to be removed:

colors = ['red skies', 'red sky', 'yellow sky', 'yellow skies', 'red', 'blue', 'black', 'yellow', 'green', 'grey']
seasons = ['summer times', 'summer time', 'autumn times', 'autumn time', 'spring', 'summer', 'winter', 'autumn']
numbers = ['first', 'second', 'third', 'fourth']

stuff = [colors, seasons, numbers]


df['A_new'] = df['A']
df['B_new'] = df['B']


def f_indicator(S,y):
    for word in listed:
        if re.search(r'\b' + re.escape(word) + r'\b', S):
            y = 1
    return y


def fRemove(S):
    for word in listed:
        if re.search(r'\b' + re.escape(word) + r'\b', S):
            S=re.sub(r'\b{}\b'.format(re.escape(word)), ' ', S)
    return S


for listed in stuff:

    df['A_Ind'] = 0
    df['B_Ind'] = 0

    df['A_Ind'] = df.apply(lambda x: f_indicator(x.A_new, x.A_Ind), axis=1)
    df['B_Ind'] = df.apply(lambda x: f_indicator(x.B_new, x.B_Ind), axis=1)

    df['inboth'] = 0
    df.loc[((df.A_Ind == 1) & (df.B_Ind == 1)), 'inboth'] = 1



    df.loc[df.inboth == 1, 'A_new'] = df.apply(lambda x: fRemove(x.A_new), axis=1)
    df.loc[df.inboth == 1, 'B_new'] = df.apply(lambda x: fRemove(x.B_new), axis=1)


    del df['inboth']
    del df['A_Ind']
    del df['B_Ind']

    
    df['A_new'] = df['A_new'].str.replace('\s{2,}', ' ')
    df['A_new'] = df['A_new'].str.strip()
    df['B_new'] = df['B_new'].str.replace('\s{2,}', ' ')
    df['B_new'] = df['B_new'].str.strip()

del df['A']
del df['B']
print(df)

输出：

         A_new              B_new
0     grey abc         defg table
1  hello table  not second garnet
2   autumnwind  blue chair winter

Answer 2

这需要 python 3.7+ 才能工作（否则需要更多代码）。根据您的关键字列表，我认为您正在尝试优先考虑多字匹配。

dummy=0
def splitter(text):
    global dummy
    text=text.strip()
    if not text:
        return []
    for n,s in enumerate(stuff):
        for keyword in s:
            p=text.find(keyword)
            if p>=0:
                return splitter(text[:p])+[((dummy,keyword),n)]+splitter(text[p+len(keyword):])
    else:
        return [((dummy,text),-1)]

def remover(row):
    A=dict(splitter(row['A']))
    B=dict(splitter(row['B']))
    s=set(A.values()).intersection(set(B.values()))
    return [' '.join([k[1] for k,v in A.items() if v<0 or v not in s]),' '.join([k[1] for k,v in B.items() if v<0 or v not in s])]
pd.concat([df,pd.DataFrame(df.apply(remover, axis=1).to_list(), columns=['newA','newB'])],  axis=1)

Answer 3

import re

flatten_list = lambda l: [item for subl in l for item in subl]
def remove_recursive(s, l):
    while len(l) > 0:
        s = s.replace(l[0], '')
        l = l[1:]

    return re.sub(r'\ +', ' ', s).strip()


df['A_new'] = df.apply(lambda x: remove_recursive(x.A, flatten_list([l for l in stuff if (len([e for e in l if e in x.A]) > 0 and len([e for e in l if e in x.B]) > 0)])), axis = 1)
df['B_new'] = df.apply(lambda x: remove_recursive(x.B, flatten_list([l for l in stuff if (len([e for e in l if e in x.A]) > 0 and len([e for e in l if e in x.B]) > 0)])), axis = 1)

df.head()

#            A_new              B_new
# 0  time grey abc         defg table
# 1    hello table  not second garnet
# 2           wind         blue chair

这与 cmets 中的代码类似，使用递归 lambda 来匹配单词，并使用扁平列表来计算列表中在两列中匹配的单词。