去除停用词，从文件中获取停用词

Question

这里我使用了一个包含停用词列表的文件。我想从文本中删除所有停用词。

def print_stopWords(self):

    #infile = open("D:\Komal\MyPrograms\Pkg\PkgSubfolder\StopWords.txt", 'r')
    stopwords = ()
    print '\nstopwords are-'
    for line in open('D:\Komal\MyPrograms\Pkg\PkgSubfolder\StopWords.txt'):
        stopwords += (line,)

    print stopwords
    return stopwords


def strip_stopwords(self,text,stopword):
    print '\n Text after removing all stopwords is --'
    words = text.split()
    text = []
    for word in words:
        if word.lower() not in stopword:
            text.append(word)
    print u' '.join(text)        #'u' prefix allows you to write a unicode string literal
    return text

Answer 1

问题不清楚（您应该显示所有代码），但我认为您的主要问题是：

stopwords = ()

() 用于声明一个元组，该元组是不可变的，即一旦定义，您就无法更改其内容。您可能正在寻找一个列表、字典或集合（在这种情况下是最好的），所有这些都可以添加元素（例如在您的 for 循环中）。您应该查看 Python 教程以了解这些基本数据结构。

Answer 2

你的问题不清楚。我看到的唯一功能问题是 stopwords 是一个元组，它是不可变的，所以你不能附加到它，不像列表。

无论如何，为了性能stopwords 应该是一个集合（/dict），而不是一个列表/元组。一个集合的查找是 O(1) 而不是 O(N)。

def print_stopWords(self):
    stopwords = set()
    print '\nstopwords are-'
    for word in ...:
        stopwords.add(line)
    return stopwords

print_stopWords() 是一个方法有点奇怪，但它不会在任何地方修改对象（即从不使用 self，例如分配给 self.stopwords）

strip_stopwords() 可以简单地使用列表理解：

u' '.join(w for w in text.split() if w.lower() not in stopwords)

Answer 3

将停用词导入您的 Python 代码的整个业务可以通过一行代码完成。然而重要的是理解代码背后的逻辑。

为了选择正确的数据结构：为我们的项目存储停用词列表，我们需要一个不可变的数据结构（Sets/Tuples）并最小化内存使用量（Sets）。所以我们使用集合。

stopword = set(line.strip() for line in open('Stopwords' ,'r'))

Answer 4

在这里你可以用我的代码来做。

from __future__ import division
import re
import itertools
import re
import hashlib

def splitFile(lines,splitvalue):
    documents={};
    documentCount=1
    dcmnt="";
    for line in lines:
        dcmnt+=line;
        if (line.__contains__(splitvalue)):
            key="documents"+(str)(documentCount);
            documents[key]=dcmnt;
            dcmnt="";
            documentCount=documentCount+1;
    return documents;

def findSearchSaveMostRepeated(myHash, documentswords):
    tempvalue=0
    for ii in range(0, myHash.__len__()):

        if myHash[documentswords[ii]] > tempvalue :
            tempvalue= myHash[documentswords[ii]]
            x=documentswords[ii]
        if ii + 1 == myHash.__len__():
            #if tempvalue < myHash[documentswords[ii+1]]:
                myHash[x] = 0;
                myLastHash[x] = tempvalue
                return myLastHash[x]





readFile=open("reuters.txt","r");
lines=readFile.readlines();
readFile.close();
alldocuments=splitFile(lines, "</REUTERS>");
#if (alldocuments.count()!=0):
#    print("");
readstopwordFile=open("stopwords.txt","r");
lines2=readstopwordFile.readlines();
readstopwordFile.close();
allstopwords=splitFile(lines2, "\n");
#if (lines.count()!=0):
#    print("");
readFile3=open("reuterdeneme.txt","r");
lines3=readFile3.readlines();
readFile3.close();
alldocumentssecond=splitFile(lines3, "</REUTERS>");

documentswords = []
with open('reuterdeneme.txt','r') as f:  #reading a text file and splitting it into single words
    for line in f:
        for word in line.split():
            documentswords.append(word)
print(documentswords[6]+"example document value")
stopwords=[]
with open('stopwords.txt','r') as f:  #reading a text file and splitting it into single words
    for line in f:
        for word in line.split():
            stopwords.append(word)

print(stopwords[9]+"example stopword value");

#print(stopwords.__len__()) # length of it
numberofwords=documentswords.__len__()
print(numberofwords);

myHash={}
tempofStopwords = []
countTEMP=[]

for i in range(0,documentswords.__len__()):
    count=0;
    for ii in range(0,documentswords.__len__()):


        if documentswords[i]==documentswords[ii]:
            count=count+1;
        if ii+1  == documentswords.__len__():
            print("word")
            print(documentswords[i]);
            print("tanesi");
            print(count);
            tempofStopwords.append(documentswords[i]);
            countTEMP.append(count);
            myHash[documentswords[i]] = count  #words' count



#for i in range(0, myHash.__len__()):
print("hash")
# print(myHash[documentswords[6]]); # I am getting count value with this

print(myHash.keys()); # my key values in myHash


myLastHash={}

tempvalue=0;
print(myHash.__len__())

for i in range(0,100):

    findSearchSaveMostRepeated(myHash,documentswords )  # i am getting the most repeated 100 stopwords


print("my last hash")
print(myLastHash.keys());
print(myLastHash.__len__())
count1=0
count2=0
tempSameStopwords=[]
tempnotSameStopwords=[]
for i in range(0,stopwords.__len__()):
    print(stopwords[i])
    if True==myLastHash.keys().__contains__(stopwords[i]):
        #print("THERE IS")
        count1=count1+1;
        tempSameStopwords.append(stopwords[i])
    if False==myLastHash.keys().__contains__(stopwords[i]):
        #print("THERE IS NOT)
        count2=count2+1;
        tempnotSameStopwords.append(stopwords[i])




print("SAME STOPWORDS")
for i in range(0, tempSameStopwords.__len__()):
    print(tempSameStopwords[i])


print("DIFFERENT STOPWORDS")
for i in range(0, tempnotSameStopwords.__len__()):
    print(tempnotSameStopwords[i])

print("COUNT OF SAME STOPWORDS")
print(count1);
print("COUNT OF DIFFERENT STOP WORDS")
print(count2);



newdocumentswords=[]


print(alldocumentssecond.keys()); #got number of documents -> doc2 from reuterdeneme.txt



for i in range(0,stopwords.__len__()):           #inverted index part.. i am trying to do splitting all documents with single word in every line, and after that i will check which term in which document..
    for ii in range(1,3):
        x = 'file'
        y = ii.__str__();
        x = x + y.__str__();
        z = '.txt'
        m = x + z.__str__();
        #print(m);
        h='documents'.__str__()
        h=h+y;
        #print(h)
        xn = alldocuments[h]
        f = open(m, 'w')
        f.write(xn)
        f.close()
        with open(m, 'r') as f:  # reading a text file and splitting it into single words
            for line in f:
                for word in line.split():
                    newdocumentswords.append(word)