将这4个停用词表进行组合并去重。
# _*_coding:utf-8 _*_
import os
#得到当前文件路径
current_path=os.getcwd()
#获取文件列表
list_file=os.listdir(current_path)
#用来存储停用词
temp_stopwords=[]
for file in list_file:
file_tail=file.split('.')[-1]
#只要txt格式的文件
if file_tail=='txt':
file_path=os.path.join(current_path,file)
with open(file_path,'r',encoding='utf-8') as doc:
content=doc.readlines()
for words in content:
if words not in temp_stopwords:
temp_stopwords.append(words)
# print(len(temp_stopwords))
#去重之后又2317个停用词
output:
2317
#将去重之后的停用词存储下来
with open('./merge_stopword.txt','w',encoding='utf-8') as f:
for word in temp_stopwords:
f.write(word)
可以在上面直接下载去重之后的停用词