python 分词计算文档TF-IDF值并排序
文章来自于我的个人博客:python 分词计算文档TF-IDF值并排序
该程序实现的功能是:首先读取一些文档,然后通过jieba来分词,将分词存入文件,然后通过sklearn计算每个分词文档中的tf-idf值,再将文档排序输入一个大文件中
依赖包:
sklearn
jieba
注:此程序参考了一位同行的程序后进行了修改
# -*- coding: utf-8 -*- """ @author: jiangfuqiang """ import os import jieba import jieba.posseg as pseg import sys import re import time import string from sklearn import feature_extraction from sklearn.feature_extraction.text import TfidfTransformer from sklearn.feature_extraction.text import CountVectorizer reload(sys) sys.setdefaultencoding(‘utf-8‘) def getFileList(path): filelist = [] files = os.listdir(path) for f in files: if f[0] == ‘.‘: pass else: filelist.append(f) return filelist,path def fenci(filename,path,segPath): f = open(path +"/" + filename,‘r+‘) file_list = f.read() f.close() #保存粉刺结果的目录 if not os.path.exists(segPath): os.mkdir(segPath) #对文档进行分词处理 seg_list = jieba.cut(file_list,cut_all=True) #对空格,换行符进行处理 result = [] for seg in seg_list: seg = ‘‘.join(seg.split()) reg = ‘w+‘ r = re.search(reg,seg) if seg != ‘‘ and seg != ‘ ‘ and seg != ‘ ‘ and seg != ‘=‘ and seg != ‘[‘ and seg != ‘]‘ and seg != ‘(‘ and seg != ‘)‘ and not r: result.append(seg) #将分词后的结果用空格隔开,保存至本地 f = open(segPath+"/"+filename+"-seg.txt","w+") f.write(‘ ‘.join(result)) f.close() #读取已经分词好的文档,进行TF-IDF计算 def Tfidf(filelist,sFilePath,path): corpus = [] for ff in filelist: fname = path + ff f = open(fname+"-seg.txt",‘r+‘) content = f.read() f.close() corpus.append(content) vectorizer = CountVectorizer() transformer = TfidfTransformer() tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus)) word = vectorizer.get_feature_names() #所有文本的关键字 weight = tfidf.toarray() if not os.path.exists(sFilePath): os.mkdir(sFilePath) for i in range(len(weight)): print u‘----------writing all the tf-idf in the ‘,i,u‘file into ‘, sFilePath+‘/‘ +string.zfill(i,5)+".txt" f = open(sFilePath+"/"+string.zfill(i,5)+".txt",‘w+‘) for j in range(len(word)): f.write(word[j] + " " + str(weight[i][j]) + " ") f.close() if __name__ == "__main__": #保存tf-idf的计算结果目录 sFilePath = "/home/lifeix/soft/allfile/tfidffile"+str(time.time()) #保存分词的目录 segPath = ‘/home/lifeix/soft/allfile/segfile‘ (allfile,path) = getFileList(‘/home/lifeix/soft/allkeyword‘) for ff in allfile: print "Using jieba on " + ff fenci(ff,path,segPath) Tfidf(allfile,sFilePath,segPath) #对整个文档进行排序 os.system("sort -nrk 2 " + sFilePath+"/*.txt >" + sFilePath + "/sorted.txt")
郑重声明:本站内容如果来自互联网及其他传播媒体,其版权均属原媒体及文章作者所有。转载目的在于传递更多信息及用于网络分享,并不代表本站赞同其观点和对其真实性负责,也不构成任何其他建议。