LDA模型（一）：基于python中sklearn库的实现

废话不多说，直接上代码

'''LDA模型的实现及可视化'''
import pandas as pd
import numpy as np
import jieba
import jieba.posseg as peg
from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation
import pyLDAvis.sklearn
import time


#定义函数，加载txt文件
def readtxt(filepath,encoding='utf-8'):
    words=[line.strip() for line in open(filepath,mode='r',encoding=encoding).readlines()]
    return words

#定义分词函数
def cut_word(text):
    #加载用户自定义词典
    #jieba.load_userdict('user_dict.txt')
    #加载停用词表
    stopwords=readtxt('...\stopwords_cn.txt',encoding='utf-8')
    sentence=""
    checkarr=['n']
    for word,flag in peg.lcut(text):
        if (flag in checkarr) and (word not in stopwords) and (len(word)>1):
            sentence = sentence + word + " "
    return sentence     

#文本向量化
def word_vectorizer(n_features,max_df=0.5,min_df=3):
    cv = CountVectorizer(strip_accents = 'unicode',#将使用unicode编码在预处理步骤去除raw document中的重音符号
                                    max_features=n_features,
                                    max_df = 0.5,# 阈值如果某个词的document frequence大于max_df，不当作关键词
                                    min_df = 3 # 如果某个词的document frequence小于min_df，则这个词不会被当作关键词
                                    )
    return cv


def lda_model(k,max_iter=50,method='online',learning_offset=50.,random_state=0):  
    lda=LatentDirichletAllocation(n_components=k,max_iter=max_iter,
                                  learning_method=method,
                                  learning_offset=learning_offset,
                                  random_state=random_state)

    return lda

def print_keywords(lda,cv,therahold,p):
    weight_matrix=lda.components_
    tf_feature_names=cv.get_feature_names()
    id = 0
    for weights in weight_matrix:
        dicts = [(name, weight) for name, weight in zip(tf_feature_names, weights)]
        dicts = sorted(dicts, key=lambda x: x[1], reverse=True)#根据特征词的权重降序排列
        dicts = [word for word in dicts if word[1] > therahold]# 打印权重值大于0.6的主题词
        dicts = dicts[:p]# 打印每个主题前5个主题词
        print('主题%d:' % (id), dicts)
        id += 1

if __name__=='__main__':    
    #调用函数
    text=readtxt(r'...\data\reviews.txt')
    #分词
    segged_words=[cut_word(x) for x in text]
    print(segged_words[0])
    #向量化
    n_features = 1000# 指定特征关键词提取最大值
    cv=word_vectorizer(n_features)
    tf = cv.fit_transform(segged_words)#将评论关键字列表转换为词向量空间,TFIDF矩阵
    
    #构建lda模型
    time_start=time.time()
    lda=lda_model(4)
    ldamodel=lda.fit_transform(tf)
    time_end=time.time()
    print('time cost',time_end-time_start,'s')
    
    '''对于构建的词典，一些查看操作'''
    # #查看构建的词典
    # print(cv.vocabulary_)
    # #查看词典大小
    # print(len(cv.vocabulary_))
    # print(cv.get_feature_names())
    # #查看抽取出的特征词个数
    # print(len(cv.get_feature_names()))

    # #查看每个特征词在单个文摘中的词频
    # print(tf)
    # #查看全部文摘向量化表示的结果
    # print(tf.toarray())
    # #计算每个词在所有文摘中的累积词频
    # print(tf.toarray().sum(axis=0))

    # #根据累积词频，提取高频词
    # #（1）获取高频词的索引
    # fre=tf.toarray().sum(axis=0)
    # index_lst=[]
    # for i in range(len(fre)):
    #     if fre[i]>10:
    #         index_lst.append(i)
            
    # #(2)对词典按词频升序排列
    # voca=list(cv.vocabulary_.items())
    # sorted_voca=sorted(voca,key=lambda x:x[1],reverse=False)

    # #(3)提取高频词
    # high_fre_voca=[]
    # for i in sorted_voca:
    #     if i[1] in index_lst:
    #         high_fre_voca.append(i[0])
    # print(high_fre_voca)
    
    '''对于构建的lda模型，查看已有语料库属于各个主题的概率'''
    # #查看每个文摘属于各个主题的概率
    # proba=np.array(ldamodel)
    # print('每个文摘属于各个主题的概率：\n',proba)
    # #构建一个零矩阵
    # zero_matrix=np.zeros([proba.shape[0]])
    # # 对比所属概率的大小，确定属于的类别
    # max_proba = np.argmax(proba, axis=1) # 返回沿轴axis最大值的索引，axis=1代表行；最大索引即表示最可能表示的数字是多少
    # print('每个文档所属类别：', max_proba)

    #查看每个特征词属于各个主题的权重
    # weight_matrix=ldamodel.components_
    # print(weight_matrix)
    # print(len(weight_matrix))

    #打印每个主题前5个关键词【要求每个关键词的权重大于0.6】
    print_keywords(lda, cv, 0.6, 5)
    #可视化
    d=pyLDAvis.sklearn.prepare(lda, tf, cv)
    pyLDAvis.show(d)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132