基于出行住宿评论数据的情感分析研究（酒店篇，含python代码）_酒店爬取评论数据做情感分析

文章目录

前言

一、数据集介绍

二、数据预处理

1.引入库

2.剔除无价值数据

三、情感信息提取及可视化

四、使用线性支持向量分类模型进行情感预测

优化处理操作——向下采样：

五、使用LDA主题分类模型进行数据分析

1. 积极评论：

2. 消极评论：

六、结果分析及结论

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
 
import jieba 
import re
 
plt.rcParams["font.sans-serif"] = ["SimHei"]
 
plt.rcParams["axes.unicode_minus"] = False
import warnings
warnings.filterwarnings("ignore")

data["review"] = data["review"].astype('str')
 
import re
# 去除数字、携程等词语
strinfo = re.compile('[0-9]|酒店|携程|年月日|北京|上海|重庆|广州|杭州|南京|成都|苏州|西安|东莞|长沙|济南|深圳|西路|东路')
data["review"] =  data["review"].apply(lambda x: strinfo.sub('',x))
 
import re
# 由于有的时候jupyternotebook会出bug,去除一次并不能去除掉年月日，所以保险起见，再去除一次
strinfo = re.compile('[0-9]|酒店|携程|年月日')
data["review"] =  data["review"].apply(lambda x: strinfo.sub('',x))
 
#第一步 将空字符的行替换为nan，方便进行删除
data.replace(to_replace=r'^\s*$', value=np.nan, regex=True, inplace=True)
data.replace(to_replace=r'[a-zA-Z]', value=np.nan, regex=True, inplace=True)
print(data)
 
#第二步 删除所有值为nan的行
data.dropna(axis=0, how='any', inplace=True)

# 提取正负面评论信息
posdata = data[data['label'] == 1]['review']
negdata = data[data['label'] == 0]['review']

from wordcloud import WordCloud
import jieba
import matplotlib.pyplot as plt
import PIL
 
text = ''
for s in posdata:
    text += s
data_cut = ' '.join(jieba.lcut(text))
 
 
path = '停用词汇总.txt'
f = open(path,"r",encoding='utf-8').read()  #设置文件对象
 
 
word_cloud = WordCloud(font_path="simsun.ttc", 
                       background_color="white", 
                       stopwords=f
                       )
 
 
word_cloud.generate(data_cut)
 
plt.subplots(figsize=(12,8))
plt.imshow(word_cloud)
plt.axis("off")
plt.show()

from jieba import analyse 
key_words = jieba.analyse.extract_tags(sentence=text, topK=20, withWeight=True, allowPOS=())
key_words

from wordcloud import WordCloud
import jieba
import matplotlib.pyplot as plt
import PIL
 
text1 = ''
for s in negdata:
    text1 += s
data_cut = ' '.join(jieba.lcut(text1))
 
 
path = '停用词汇总.txt'
f = open(path,"r",encoding='utf-8').read()  #设置文件对象
 
 
word_cloud = WordCloud(font_path="simsun.ttc", 
                       background_color="white", 
                       stopwords=f
                       )
 
 
word_cloud.generate(data_cut)
 
plt.subplots(figsize=(12,8))
plt.imshow(word_cloud)
plt.axis("off")
plt.show()

from jieba import analyse 
key_words = jieba.analyse.extract_tags(sentence=text1, topK=20, withWeight=True, allowPOS=())
key_words

from sklearn.feature_extraction.text import TfidfVectorizer as TFIDF  # 原始文本转化为tf-idf的特征矩阵
from sklearn.svm import LinearSVC
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split
 
# 将有标签的数据集划分成训练集和测试集
train_X,valid_X,train_y,valid_y = train_test_split(data['review'],data['label'],test_size=0.3,random_state=42)
 
train_X.shape,train_y.shape,valid_X.shape,valid_y.shape

# 模型构建
model_tfidf = TFIDF(min_df=5, max_features=5000, ngram_range=(1,3), use_idf=1, smooth_idf=1)
# 学习idf vector
model_tfidf.fit(train_X)
# 把文档转换成 X矩阵（该文档中该特征词出现的频次），行是文档个数，列是特征词的个数
train_vec = model_tfidf.transform(train_X)
 
# 模型训练
model_SVC = LinearSVC()
clf = CalibratedClassifierCV(model_SVC)
clf.fit(train_vec,train_y)

# 把文档转换成矩阵
valid_vec = model_tfidf.transform(valid_X)
# 验证
pre_valid = clf.predict_proba(valid_vec)
 
pre_valid = clf.predict(valid_vec)
print('正例:',sum(pre_valid == 1))
print('负例:',sum(pre_valid == 0))

from sklearn.metrics import accuracy_score
 
score = accuracy_score(pre_valid,valid_y)
print("准确率:",score)

def get_balanced_words(size,
                       positive_comment=data[data['label'] == 1],
                       negtive_comment=data[data['label'] == 0]):
    word_size = size // 2
    #获取正负评论数
    num_pos = positive_comment.shape[0]
    num_neg = negtive_comment.shape[0]
    #     当 正(负)品论数中<采样数量/2 时，进行上采样，否则都是下采样；
    #     其中pandas的sample方法里的repalce参数代表是否进行上采样，默认不进行
    balanced_words = pd.concat([
        positive_comment.sample(word_size,
                                replace=num_pos < word_size,
                                random_state=0),
        negtive_comment.sample(word_size,
                               replace=num_neg < word_size,
                               random_state=0)
    ])
    #     打印样本个数
    print('样本总数：', balanced_words.shape[0])
    print('正样本数：', balanced_words[data['label'] == 1].shape[0])
    print('负样本数：', balanced_words[data['label'] == 0].shape[0])
    print('')
    return balanced_words
 
data_4888 = get_balanced_words(4888)

from sklearn.feature_extraction.text import TfidfVectorizer as TFIDF  # 原始文本转化为tf-idf的特征矩阵
from sklearn.svm import LinearSVC
from sklearn.calibration import CalibratedClassifierCV
from sklearn.model_selection import train_test_split
 
# 将有标签的数据集划分成训练集和测试集
train_X,valid_X,train_y,valid_y = train_test_split(data_4888['review'],data_4888['label'],test_size=0.3,random_state=23)
 
train_X.shape,train_y.shape,valid_X.shape,valid_y.shape

# 模型构建
model_tfidf = TFIDF(min_df=2, max_features=5000, ngram_range=(1,3), use_idf=1, smooth_idf=1)
# 学习idf vector
model_tfidf.fit(train_X)
# 把文档转换成 X矩阵（该文档中该特征词出现的频次），行是文档个数，列是特征词的个数
train_vec = model_tfidf.transform(train_X)
 
# 模型训练
model_SVC = LinearSVC()
clf = CalibratedClassifierCV(model_SVC)
clf.fit(train_vec,train_y)

# 把文档转换成矩阵
valid_vec = model_tfidf.transform(valid_X)
# 验证
pre_valid = clf.predict_proba(valid_vec)
 
pre_valid = clf.predict(valid_vec)
print('正例:',sum(pre_valid == 1))
print('负例:',sum(pre_valid == 0))

from sklearn.metrics import accuracy_score
 
score = accuracy_score(pre_valid,valid_y)
print("准确率:",score)

posdata=pd.DataFrame(posdata).dropna(axis=0)
posdata.columns=['comment']
 
negdata=pd.DataFrame(negdata).dropna(axis=0)
negdata.columns=['comment']

#利用jieba中文分词 
import jieba
import jieba.posseg as psg
 
#格式转换 否则会报错  'float' object has no attribute 'decode'
df1 = pd.DataFrame(posdata.astype(str))
 
def chinese_word_cut(mytext):
    return ' '.join(jieba.cut(mytext))
 
#增加一列数据
df1['content_cutted'] = df1['comment'].apply(chinese_word_cut)
 
 
 
#格对负面评论进行操作
df2 = pd.DataFrame(negdata.astype(str))
 
def chinese_word_cut(mytext):
    return ' '.join(jieba.cut(mytext))
 
#增加一列数据
df2['content_cutted'] = df2['comment'].apply(chinese_word_cut)

path = '停用词汇总.txt'
f = open(path,"r",encoding='utf-8').read()
stopwords=list(f)
 
#计算TF-IDF值
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
 
#设置特征数
n_features = 2000
 
 
tf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',
                                max_features=n_features,
                                stop_words=stopwords,
                                max_df = 0.99,
                                min_df = 0.002) #去除文档内出现几率过大或过小的词汇
tf = tf_vectorizer.fit_transform(df1.content_cutted)
 
print(tf.shape)
print(tf)

#LDA分析
from sklearn.decomposition import LatentDirichletAllocation
 
#设置主题数
n_topics = 3
 
lda = LatentDirichletAllocation(n_components=n_topics,
                                max_iter=100,
                                learning_method='online',
                                learning_offset=50,
                                random_state=0)
lda.fit(tf)
 
#显示主题数 model.topic_word_
print(lda.components_)
#几个主题就是几行 多少个关键词就是几列 
print(lda.components_.shape)                         
   
 
#主题-关键词分布
def print_top_words(model, tf_feature_names, n_top_words):
    for topic_idx,topic in enumerate(model.components_):    # lda.component相当于model.topic_word_
        print('Topic #%d:' % topic_idx)
        print(' '.join([tf_feature_names[i] for i in topic.argsort()[:-n_top_words-1:-1]]))
        print("")
 
#定义好函数之后 暂定每个主题输出前20个关键词
n_top_words = 20                                       
tf_feature_names = tf_vectorizer.get_feature_names()
#调用函数
print_top_words(lda, tf_feature_names, n_top_words)

import pyLDAvis
import pyLDAvis.gensim_models
red_vis_data = pyLDAvis.sklearn.prepare(lda,tf,tf_vectorizer)
pyLDAvis.display(red_vis_data)

#计算TF-IDF值
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
 
 
n_features = 2000
 
 
tf_vectorizer = TfidfVectorizer(strip_accents = 'unicode',
                                max_features=n_features,
                                stop_words=stopwords,
                                max_df = 0.99,
                                min_df = 0.002)
tf = tf_vectorizer.fit_transform(df2.content_cutted)
 
print(tf.shape)
print(tf)

#LDA分析
from sklearn.decomposition import LatentDirichletAllocation
 
#设置主题数
n_topics = 2
 
lda = LatentDirichletAllocation(n_components=n_topics,
                                max_iter=100,
                                learning_method='online',
                                learning_offset=50,
                                random_state=0)
lda.fit(tf)
 
#显示主题数 model.topic_word_
print(lda.components_)
#几个主题就是几行 多少个关键词就是几列 
print(lda.components_.shape)                         
   
 
#主题-关键词分布
def print_top_words(model, tf_feature_names, n_top_words):
    for topic_idx,topic in enumerate(model.components_):    # lda.component相当于model.topic_word_
        print('Topic #%d:' % topic_idx)
        print(' '.join([tf_feature_names[i] for i in topic.argsort()[:-n_top_words-1:-1]]))
        print("")
 
#定义好函数之后 暂定每个主题输出前20个关键词
n_top_words = 20                                       
tf_feature_names = tf_vectorizer.get_feature_names()
#调用函数
print_top_words(lda, tf_feature_names, n_top_words)

import pyLDAvis
import pyLDAvis.gensim_models
red_vis_data = pyLDAvis.sklearn.prepare(lda,tf,tf_vectorizer)
pyLDAvis.display(red_vis_data)