Python自然语言处理实战（8）：情感分析技术_自然语言boson情感分析实操

实战电影评论情感分析

        情感分析是一段文字表达的情绪状态。其中，一段文本可以使一个句子、一个段落或者一个文档。主要涉及两个问题：文本表达和文本分类。在深度学习出现之前，主流的表示方法有BOW(词袋模型)和topic model(主题模型)，分类模型主要有SVM和LR。

         载入数据：IMDB情感分析数据集，训练集和测试集分别包含了25000条已标注的电影评论，满分了10分，小于等于4为负面评论。

# -*- coding: utf-8 -*-
 
import numpy as np 
# 加载已训练好的词典向量模型，包含400000的文本向量，每行有50维的数据
words_list = np.load('wordsList.npy')
print('载入word列表')
words_list = words_list.tolist()   # 转化为list
words_list = [word.decode('UTF-8') for word in words_list]
word_vectors = np.load('wordVectors.npy')
print('载入文本向量')
 
print(len(words_list))
print(word_vectors.shape)
 
Home_index = words_list.index("home")
print(word_vectors[Home_index])
 
# 加载电影数据
import os
from os.path import isfile, join
pos_files = ['pos/' + f for f in os.listdir('pos/') if isfile(join('pos/', f))]
neg_files = ['neg/' + f for f in os.listdir('neg/') if isfile(join('neg/', f))]
num_words = []
for pf in pos_files:
	with open(pf, "r", encoding='utf-8') as f:
		line = f.readline()
		counter = len(line.split())
		num_words.append(counter)
print('正面评价完结')
 
for pf in neg_files:
	with open(pf, "r", encoding='utf-8') as f:
		line = f.readline()
		counter = len(line.split())
		num_words.append(counter)
print('负面评价完结')
 
num_files = len(num_words)
print('文件总数', num_files)
print('所有的词的数量', sum(num_words))
print('平均文件词的长度', sum(num_words)/len(num_words))
 
'''
# 可视化
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use('qt4agg')
# 指定默认字体
matplotlib.rcParams['font.sans-serif'] = ['SimHei']
matplotlib.rcParams['font.family'] = 'sans-serif'
#%matplotlib inline
plt.hist(num_words, 50, facecolor='g')
plt.xlabel('文本长度')
plt.ylabel('频次')
plt.axis([0, 1200, 0, 8000])
plt.show()
'''
 
# 大部分文本都在230之内
max_seg_len = 300
 
# 将文本生成一个索引矩阵，得到一个25000x300矩阵
import re
strip_special_chars = re.compile("[^A-Za-z0-9 ]+")
 
def cleanSentence(string):
	string = string.lower().replace("<br />", " ")
	return re.sub(strip_special_chars, "", string.lower())
print('保存idxMatrix...')
max_seg_num = 300
ids = np.zeros((num_files, max_seg_num), dtype="int32")
file_count = 0
'''
for pf in pos_files:
	with open(pf, "r", encoding="utf-8") as f:
		indexCounter = 0
		line = f.readline()
		cleanedLine = cleanSentence(line)
		split = cleanedLine.split()
		for word in split:
			try:
				ids[file_count][indexCounter] = words_list.index(word)
			except ValueError:
				ids[file_count][indexCounter] = 399999 # 未知的词
			indexCounter = indexCounter + 1
			if indexCounter >= max_seg_num:
				break
		file_count = file_count + 1
		print(file_count)
print('保存完成1')
for nf in neg_files:
	with open(nf, "r", encoding="utf-8") as f:
		indexCounter = 0
		line = f.readline()
		cleanedLine = cleanSentence(line)
		split = cleanedLine.split()
		for word in split:
			try:
				ids[file_count][indexCounter] = words_list.index(word)
			except ValueError:
				ids[file_count][indexCounter] = 399999 # 未知的词
			indexCounter = indexCounter + 1
			if indexCounter >= max_seg_num:
				break
		file_count = file_count + 1
# 保存到文件
np.save('idxMatrix', ids)
print('保存完成2')
'''
 
# 模型设置
batch_size = 24
lstm_units = 64
num_labels = 2
iterations = 200000
max_seg_num = 250
ids = np.load('idsMatrix.npy')
 
# 返回一个数据集的迭代器， 返回一批训练集合
from random import randint
def get_train_batch():
	labels = []
	arr = np.zeros([batch_size, max_seg_num])
	for i in range(batch_size):
		if (i % 2 == 0):
			num = randint(1, 11499)
			labels.append([1, 0])
		else:
			num = randint(13499, 24999)
			labels.append([0, 1])
		arr[i] = ids[num-1: num]
	return arr, labels
 
def get_test_batch():
	labels = []
	arr = np.zeros([batch_size, max_seg_num])
	for i in range(batch_size):
		num = randint(11499, 13499)
		if (num <= 12499):
			labels.append([1, 0])
		else:
			labels.append([0, 1])
		arr[i] = ids[num-1:num]
	return arr, labels
 
num_dimensions = 300  # Dimensions for each word vector
 
import tensorflow as tf
tf.reset_default_graph()
labels = tf.placeholder(tf.float32, [batch_size, num_labels])
input_data = tf.placeholder(tf.int32, [batch_size, max_seg_num])
 
data = tf.Variable(tf.zeros([batch_size, max_seg_num, num_dimensions]), dtype=tf.float32)
data = tf.nn.embedding_lookup(word_vectors, input_data)
 
# 配置LSTM网络
lstmCell = tf.contrib.rnn.BasicLSTMCell(lstm_units)
lstmCell = tf.contrib.rnn.DropoutWrapper(cell=lstmCell, output_keep_prob=0.75) # 避免一些过拟合
value, _ = tf.nn.dynamic_rnn(lstmCell, data, dtype=tf.float32)
 
# 第一个输出可以被认为是最后的隐藏状态，该向量将重新确定维度，然后乘以一个权重加上偏置，获得最终的label
weight = tf.Variable(tf.truncated_normal([lstm_units, num_labels]))
bias = tf.Variable(tf.constant(0.1, shape=[num_labels]))
value = tf.transpose(value, [1, 0, 2])
last = tf.gather(value, int(value.get_shape()[0]) - 1)
prediction = (tf.matmul(last, weight) + bias)
 
# 预测函数以及正确率评估参数
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
 
# 将标准的交叉熵损失函数定义为损失值，选择Adam作为优化函数
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=labels))
optimizer = tf.train.AdamOptimizer().minimize(loss)
 
#sess = tf.InteractiveSession(config=tf.ConfigProto(allow_soft_placement, log_device_placement))
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False))
 
#saver = tf.train.Saver()
#saver.restore(sess, tf.train.latest_checkpoint('models'))
 
iterations = 10
for i in range(iterations):
  next_batch, next_batch_labels = get_test_batch()
  print("正确率:", (sess.run(
      accuracy, {input_data: next_batch, labels: next_batch_labels})) * 100)
 
'''
# 使用tensorboard可视化损失值和正确值
import datetime
sess = tf.InteractiveSession()
#tf.device("/cpu:0")
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
tf.summary.scalar('Loss', loss)
tf.summary.scalar('Accuracy', accuracy)
merged = tf.summary.merge_all()
logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
writer = tf.summary.FileWriter(logdir, sess.graph)
for i in range(iterations):
	# 下个批次的数据
	nextBatch, nextBatchLabels = get_train_batch();
	sess.run(optimizer, {input_data: nextBatch, labels: nextBatchLabels})
	# 每50次写入一次leadboard
	if (i % 50 == 0):
		summary = sess.run(merged, {input_data: nextBatch, labels: nextBatchLabels})
		writer.add_summary(summary, i)
	# 每10,000次保存一个模型
	if (i % 10000 == 0 and i != 0):
		save_path = saver.save(sess, "models/pretrained_lstm.ckpt", global_step=i)
		print("saved to %s" % save_path)
writer.close()
'''