AI实战：从入门到精通系列——用感知器实现情感分类（二）_weibo100k数据集

作者：从前慢现在也慢 | 2024-05-02 03:54:37

踩

weibo100k数据集

AI实战：从入门到精通系列——用感知器实现情感分类（一）
AI实战：从入门到精通系列——用感知器实现情感分类（二）

用感知器实现情感分类

环境
- Ubuntu16.04 LTS
- python 3.x
- numpy
- gensim
- jieba
数据集

weibo_senti_100k 微博情感数据集

查看数据信息

import pandas as pd


pd_all = pd.read_csv('./data/weibo_senti_100k.csv')

print('评论数目（总体）：%d' % pd_all.shape[0])
print('评论数目（正向）：%d' % pd_all[pd_all.label==1].shape[0])
print('评论数目（负向）：%d' % pd_all[pd_all.label==0].shape[0])

print(pd_all.sample(10))

1
2
3
4
5
6
7
8
9
10
11

输出
数据信息及样式：

数据集拆分为train、test

import os
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.utils import shuffle
 
def train_valid_test_split(x_data, y_data, test_size=0.1, shuffle=True):
    x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=test_size, shuffle=shuffle)
    
    return x_train, x_test, y_train, y_test
 
if __name__ == '__main__':
    pd_all = pd.read_csv("./data/weibo_senti_100k.csv")
    pd_all = shuffle(pd_all)
    x_data, y_data = pd_all.review, pd_all.label
 
    x_train, x_test, y_train, y_test = train_valid_test_split(x_data, y_data, 0.1)
 
    train = pd.DataFrame({'label':y_train, 'x_train': x_train})
    train.to_csv("./data/train.csv", index=False, sep='\t')
    test = pd.DataFrame({'label':y_test, 'x_test': x_test})
    test.to_csv("./data/test.csv", index=False, sep='\t')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

结果如下
```
test.csv
train.csv
1
2
```

生成词向量

def train_word2vec(train_path, test_path):
    import multiprocessing
    from gensim.models import Word2Vec
    from gensim.models.word2vec import LineSentence

    #加载数据
    pd_all = pd.read_csv(train_path)
    train_data_x, train_data_y = pd_all.text.tolist(), pd_all.label.tolist()
    
    pd_all = pd.read_csv(test_path)
    test_data_x, test_data_y = pd_all.text.tolist(), pd_all.label.tolist()
    
    #分词
    train_data_x = [segment(k) for k in train_data_x]
    test_data_x = [segment(k) for k in test_data_x]
    
    text = test_data_x + train_data_x
    
    
    input_file_name = './data/sentence.txt'
    open(input_file_name, 'w').write('\n'.join(text))

    model = Word2Vec(LineSentence(input_file_name),
                     word2vec_dimension = 100, # 词向量维度为100
                     window = 5,
                     min_count = 1,
                     workers = multiprocessing.cpu_count())

    model.save('./data/word2vec.model')

def segment(text):
    words = jieba.cut(text)
    words = list(words)[:max_text_len]
    return ' '.join(words)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

执行：

train_path = './data/train.csv'
test_path = './data/test.csv'
train_word2vec(train_path, test_path)
1
2
3

输出：
./data/word2vec.model
./data/ word2vec.model.trainables.syn1neg.npy
./data/word2vec.model.wv.vectors.npy

测试词向量：

from gensim.models import Word2Vec
model = Word2Vec.load("./data/word2vec.model")
vector = model.wv['好吃']
1
2
3

输出：

[-0.73283386  0.86413544  0.75687975 -0.5055297  -0.42515302  0.18348737
  2.3790839   1.0248554   2.101729   -0.4618316   0.43203285 -0.5404889
 -1.017284   -2.2938926   2.3901055  -0.69351804  1.6101934  -0.59710294
 -0.03683157  0.57503146 -1.250644    2.980576    1.1501396  -0.81633765
  0.6402967   2.3379786  -0.877263   -1.9016323   1.1057235   0.06841037
 -0.05232436 -0.08345098 -0.30685595 -1.1040177  -1.7571559  -1.7910484
 -0.7331836   0.1513317  -0.621015    0.8975967   2.5499363   1.1568907
  0.3688762  -0.5182226  -0.30297205  0.5822141  -1.0808538  -0.01062215
 -1.4400197  -2.2848194   2.1822946   0.15740396  1.0032029  -0.8410342
 -1.1311824  -0.33163172  1.3151053  -0.2986618   1.9608823  -0.2602172
  0.63158864  1.239699    0.10924603 -1.7023547  -1.554196    0.03117983
  0.6561903  -0.4397268  -1.9914472   0.79718435 -1.4864717  -2.9809866
 -0.46287113  0.4837672  -0.71872777  2.4697163  -0.53781223  0.23790799
  2.0566401   1.6394123  -0.9129417   1.5896504   1.5701648   1.1339688
 -1.8522842   2.0832975  -1.9120314  -0.23889321  2.8850334   0.70530176
  1.6666977  -1.0355597   0.36848044 -0.02313641 -1.3314507  -0.52943283
  0.29032257 -1.952622   -0.674098   -0.20572844]
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

数据处理
data_helper.py

import numpy as np
import pandas as pd
import jieba
import multiprocessing
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence

max_text_len = 50#50个词
word2vec_dimension = 100

def load_train_test_data(train_path, test_path, w2v_model_path):
    
    #加载数据
    pd_all = pd.read_csv(train_path)
    train_data_x, train_data_y = pd_all.text.tolist(), pd_all.label.tolist()
    
    pd_all = pd.read_csv(test_path)
    test_data_x, test_data_y = pd_all.text.tolist(), pd_all.label.tolist()
    
    train_data_x = train_data_x[:5000]#数据太大了，只取5000个作为训练【我的电脑内存不足，故仅取部分数据作为训练集】
    
    #分词
    train_data_x = [segment(k) for k in train_data_x]
    test_data_x = [segment(k) for k in test_data_x]
    
    
    #文本转向量表示
    w2v_model = Word2Vec.load(w2v_model_path)
    train_data_x = text_to_word2vec(w2v_model, train_data_x)
    test_data_x = text_to_word2vec(w2v_model, test_data_x)
    
    return train_data_x, test_data_x, train_data_y, test_data_y

def text_to_word2vec(w2v_model, text_list):
    #文本转为向量表示
    
    text_array = np.zeros((len(text_list), max_text_len, word2vec_dimension))
    i = 0
    while i < len(text_list):
        words = text_list[i].split(' ')
        for index, word in enumerate(words):
            if index >= max_text_len:
                break
            if word in w2v_model.wv:
                text_array[i, index] = w2v_model.wv[word]
            else:
                text_array[i, index] = [0.0]*word2vec_dimension
        i += 1
    
    return text_array
    
def segment(text):
	#分词
	
    words = jieba.cut(text)
    words = list(words)[:max_text_len]
    
    return ' '.join(words)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

说明：
【我的电脑内存不足，故仅取部分数据作为训练集】
删除下列这行代码即可：
train_data_x = train_data_x[:5000]#数据太大了，只取5000个作为训练

训练感知器
train.py

import os, sys
import data_helper
import perception


def train(train_path, test_path, w2v_model_path):
    
    x_train, x_test, y_train, y_test = data_helper.load_train_test_data(train_path, test_path, w2v_model_path)
    print(x_train[0], y_train[0])
    
    p = perception.Perceptron(data_helper.max_text_len, data_helper.word2vec_dimension, perception.f)
    
    # 训练迭代100轮, 学习率为0.05
    p.train(x_train, y_train, 100, 0.05)
    
    # 打印权重
    p.print_weights()
    
    #测试
    p.test(x_test, y_test)
    

if __name__ == "__main__":  
    train_path = './data/train.csv'
    test_path = './data/test.csv'
    w2v_model_path = './data/word2vec.model'
    train(train_path, test_path, w2v_model_path)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

执行：python3 train.py

输出：

weights [[-1.42558697 -2.48767451  1.28752376 ... -0.78376229  3.16459166
  -3.28389434]
 [-3.68905224 -4.80877013 -3.13396478 ... -4.25494364 -3.01798689
  -4.91744347]
 [ 1.94075086 -1.94479774  5.51378438 ... -5.19175698 -4.50725763
   3.28213941]
 ...
 [-0.60414949  0.84948442  3.2864892  ... -3.96489623  0.9902426
   7.86129972]
 [-0.52215719 -2.85837685 -0.89045009 ... -1.01795905 -1.21213078
  -0.16342622]
 [-0.12955836 -3.43814853  0.094599   ... -2.52779952 -4.71311826
  -1.97031286]]
bias 4.099999999999993
acc: 81.89%	
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15

结果说明

可以看到使用5000条数据训练微博情感分类的感知器模型，在5000条测试数据集上面的准确率为81.89%，效果还是可以的。

参考

训练词向量：https://radimrehurek.com/gensim/models/word2vec.html

上一篇：AI实战：从入门到精通系列——用感知器实现情感分类（一）

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/从前慢现在也慢/article/detail/522073