NLP（9）--rnn实现中文分词

前言

仅记录学习过程，有问题欢迎讨论

• pip install jieba

代码

import jieba
import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader

"""
基于pytorch的网络编写一个分词模型
我们使用jieba分词的结果作为训练数据
看看是否可以得到一个效果接近的神经网络模型

中文分词缺点：
1.对词表极为依赖，如果没有词表，则无法进行；如果词表中缺少需要的词，结果也不会正确
2.切分过程中不会关注整个句子表达的意思，只会将句子看成一个个片段
3.如果文本中出现一定的错别字，会造成一连串影响
4.对于人名等的无法枚举实体词无法有效的处理

"""


class TorchModel(nn.Module):
    def __init__(self, vocab, input_dim, hidden_size, rnn_layer_size):
        super(TorchModel, self).__init__()

        self.emb = nn.Embedding(len(vocab) + 1, input_dim)
        # 多层rnn效果会比 单层好
        self.rnn = nn.RNN(input_size=input_dim,
                          hidden_size=hidden_size,
                          num_layers=rnn_layer_size,
                          batch_first=True)
        # 不能使用pool
        # self.pool = nn.AvgPool1d(sentence_length)
        # 输出为0/1 2分类的
        self.classify = nn.Linear(hidden_size, 2)
        # -1 不参与计算
        self.loss = nn.CrossEntropyLoss(ignore_index=-1)

    def forward(self, x, y=None):
        x = self.emb(x)
        x, _ = self.rnn(x)
        # 用 polling 层
        # x= self.pool(x.transpose(1,2)).squeeze()
        y_pred = self.classify(x)
        if y is not None:
            # y_pred : n,class_num [[1,2,3][3,2,1]]
            # y : n                 [0     ,1     ]
            # 20*20*2===>view ===> 400 * 2   y===> 400 *1
            return self.loss(y_pred.view(-1, 2), y.view(-1))
        else:
            return y_pred


# 使用jieba获取切分好的数据 来作为样本数据
# 我爱你们 === 1,1,0,1
def sequence_to_label(sentence):
    words = jieba.lcut(sentence)
    labels = [0] * len(sentence)
    pointer = 0
    for word in words:
        pointer += len(word)
        labels[pointer - 1] = 1
    return labels


# 读取给定词表数据 构建字符集
def build_vocab(path):
    vocab = {}
    with open(path, encoding="utf8") as f:
        for index, line in enumerate(f):
            char = line.strip()
            vocab[char] = index + 1
        vocab['unk'] = len(vocab) + 1
    return vocab


class Dataset:
    def __init__(self, vocab, corpus_path, max_length):
        self.vocab = vocab
        self.corpus_path = corpus_path
        self.max_length = max_length
        self.load()

    # 构建数据集
    def load(self):
        # data 的结构为 [x,y]
        self.data = []
        with open(self.corpus_path, encoding="utf8") as f:
            for line in f:
                vocab = self.vocab
                # 转化为 切分好的数据 y
                y = sequence_to_label(line)
                # 转化为数字
                x = [vocab.get(char, vocab['unk']) for char in line]
                # 都 标准化为最大长度
                x, y = self.padding(x, y)
                self.data.append([torch.LongTensor(x), torch.LongTensor(y)])
                # 使用部分数据做展示，使用全部数据训练时间会相应变长
                if len(self.data) > 10000:
                    break

    def padding(self, x, y):
        # 长了就截取
        x = x[:self.max_length]
        # 短了就 补0
        x += [0] * (self.max_length - len(x))
        y = y[:self.max_length]
        # y 不能用 0
        y += [-1] * (self.max_length - len(y))
        return x, y

    # 为了给 data_load 使用 做小批量数据分割
    def __len__(self):
        return len(self.data)

    def __getitem__(self, item):
        return self.data[item]


def build_dataset(vocab, corpus_path, max_length, batch_size):
    dataset = Dataset(vocab, corpus_path, max_length)
    # shuffle 随机打乱样本
    data_loader = DataLoader(dataset, shuffle=True, batch_size=batch_size)  # torch
    return data_loader


def main():
    batch_size = 20
    lr = 1e-3
    epoch_size = 10
    vocab = build_vocab("D:\\NLP\\test\\week4\\chars.txt")
    hidden_size = 100
    # 每个字符的维度
    input_dim = 20
    rnn_layer_size = 2
    # 样本最大长度
    max_length = 20
    model = TorchModel(vocab, input_dim, hidden_size, rnn_layer_size)
    optim = torch.optim.Adam(model.parameters(), lr=lr)
    # 语料库（样本数据）路径
    corpus_path = "D:\\NLP\\test\\week4\\corpus.txt"

    dataiter = build_dataset(vocab, corpus_path, max_length, batch_size)

    for epoch in range(epoch_size):
        epoch_loss = []
        model.train()
        for x, y_true in dataiter:
            loss = model(x, y_true)
            loss.backward()
            optim.step()
            optim.zero_grad()
            epoch_loss.append(loss.item())
        print("第%d轮 loss = %f" % (epoch + 1, np.mean(epoch_loss)))
    # save model
    torch.save(model.state_dict(), "model.pth")
    return


# 最终预测
def predict(model_path, vocab_path, input_strings):
    # 配置保持和训练时一致
    char_dim = 20  # 每个字的维度
    hidden_size = 100  # 隐含层维度
    num_rnn_layers = 2  # rnn层数
    vocab = build_vocab(vocab_path)  # 建立字表（字符集）
    model = TorchModel(vocab, char_dim, hidden_size, num_rnn_layers)  # 建立模型
    model.load_state_dict(torch.load(model_path))  # 加载训练好的模型权重
    model.eval()
    for input_string in input_strings:
        # 逐条预测
        x = [vocab.get(char, vocab['unk']) for char in input_string]
        with torch.no_grad():
            result = model.forward(torch.LongTensor([x]))[0]
            result = torch.argmax(result, dim=-1)  # 预测出的01序列
            # 在预测为1的地方切分，将切分后文本打印出来
            for index, p in enumerate(result):
                if p == 1:
                    print(input_string[index], end=" ")
                else:
                    print(input_string[index], end="")
            print()


if __name__ == '__main__':
    main()

    # input_strings = ["同时国内有望出台新汽车刺激方案",
    #                  "沪胶后市有望延续强势",
    #                  "经过两个交易日的强势调整后",
    #                  "昨日上海天然橡胶期货价格再度大幅上扬"]
    # predict("model.pth", "D:\\NLP\\test\\week4\\chars.txt", input_strings)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191