devbox
菜鸟追梦旅行
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

NLP之文本分类_nlp文本分类

作者：菜鸟追梦旅行 | 2024-04-05 00:25:45

赞

踩

nlp文本分类
注意：本实验在百度平台的Al Studio运行

1.什么是文本分类

文本分类就是根据文本内容将文本划分到不同类别，例如新闻系统中，每篇新闻报道会划归到不同的类别。

2.文本分类的应用

  • 内容分类（新闻分类）

  • 邮件过滤（例如垃圾邮件过滤）

  • 用户分类（如商城消费级别、喜好）

  • 评论、文章、对话的情感分类（正面、负面、中性）

3.文本分类案例

  • 任务：建立文本分类模型，并对模型进行训练、评估，从而实现对中文新闻摘要类别正确划分

  • 数据集：从网站上爬取56821条数据中文新闻摘要，包含10种类别，国际、文化、娱乐、体育、财经、汽车、教育、科技、房产、证券，各类别样本数量如下表所示：

  • 模型选择：

  • 步骤：

  • 代码

    【预处理部分】

    1. ########################### 数据预处理　#########################
    2. import os
    3. from multiprocessing import cpu_count
    4. import numpy as np
    5. import paddle
    6. import paddle.fluid as fluid
    7.  
    8. #　定义一组公共变量
    9. data_root = "data/" # 数据集所在目录
    10. data_file = "news_classify_data.txt" # 原始数据集
    11. train_file = "train.txt" # 训练集文件
    12. test_file = "test.txt" # 测试集文件
    13. dict_file = "dict_txt.txt" # 字典文件(存放字和编码映射关系)
    14.  
    15. data_file_path = data_root + data_file # 数据集完整路径
    16. train_file_path = data_root + train_file # 训练集文件完整路径
    17. test_file_path = data_root + test_file # 测试集文件完整路径
    18. dict_file_path = data_root + dict_file # 字典文件完整路径
    19.  
    20. # 取出样本中所有字，对每个字进行编码，将编码结果存入字典文件
    21. def create_dict():
    22.     dict_set = set() # 集合，用作去重
    23.     with open(data_file_path, "r", encoding="utf-8") as f:
    24.         for line in f.readlines(): # 遍历每行
    25.             line = line.replace("\n", "") # 去除换行符
    26.             tmp_list = line.split("_!_") # 根据分隔符拆分
    27.             title = tmp_list[-1] # 最后一个字段即为标题
    28.             for word in title: # 取出每个字
    29.                 dict_set.add(word)
    30.  
    31.     # 遍历集合，取出每个字进行编号
    32.     dict_txt = {} # 定义字典
    33.     i = 1 # 编码使用的计数器
    34.     for word in dict_set:
    35.         dict_txt[word] = i # 字-编码 键值对添加到字典
    36.         i += 1
    37.  
    38.     dict_txt["<unk>"] = i # 未知字符(在样本中未出现过的字)
    39.  
    40.     # 将字典内容存入文件
    41.     with open(dict_file_path, "w", encoding="utf-8") as f:
    42.         f.write(str(dict_txt))
    43.  
    44.     print("生成字典结束.")
    45.  
    46. # 传入一个句子，将每个字替换为编码值，和标签一起返回
    47. def line_encoding(title, dict_txt, label):
    48.     new_line = "" # 编码结果
    49.     for word in title:
    50.         if word in dict_txt: # 在字典中
    51.             code = str(dict_txt[word]) # 取出编码值
    52.         else: # 不在字典中
    53.             code = str(dict_txt["<unk>"]) # 取未知字符编码值
    54.         new_line = new_line + code + "," # 追加到字符串后面
    55.     new_line = new_line[:-1] # 去掉最后一个多余的逗号
    56.     new_line = new_line + "\t" + label + "\n" # 追加标签值
    57.     return new_line
    58.  
    59. # 读取原始样本，取出标题部分进行编码，将编码后的划分测试集/训练集
    60. def create_train_test_file():
    61.     # 清空训练集/测试集
    62.     with open(train_file_path, "w") as f:
    63.         pass
    64.     with open(test_file_path, "w") as f:
    65.         pass
    66.  
    67.     # 读取字典文件
    68.     with open(dict_file_path, "r", encoding="utf-8") as f_dict:
    69.         dict_txt = eval(f_dict.readlines()[0]) # 读取字典文件第一行，生成字典对象
    70.  
    71.     # 读取原始样本
    72.     with open(data_file_path, "r", encoding="utf-8") as f_data:
    73.         lines = f_data.readlines()
    74.  
    75.     i = 0
    76.     for line in lines:
    77.         tmp_list = line.replace("\n", "").split("_!_") # 拆分
    78.         title = tmp_list[3] # 标题
    79.         label = tmp_list[1] # 类别
    80.         new_line = line_encoding(title, dict_txt, label) # 对标题编码
    81.  
    82.         if i % 10 == 0: # 写入测试集
    83.             with open(test_file_path, "a", encoding="utf-8") as f:
    84.                 f.write(new_line)
    85.         else: # 写入训练集
    86.             with open(train_file_path, "a", encoding="utf-8") as f:
    87.                 f.write(new_line)
    88.         i += 1
    89.     print("生成训练集/测试集结束.")
    90.  
    91. create_dict() # 根据样本生成字典
    92. create_train_test_file()

    输出：

    1. 生成字典结束.
    2. 生成训练集/测试集结束.

    【模型定义与训练】

    1. paddle.enable_static()
    2. # 读取字典文件，返回字典长度
    3. def get_dict_len(dict_path):
    4.     with open(dict_path, "r", encoding="utf-8") as f:
    5.       dict_txt = eval(f.readlines()[0])
    6.     return len(dict_txt.keys())
    7.  
    8. def data_mapper(sample):
    9.     data, label = sample # 赋值到变量
    10.     val = [int(w) for w in data.split(",")] # 将编码值转换位数字(从文件读取为字符串)
    11.     return val, int(label)
    12.  
    13. def train_reader(train_file_path): # 训练集读取器
    14.     def reader():
    15.         with open(train_file_path, "r") as f:
    16.             lines = f.readlines()
    17.             np.random.shuffle(lines) # 随机化处理
    18.             for line in lines:
    19.                 data, label = line.split("\t") # 拆分
    20.                 yield data, label
    21.     return paddle.reader.xmap_readers(data_mapper, reader, cpu_count(), 1024)
    22.  
    23. def test_reader(test_file_path): # 训练集读取器
    24.     def reader():
    25.         with open(test_file_path, "r") as f:
    26.             lines = f.readlines()
    27.             
    28.             for line in lines:
    29.                 data, label = line.split("\t") # 拆分
    30.                 yield data, label
    31.     return paddle.reader.xmap_readers(data_mapper, reader, cpu_count(), 1024)
    32.  
    33. # 定义网络
    34. def Text_CNN(data, dict_dim, class_dim=10, emb_dim=128,
    35.              hid_dim=128, hid_dim2=128):
    36.     """
    37.     定义TextCNN模型
    38.     :param data:　输入
    39.     :param dict_dim:　词典大小(词语总的数量)
    40.     :param class_dim:　分类的数量
    41.     :param emb_dim: 词嵌入长度
    42.     :param hid_dim:　第一个卷基层卷积核数量
    43.     :param hid_dim2:　第二个卷基层卷积核数量
    44.     :return:　模型预测结果
    45.     """
    46.     # embedding层
    47.     emb = fluid.layers.embedding(input=data, size=[dict_dim, emb_dim])
    48.     # 并列两个卷积/池化层
    49.     conv1 = fluid.nets.sequence_conv_pool(input=emb, # 输入(词嵌入层输出)
    50.                                           num_filters=hid_dim,#　卷积核数量
    51.                                           filter_size=3,#卷积核大小
    52.                                           act="tanh",#激活函数
    53.                                           pool_type="sqrt")#池化类型
    54.     conv2 = fluid.nets.sequence_conv_pool(input=emb, # 输入(词嵌入层输出)
    55.                                           num_filters=hid_dim2,#　卷积核数量
    56.                                           filter_size=4,#卷积核大小
    57.                                           act="tanh",#激活函数
    58.                                           pool_type="sqrt")#池化类型
    59.     # fc
    60.     output = fluid.layers.fc(input=[conv1, conv2], # 输入
    61.                              size=class_dim,#输出值个数
    62.                              act="softmax")#激活函数
    63.     return output
    64.  
    65. # 定义占位符张量
    66. words = fluid.layers.data(name="words",
    67.                           shape=[1],
    68.                           dtype="int64",
    69.                           lod_level=1) # LOD张量用来表示变长数据
    70. label = fluid.layers.data(name="label",
    71.                           shape=[1],
    72.                           dtype="int64")
    73. dict_dim = get_dict_len(dict_file_path) # 获取字典长度
    74. # 调用模型函数
    75. model = Text_CNN(words, dict_dim)
    76. # 损失函数
    77. cost = fluid.layers.cross_entropy(input=model, label=label)
    78. avg_cost = fluid.layers.mean(cost)
    79. # 优化器
    80. optimizer = fluid.optimizer.Adam(learning_rate=0.0001)
    81. optimizer.minimize(avg_cost)
    82. # 准确率
    83. accuracy = fluid.layers.accuracy(input=model, label=label)
    84.  
    85. # 执行器
    86. place = fluid.CUDAPlace(0)
    87. exe = fluid.Executor(place)
    88. exe.run(fluid.default_startup_program())
    89.  
    90. # reader
    91. ## 训练集reader
    92. tr_reader = train_reader(train_file_path)
    93. batch_train_reader = paddle.batch(tr_reader, batch_size=128)
    94. ## 测试集reader
    95. ts_reader = test_reader(test_file_path)
    96. batch_test_reader = paddle.batch(ts_reader, batch_size=128)
    97.  
    98. # feeder
    99. feeder = fluid.DataFeeder(place=place, feed_list=[words, label])
    100.  
    101. # 开始训练
    102. for epoch in range(80): # 外层循环控制训练轮次
    103.     for batch_id, data in enumerate(batch_train_reader()): # 内层循环控制批次
    104.         train_cost, train_acc = exe.run(fluid.default_main_program(),#program
    105.                                         feed=feeder.feed(data),#喂入的参数
    106.                                         fetch_list=[avg_cost, accuracy])#返回值
    107.         if batch_id % 100 == 0:
    108.             print("epoch:%d, batch:%d, cost:%f, acc:%f" %
    109.                   (epoch, batch_id, train_cost[0], train_acc[0]))
    110.  
    111.     # 每轮训练结束后进行模型评估
    112.     test_costs_list = [] # 存放测试集损失值
    113.     test_accs_list = [] # 存放测试集准确率
    114.  
    115.     for batch_id, data in enumerate(batch_test_reader()):
    116.         test_cost, test_acc = exe.run(fluid.default_main_program(), 
    117.                                       feed=feeder.feed(data),
    118.                                       fetch_list=[avg_cost, accuracy])
    119.         test_costs_list.append(test_cost[0])
    120.         test_accs_list.append(test_acc[0])
    121.     #　计算所有批次损失值/准确率均值
    122.     avg_test_cost = sum(test_costs_list) / len(test_costs_list)
    123.     avg_test_acc = sum(test_accs_list) / len(test_accs_list)
    124.     print("epoch:%d, test_cost:%f, test_acc:%f" %
    125.           (epoch, avg_test_cost, avg_test_acc))
    126.  
    127. # 训练结束，保存模型
    128. model_save_dir = "model/"
    129. if not os.path.exists(model_save_dir):
    130.     os.makedirs(model_save_dir)
    131. fluid.io.save_inference_model(model_save_dir, # 保存路径
    132.                               feeded_var_names=[words.name],# 使用时传入参数名称
    133.                               target_vars=[model],#预测结果
    134.                               executor=exe)#执行器
    135. print("模型保存成功.")

    输出

    1. W0301 17:44:27.392561   134 gpu\_resources.cc:61\] Please NOTE: device: 0, GPU Compute Capability: 7.0, Driver API Version: 12.0, Runtime API Version: 11.2
    2. W0301 17:44:27.396858   134 gpu\_resources.cc:91\] device: 0, cuDNN Version: 8.2.
    3.  
    4. epoch:0, batch:0, cost:2.311182, acc:0.062500
    5. epoch:0, batch:100, cost:2.063974, acc:0.398438
    6. epoch:0, batch:200, cost:1.564050, acc:0.531250
    7. epoch:0, batch:300, cost:1.113311, acc:0.726562
    8. epoch:0, test\_cost:0.984438, test\_acc:0.699619
    9. epoch:1, batch:0, cost:0.904973, acc:0.703125
    10. epoch:1, batch:100, cost:1.056342, acc:0.648438
    11. epoch:1, batch:200, cost:0.757222, acc:0.773438
    12. epoch:1, batch:300, cost:0.768483, acc:0.757812
    13. epoch:1, test\_cost:0.743878, test\_acc:0.760737
    14. epoch:2, batch:0, cost:0.799179, acc:0.773438
    15. epoch:2, batch:100, cost:0.684252, acc:0.796875
    16. epoch:2, batch:200, cost:0.700367, acc:0.773438
    17. epoch:2, batch:300, cost:0.658518, acc:0.820312
    18. epoch:2, test\_cost:0.662426, test\_acc:0.787224
    19. epoch:3, batch:0, cost:0.732371, acc:0.757812
    20. epoch:3, batch:100, cost:0.621147, acc:0.804688
    21. epoch:3, batch:200, cost:0.592039, acc:0.820312
    22. epoch:3, batch:300, cost:0.566666, acc:0.835938
    23. epoch:3, test\_cost:0.613746, test\_acc:0.799462
    24. epoch:4, batch:0, cost:0.477513, acc:0.835938
    25. epoch:4, batch:100, cost:0.510523, acc:0.820312
    26. epoch:4, batch:200, cost:0.635614, acc:0.765625
    27. epoch:4, batch:300, cost:0.487310, acc:0.820312
    28. epoch:4, test\_cost:0.579772, test\_acc:0.810488
    29. epoch:5, batch:0, cost:0.564993, acc:0.796875
    30. epoch:5, batch:100, cost:0.406841, acc:0.882812
    31. epoch:5, batch:200, cost:0.444392, acc:0.867188
    32. epoch:5, batch:300, cost:0.729928, acc:0.765625
    33. epoch:5, test\_cost:0.549828, test\_acc:0.820210
    34. epoch:6, batch:0, cost:0.676264, acc:0.757812
    35. epoch:6, batch:100, cost:0.596377, acc:0.859375
    36. epoch:6, batch:200, cost:0.532443, acc:0.773438
    37. epoch:6, batch:300, cost:0.534213, acc:0.820312
    38. epoch:6, test\_cost:0.529354, test\_acc:0.824554
    39. epoch:7, batch:0, cost:0.426611, acc:0.867188
    40. epoch:7, batch:100, cost:0.474698, acc:0.851562
    41. epoch:7, batch:200, cost:0.529180, acc:0.851562
    42. epoch:7, batch:300, cost:0.512545, acc:0.820312
    43. epoch:7, test\_cost:0.510936, test\_acc:0.830113
    44. epoch:8, batch:0, cost:0.559594, acc:0.859375
    45. epoch:8, batch:100, cost:0.670438, acc:0.812500
    46. epoch:8, batch:200, cost:0.519166, acc:0.796875
    47. epoch:8, batch:300, cost:0.487624, acc:0.835938
    48. epoch:8, test\_cost:0.495487, test\_acc:0.831584
    49. epoch:9, batch:0, cost:0.523822, acc:0.875000
    50. epoch:9, batch:100, cost:0.604066, acc:0.812500
    51. epoch:9, batch:200, cost:0.420712, acc:0.835938
    52. epoch:9, batch:300, cost:0.490420, acc:0.867188
    53. epoch:9, test\_cost:0.480619, test\_acc:0.838099
    54. epoch:10, batch:0, cost:0.337900, acc:0.875000
    55. epoch:10, batch:100, cost:0.513283, acc:0.820312
    56. epoch:10, batch:200, cost:0.471449, acc:0.835938
    57. epoch:10, batch:300, cost:0.387642, acc:0.859375
    58. epoch:10, test\_cost:0.470769, test\_acc:0.842698
    59. epoch:11, batch:0, cost:0.448336, acc:0.835938
    60. epoch:11, batch:100, cost:0.440521, acc:0.843750
    61. epoch:11, batch:200, cost:0.436573, acc:0.875000
    62. epoch:11, batch:300, cost:0.694043, acc:0.781250
    63. epoch:11, test\_cost:0.460047, test\_acc:0.846606
    64. epoch:12, batch:0, cost:0.481746, acc:0.882812
    65. epoch:12, batch:100, cost:0.552047, acc:0.804688
    66. epoch:12, batch:200, cost:0.386960, acc:0.882812
    67. epoch:12, batch:300, cost:0.341771, acc:0.882812
    68. epoch:12, test\_cost:0.450299, test\_acc:0.850769
    69. epoch:13, batch:0, cost:0.547187, acc:0.859375
    70. epoch:13, batch:100, cost:0.356574, acc:0.875000
    71. epoch:13, batch:200, cost:0.379301, acc:0.867188
    72. epoch:13, batch:300, cost:0.374310, acc:0.898438
    73. epoch:13, test\_cost:0.440953, test\_acc:0.856413
    74. epoch:14, batch:0, cost:0.689324, acc:0.843750
    75. epoch:14, batch:100, cost:0.493557, acc:0.851562
    76. epoch:14, batch:200, cost:0.332372, acc:0.898438
    77. epoch:14, batch:300, cost:0.462879, acc:0.867188
    78. epoch:14, test\_cost:0.434527, test\_acc:0.858323
    79. epoch:15, batch:0, cost:0.539430, acc:0.859375
    80. epoch:15, batch:100, cost:0.610829, acc:0.828125
    81. epoch:15, batch:200, cost:0.567205, acc:0.773438
    82. epoch:15, batch:300, cost:0.448473, acc:0.851562
    83. epoch:15, test\_cost:0.429657, test\_acc:0.858929
    84. epoch:16, batch:0, cost:0.275449, acc:0.914062
    85. epoch:16, batch:100, cost:0.322028, acc:0.890625
    86. epoch:16, batch:200, cost:0.433238, acc:0.843750
    87. epoch:16, batch:300, cost:0.435924, acc:0.875000
    88. epoch:16, test\_cost:0.420777, test\_acc:0.864314
    89. epoch:17, batch:0, cost:0.463425, acc:0.859375
    90. epoch:17, batch:100, cost:0.498210, acc:0.890625
    91. epoch:17, batch:200, cost:0.347210, acc:0.898438
    92. epoch:17, batch:300, cost:0.375353, acc:0.867188
    93. epoch:17, test\_cost:0.414920, test\_acc:0.866483
    94. epoch:18, batch:0, cost:0.371144, acc:0.906250
    95. epoch:18, batch:100, cost:0.511294, acc:0.828125
    96. epoch:18, batch:200, cost:0.431728, acc:0.828125
    97. epoch:18, batch:300, cost:0.505222, acc:0.843750
    98. epoch:18, test\_cost:0.412018, test\_acc:0.866136
    99. epoch:19, batch:0, cost:0.417319, acc:0.859375
    100. epoch:19, batch:100, cost:0.405875, acc:0.867188
    101. epoch:19, batch:200, cost:0.466319, acc:0.843750
    102. epoch:19, batch:300, cost:0.524598, acc:0.820312
    103. epoch:19, test\_cost:0.408254, test\_acc:0.870387
    104. epoch:20, batch:0, cost:0.278774, acc:0.921875
    105. epoch:20, batch:100, cost:0.375402, acc:0.875000
    106. epoch:20, batch:200, cost:0.512493, acc:0.851562
    107. epoch:20, batch:300, cost:0.352869, acc:0.867188
    108. epoch:20, test\_cost:0.402862, test\_acc:0.870646
    109. epoch:21, batch:0, cost:0.328388, acc:0.890625
    110. epoch:21, batch:100, cost:0.474930, acc:0.843750
    111. epoch:21, batch:200, cost:0.279459, acc:0.898438
    112. epoch:21, batch:300, cost:0.480916, acc:0.843750
    113. epoch:21, test\_cost:0.398193, test\_acc:0.870476
    114. epoch:22, batch:0, cost:0.360476, acc:0.914062
    115. epoch:22, batch:100, cost:0.399123, acc:0.867188
    116. epoch:22, batch:200, cost:0.330940, acc:0.898438
    117. epoch:22, batch:300, cost:0.449070, acc:0.851562
    118. epoch:22, test\_cost:0.396272, test\_acc:0.872644
    119. epoch:23, batch:0, cost:0.311765, acc:0.882812
    120. epoch:23, batch:100, cost:0.430598, acc:0.859375
    121. epoch:23, batch:200, cost:0.371466, acc:0.867188
    122. epoch:23, batch:300, cost:0.497460, acc:0.859375
    123. epoch:23, test\_cost:0.391935, test\_acc:0.874990
    124. epoch:24, batch:0, cost:0.278461, acc:0.921875
    125. epoch:24, batch:100, cost:0.384332, acc:0.867188
    126. epoch:24, batch:200, cost:0.687089, acc:0.804688
    127. epoch:24, batch:300, cost:0.465835, acc:0.835938
    128. epoch:24, test\_cost:0.386384, test\_acc:0.874905
    129. epoch:25, batch:0, cost:0.359800, acc:0.914062
    130. epoch:25, batch:100, cost:0.370942, acc:0.906250
    131. epoch:25, batch:200, cost:0.343612, acc:0.906250
    132. epoch:25, batch:300, cost:0.373149, acc:0.859375
    133. epoch:25, test\_cost:0.385754, test\_acc:0.875249
    134. epoch:26, batch:0, cost:0.359912, acc:0.859375
    135. epoch:26, batch:100, cost:0.299233, acc:0.906250
    136. epoch:26, batch:200, cost:0.321898, acc:0.882812
    137. epoch:26, batch:300, cost:0.506139, acc:0.820312
    138. epoch:26, test\_cost:0.382092, test\_acc:0.877597
    139. epoch:27, batch:0, cost:0.438806, acc:0.882812
    140. epoch:27, batch:100, cost:0.351698, acc:0.867188
    141. epoch:27, batch:200, cost:0.413263, acc:0.875000
    142. epoch:27, batch:300, cost:0.327677, acc:0.875000
    143. epoch:27, test\_cost:0.379122, test\_acc:0.880460
    144. epoch:28, batch:0, cost:0.329184, acc:0.921875
    145. epoch:28, batch:100, cost:0.489258, acc:0.882812
    146. epoch:28, batch:200, cost:0.375317, acc:0.890625
    147. epoch:28, batch:300, cost:0.355702, acc:0.859375
    148. epoch:28, test\_cost:0.377964, test\_acc:0.881066
    149. epoch:29, batch:0, cost:0.360147, acc:0.882812
    150. epoch:29, batch:100, cost:0.361545, acc:0.906250
    151. epoch:29, batch:200, cost:0.535644, acc:0.812500
    152. epoch:29, batch:300, cost:0.463827, acc:0.789062
    153. epoch:29, test\_cost:0.374992, test\_acc:0.879156
    154. epoch:30, batch:0, cost:0.386321, acc:0.843750
    155. epoch:30, batch:100, cost:0.450116, acc:0.851562
    156. epoch:30, batch:200, cost:0.380319, acc:0.867188
    157. epoch:30, batch:300, cost:0.357393, acc:0.914062
    158. epoch:30, test\_cost:0.372232, test\_acc:0.880198
    159. epoch:31, batch:0, cost:0.338851, acc:0.882812
    160. epoch:31, batch:100, cost:0.418707, acc:0.890625
    161. epoch:31, batch:200, cost:0.349568, acc:0.875000
    162. epoch:31, batch:300, cost:0.414638, acc:0.882812
    163. epoch:31, test\_cost:0.373127, test\_acc:0.879245
    164. epoch:32, batch:0, cost:0.278832, acc:0.906250
    165. epoch:32, batch:100, cost:0.538143, acc:0.851562
    166. epoch:32, batch:200, cost:0.418359, acc:0.890625
    167. epoch:32, batch:300, cost:0.510367, acc:0.875000
    168. epoch:32, test\_cost:0.370239, test\_acc:0.880896
    169. epoch:33, batch:0, cost:0.410598, acc:0.835938
    170. epoch:33, batch:100, cost:0.295002, acc:0.906250
    171. epoch:33, batch:200, cost:0.430560, acc:0.828125
    172. epoch:33, batch:300, cost:0.417476, acc:0.859375
    173. epoch:33, test\_cost:0.367410, test\_acc:0.881155
    174. epoch:34, batch:0, cost:0.337740, acc:0.937500
    175. epoch:34, batch:100, cost:0.304080, acc:0.906250
    176. epoch:34, batch:200, cost:0.359049, acc:0.890625
    177. epoch:34, batch:300, cost:0.373999, acc:0.890625
    178. epoch:34, test\_cost:0.367002, test\_acc:0.880113
    179. epoch:35, batch:0, cost:0.411581, acc:0.898438
    180. epoch:35, batch:100, cost:0.400797, acc:0.851562
    181. epoch:35, batch:200, cost:0.482271, acc:0.828125
    182. epoch:35, batch:300, cost:0.340450, acc:0.890625
    183. epoch:35, test\_cost:0.363663, test\_acc:0.883068
    184. epoch:36, batch:0, cost:0.338912, acc:0.875000
    185. epoch:36, batch:100, cost:0.416916, acc:0.867188
    186. epoch:36, batch:200, cost:0.313621, acc:0.882812
    187. epoch:36, batch:300, cost:0.677497, acc:0.796875
    188. epoch:36, test\_cost:0.361819, test\_acc:0.882983
    189. epoch:37, batch:0, cost:0.329249, acc:0.867188
    190. epoch:37, batch:100, cost:0.375915, acc:0.890625
    191. epoch:37, batch:200, cost:0.290267, acc:0.906250
    192. epoch:37, batch:300, cost:0.388264, acc:0.859375
    193. epoch:37, test\_cost:0.363713, test\_acc:0.880025
    194. epoch:38, batch:0, cost:0.452093, acc:0.875000
    195. epoch:38, batch:100, cost:0.237014, acc:0.898438
    196. epoch:38, batch:200, cost:0.334976, acc:0.898438
    197. epoch:38, batch:300, cost:0.386618, acc:0.875000
    198. epoch:38, test\_cost:0.357681, test\_acc:0.884889
    199. epoch:39, batch:0, cost:0.397014, acc:0.867188
    200. epoch:39, batch:100, cost:0.387132, acc:0.882812
    201. epoch:39, batch:200, cost:0.262646, acc:0.921875
    202. epoch:39, batch:300, cost:0.295718, acc:0.906250
    203. epoch:39, test\_cost:0.358814, test\_acc:0.884542
    204. epoch:40, batch:0, cost:0.336061, acc:0.875000
    205. epoch:40, batch:100, cost:0.393282, acc:0.867188
    206. epoch:40, batch:200, cost:0.453071, acc:0.867188
    207. epoch:40, batch:300, cost:0.276213, acc:0.921875
    208. epoch:40, test\_cost:0.355846, test\_acc:0.886278
    209. epoch:41, batch:0, cost:0.362588, acc:0.867188
    210. epoch:41, batch:100, cost:0.293396, acc:0.914062
    211. epoch:41, batch:200, cost:0.351766, acc:0.890625
    212. epoch:41, batch:300, cost:0.437711, acc:0.820312
    213. epoch:41, test\_cost:0.356017, test\_acc:0.886799
    214. epoch:42, batch:0, cost:0.431722, acc:0.843750
    215. epoch:42, batch:100, cost:0.296809, acc:0.914062
    216. epoch:42, batch:200, cost:0.300333, acc:0.898438
    217. epoch:42, batch:300, cost:0.392034, acc:0.859375
    218. epoch:42, test\_cost:0.354504, test\_acc:0.885580
    219. epoch:43, batch:0, cost:0.237395, acc:0.945312
    220. epoch:43, batch:100, cost:0.274653, acc:0.914062
    221. epoch:43, batch:200, cost:0.320165, acc:0.898438
    222. epoch:43, batch:300, cost:0.233366, acc:0.937500
    223. epoch:43, test\_cost:0.352862, test\_acc:0.885410
    224. epoch:44, batch:0, cost:0.309431, acc:0.953125
    225. epoch:44, batch:100, cost:0.371803, acc:0.843750
    226. epoch:44, batch:200, cost:0.309721, acc:0.898438
    227. epoch:44, batch:300, cost:0.330030, acc:0.898438
    228. epoch:44, test\_cost:0.348967, test\_acc:0.888017
    229. epoch:45, batch:0, cost:0.382172, acc:0.890625
    230. epoch:45, batch:100, cost:0.292855, acc:0.929688
    231. epoch:45, batch:200, cost:0.445127, acc:0.898438
    232. epoch:45, batch:300, cost:0.365554, acc:0.890625
    233. epoch:45, test\_cost:0.352218, test\_acc:0.883932
    234. epoch:46, batch:0, cost:0.424743, acc:0.898438
    235. epoch:46, batch:100, cost:0.382699, acc:0.859375
    236. epoch:46, batch:200, cost:0.319472, acc:0.914062
    237. epoch:46, batch:300, cost:0.414162, acc:0.859375
    238. epoch:46, test\_cost:0.349987, test\_acc:0.885498
    239. epoch:47, batch:0, cost:0.304131, acc:0.890625
    240. epoch:47, batch:100, cost:0.386861, acc:0.890625
    241. epoch:47, batch:200, cost:0.608894, acc:0.820312
    242. epoch:47, batch:300, cost:0.281832, acc:0.898438
    243. epoch:47, test\_cost:0.349286, test\_acc:0.888276
    244. epoch:48, batch:0, cost:0.406423, acc:0.882812
    245. epoch:48, batch:100, cost:0.398680, acc:0.898438
    246. epoch:48, batch:200, cost:0.291706, acc:0.914062
    247. epoch:48, batch:300, cost:0.358105, acc:0.875000
    248. epoch:48, test\_cost:0.348130, test\_acc:0.888361
    249. epoch:49, batch:0, cost:0.284720, acc:0.914062
    250. epoch:49, batch:100, cost:0.341173, acc:0.898438
    251. epoch:49, batch:200, cost:0.341595, acc:0.859375
    252. epoch:49, batch:300, cost:0.442754, acc:0.820312
    253. epoch:49, test\_cost:0.347218, test\_acc:0.886012
    254. epoch:50, batch:0, cost:0.311721, acc:0.906250
    255. epoch:50, batch:100, cost:0.326822, acc:0.875000
    256. epoch:50, batch:200, cost:0.331799, acc:0.898438
    257. epoch:50, batch:300, cost:0.426647, acc:0.851562
    258. epoch:50, test\_cost:0.347288, test\_acc:0.888535
    259. epoch:51, batch:0, cost:0.389481, acc:0.867188
    260. epoch:51, batch:100, cost:0.289127, acc:0.906250
    261. epoch:51, batch:200, cost:0.328051, acc:0.929688
    262. epoch:51, batch:300, cost:0.426396, acc:0.890625
    263. epoch:51, test\_cost:0.344246, test\_acc:0.889839
    264. epoch:52, batch:0, cost:0.288156, acc:0.906250
    265. epoch:52, batch:100, cost:0.298805, acc:0.906250
    266. epoch:52, batch:200, cost:0.371176, acc:0.921875
    267. epoch:52, batch:300, cost:0.389306, acc:0.875000
    268. epoch:52, test\_cost:0.345692, test\_acc:0.891224
    269. epoch:53, batch:0, cost:0.425932, acc:0.890625
    270. epoch:53, batch:100, cost:0.415528, acc:0.882812
    271. epoch:53, batch:200, cost:0.434767, acc:0.867188
    272. epoch:53, batch:300, cost:0.331441, acc:0.914062
    273. epoch:53, test\_cost:0.340924, test\_acc:0.890101
    274. epoch:54, batch:0, cost:0.260270, acc:0.906250
    275. epoch:54, batch:100, cost:0.305412, acc:0.898438
    276. epoch:54, batch:200, cost:0.330370, acc:0.906250
    277. epoch:54, batch:300, cost:0.334084, acc:0.898438
    278. epoch:54, test\_cost:0.341799, test\_acc:0.892010
    279. epoch:55, batch:0, cost:0.239946, acc:0.937500
    280. epoch:55, batch:100, cost:0.510334, acc:0.898438
    281. epoch:55, batch:200, cost:0.331789, acc:0.898438
    282. epoch:55, batch:300, cost:0.273344, acc:0.898438
    283. epoch:55, test\_cost:0.341348, test\_acc:0.889403
    284. epoch:56, batch:0, cost:0.288282, acc:0.914062
    285. epoch:56, batch:100, cost:0.384843, acc:0.898438
    286. epoch:56, batch:200, cost:0.391903, acc:0.867188
    287. epoch:56, batch:300, cost:0.352458, acc:0.882812
    288. epoch:56, test\_cost:0.338860, test\_acc:0.891054
    289. epoch:57, batch:0, cost:0.434810, acc:0.828125
    290. epoch:57, batch:100, cost:0.257800, acc:0.953125
    291. epoch:57, batch:200, cost:0.283473, acc:0.921875
    292. epoch:57, batch:300, cost:0.337173, acc:0.867188
    293. epoch:57, test\_cost:0.339060, test\_acc:0.891575
    294. epoch:58, batch:0, cost:0.240891, acc:0.898438
    295. epoch:58, batch:100, cost:0.390225, acc:0.875000
    296. epoch:58, batch:200, cost:0.393483, acc:0.843750
    297. epoch:58, batch:300, cost:0.289487, acc:0.890625
    298. epoch:58, test\_cost:0.337302, test\_acc:0.892269
    299. epoch:59, batch:0, cost:0.210337, acc:0.960938
    300. epoch:59, batch:100, cost:0.423231, acc:0.867188
    301. epoch:59, batch:200, cost:0.319490, acc:0.921875
    302. epoch:59, batch:300, cost:0.451494, acc:0.859375
    303. epoch:59, test\_cost:0.336483, test\_acc:0.893137
    304. epoch:60, batch:0, cost:0.231775, acc:0.937500
    305. epoch:60, batch:100, cost:0.295306, acc:0.906250
    306. epoch:60, batch:200, cost:0.378960, acc:0.859375
    307. epoch:60, batch:300, cost:0.350808, acc:0.843750
    308. epoch:60, test\_cost:0.335058, test\_acc:0.894267
    309. epoch:61, batch:0, cost:0.440865, acc:0.867188
    310. epoch:61, batch:100, cost:0.270725, acc:0.882812
    311. epoch:61, batch:200, cost:0.398181, acc:0.851562
    312. epoch:61, batch:300, cost:0.363882, acc:0.921875
    313. epoch:61, test\_cost:0.336761, test\_acc:0.892875
    314. epoch:62, batch:0, cost:0.321757, acc:0.898438
    315. epoch:62, batch:100, cost:0.330311, acc:0.890625
    316. epoch:62, batch:200, cost:0.406124, acc:0.851562
    317. epoch:62, batch:300, cost:0.275819, acc:0.898438
    318. epoch:62, test\_cost:0.342463, test\_acc:0.891565
    319. epoch:63, batch:0, cost:0.321822, acc:0.898438
    320. epoch:63, batch:100, cost:0.322195, acc:0.882812
    321. epoch:63, batch:200, cost:0.432605, acc:0.882812
    322. epoch:63, batch:300, cost:0.377368, acc:0.898438
    323. epoch:63, test\_cost:0.333785, test\_acc:0.895221
    324. epoch:64, batch:0, cost:0.247617, acc:0.882812
    325. epoch:64, batch:100, cost:0.231372, acc:0.921875
    326. epoch:64, batch:200, cost:0.336805, acc:0.867188
    327. epoch:64, batch:300, cost:0.274635, acc:0.898438
    328. epoch:64, test\_cost:0.332033, test\_acc:0.894179
    329. epoch:65, batch:0, cost:0.241076, acc:0.906250
    330. epoch:65, batch:100, cost:0.377462, acc:0.906250
    331. epoch:65, batch:200, cost:0.297226, acc:0.882812
    332. epoch:65, batch:300, cost:0.440397, acc:0.867188
    333. epoch:65, test\_cost:0.330794, test\_acc:0.897045
    334. epoch:66, batch:0, cost:0.266126, acc:0.898438
    335. epoch:66, batch:100, cost:0.390715, acc:0.859375
    336. epoch:66, batch:200, cost:0.292437, acc:0.914062
    337. epoch:66, batch:300, cost:0.395078, acc:0.867188
    338. epoch:66, test\_cost:0.330902, test\_acc:0.895221
    339. epoch:67, batch:0, cost:0.301438, acc:0.929688
    340. epoch:67, batch:100, cost:0.388324, acc:0.898438
    341. epoch:67, batch:200, cost:0.439915, acc:0.890625
    342. epoch:67, batch:300, cost:0.310547, acc:0.867188
    343. epoch:67, test\_cost:0.330386, test\_acc:0.896521
    344. epoch:68, batch:0, cost:0.243119, acc:0.929688
    345. epoch:68, batch:100, cost:0.447522, acc:0.875000
    346. epoch:68, batch:200, cost:0.470691, acc:0.882812
    347. epoch:68, batch:300, cost:0.296465, acc:0.882812
    348. epoch:68, test\_cost:0.326098, test\_acc:0.896266
    349. epoch:69, batch:0, cost:0.260604, acc:0.898438
    350. epoch:69, batch:100, cost:0.417193, acc:0.882812
    351. epoch:69, batch:200, cost:0.483119, acc:0.835938
    352. epoch:69, batch:300, cost:0.405713, acc:0.875000
    353. epoch:69, test\_cost:0.328661, test\_acc:0.896957
    354. epoch:70, batch:0, cost:0.300975, acc:0.882812
    355. epoch:70, batch:100, cost:0.199427, acc:0.945312
    356. epoch:70, batch:200, cost:0.207260, acc:0.937500
    357. epoch:70, batch:300, cost:0.199148, acc:0.914062
    358. epoch:70, test\_cost:0.327545, test\_acc:0.894958
    359. epoch:71, batch:0, cost:0.281955, acc:0.914062
    360. epoch:71, batch:100, cost:0.267508, acc:0.914062
    361. epoch:71, batch:200, cost:0.561389, acc:0.828125
    362. epoch:71, batch:300, cost:0.377676, acc:0.867188
    363. epoch:71, test\_cost:0.325637, test\_acc:0.897740
    364. epoch:72, batch:0, cost:0.348661, acc:0.890625
    365. epoch:72, batch:100, cost:0.346154, acc:0.898438
    366. epoch:72, batch:200, cost:0.447819, acc:0.867188
    367. epoch:72, batch:300, cost:0.342514, acc:0.929688
    368. epoch:72, test\_cost:0.325294, test\_acc:0.897304
    369. epoch:73, batch:0, cost:0.223638, acc:0.929688
    370. epoch:73, batch:100, cost:0.394560, acc:0.859375
    371. epoch:73, batch:200, cost:0.341260, acc:0.890625
    372. epoch:73, batch:300, cost:0.283185, acc:0.898438
    373. epoch:73, test\_cost:0.326340, test\_acc:0.895394
    374. epoch:74, batch:0, cost:0.371942, acc:0.921875
    375. epoch:74, batch:100, cost:0.333636, acc:0.882812
    376. epoch:74, batch:200, cost:0.397030, acc:0.875000
    377. epoch:74, batch:300, cost:0.392802, acc:0.875000
    378. epoch:74, test\_cost:0.322571, test\_acc:0.896089
    379. epoch:75, batch:0, cost:0.275930, acc:0.921875
    380. epoch:75, batch:100, cost:0.263152, acc:0.914062
    381. epoch:75, batch:200, cost:0.296550, acc:0.898438
    382. epoch:75, batch:300, cost:0.402121, acc:0.898438
    383. epoch:75, test\_cost:0.320611, test\_acc:0.897134
    384. epoch:76, batch:0, cost:0.279775, acc:0.921875
    385. epoch:76, batch:100, cost:0.439274, acc:0.843750
    386. epoch:76, batch:200, cost:0.330266, acc:0.898438
    387. epoch:76, batch:300, cost:0.418308, acc:0.851562
    388. epoch:76, test\_cost:0.320242, test\_acc:0.900429
    389. epoch:77, batch:0, cost:0.320668, acc:0.890625
    390. epoch:77, batch:100, cost:0.168939, acc:0.960938
    391. epoch:77, batch:200, cost:0.244379, acc:0.953125
    392. epoch:77, batch:300, cost:0.621534, acc:0.875000
    393. epoch:77, test\_cost:0.319756, test\_acc:0.900865
    394. epoch:78, batch:0, cost:0.284392, acc:0.914062
    395. epoch:78, batch:100, cost:0.309243, acc:0.890625
    396. epoch:78, batch:200, cost:0.273962, acc:0.945312
    397. epoch:78, batch:300, cost:0.311928, acc:0.906250
    398. epoch:78, test\_cost:0.318491, test\_acc:0.901818
    399. epoch:79, batch:0, cost:0.242170, acc:0.898438
    400. epoch:79, batch:100, cost:0.315753, acc:0.875000
    401. epoch:79, batch:200, cost:0.252874, acc:0.937500
    402. epoch:79, batch:300, cost:0.447730, acc:0.812500
    403. epoch:79, test\_cost:0.318828, test\_acc:0.900603
    404. 模型保存成功.

    【推理预测】

    1. model_save_dir = "model/"
    2.  
    3. def get_data(sentence): # 将传入的句子根据字典中的值进行编码
    4.     with open(dict_file_path, "r", encoding="utf-8") as f:
    5.         dict_txt = eval(f.readlines()[0])
    6.  
    7.     ret = [] # 编码结果
    8.     keys = dict_txt.keys()
    9.     for w in sentence: # 取出每个字
    10.         if not w in keys: # 字不在字典中
    11.             w = "<unk>"
    12.         ret.append(int(dict_txt[w]))
    13.     return ret
    14.  
    15. # 执行器
    16. place = fluid.CPUPlace()
    17. exe = fluid.Executor(place)
    18. exe.run(fluid.default_startup_program())
    19.  
    20. infer_program, feed_names, target_var = \
    21. fluid.io.load_inference_model(model_save_dir, exe)
    22.  
    23. texts = [] # 存放待预测句子
    24.  
    25. data1 = get_data("在获得诺贝尔文学奖7年之后，莫言15日晚间在山西汾阳贾家庄如是说")
    26. data2 = get_data("综合'今日美国'、《世界日报》等当地媒体报道，芝加哥河滨警察局表示")
    27. data3 = get_data("中国队2022年冬奥会表现优秀")
    28. data4 = get_data("中国人民银行今日发布通知，降低准备金率，预计释放4000亿流动性")
    29. data5 = get_data("10月20日,第六届世界互联网大会正式开幕")
    30. data6 = get_data("同一户型，为什么高层比低层要贵那么多？")
    31. data7 = get_data("揭秘A股周涨5%资金动向：追捧2类股，抛售600亿香饽饽")
    32. data8 = get_data("宋慧乔陷入感染危机，前夫宋仲基不戴口罩露面，身处国外神态轻松")
    33. data9 = get_data("此盆栽花很好养，花美似牡丹，三季开花，南北都能养，很值得栽培")  # 不属于任何一个类别
    34.  
    35. texts.append(data1)
    36. texts.append(data2)
    37. texts.append(data3)
    38. texts.append(data4)
    39. texts.append(data5)
    40. texts.append(data6)
    41. texts.append(data7)
    42. texts.append(data8)
    43. texts.append(data9)
    44.  
    45. base_shape = [[len(c) for c in texts]] # 计算每个句子长度
    46. tensor_words = fluid.create_lod_tensor(texts, base_shape, place)
    47. result = exe.run(infer_program,
    48.                  feed={feed_names[0]: tensor_words},
    49.                  fetch_list=target_var)
    50. names = ["文化", "娱乐", "体育", "财经", "房产","汽车", "教育", "科技", "国际", "证券"]
    51. for r in result[0]:
    52.     idx = np.argmax(r) # 取出最大值的索引
    53.     print("预测结果:", names[idx], " 概率:", r[idx])

    输出

    1. 预测结果: 财经  概率: 0.81440145
    2. 预测结果: 娱乐  概率: 1.0
    3. 预测结果: 财经  概率: 1.0
    4. 预测结果: 汽车  概率: 0.9996093
    5. 预测结果: 文化  概率: 0.9404757
    6. 预测结果: 娱乐  概率: 0.8715788
    7. 预测结果: 房产  概率: 0.9625704
    8. 预测结果: 科技  概率: 0.985617
    9. 预测结果: 房产  概率: 1.0

    文章涉及到的数据资源链接如下：news_classify_data.zip - 蓝奏云文件大小：2.7 M|icon-default.png?t=N7T8https://wwt.lanzoum.com/iTF6N1q1rnti

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/菜鸟追梦旅行/article/detail/362150
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号