ERNIE3.0实践手记_ernie3.0代码复现

import random
 
# 读取自定义.txt文件中的内容
with open('weibo_senti_100k.txt', 'r',encoding='utf-8') as file:
    lines = file.readlines()
 
# 随机打乱数据
random.shuffle(lines)
 
# 计算切分的索引
total_lines = len(lines)
train_end = int(total_lines * 0.7)
dev_end = int(total_lines * 0.9)
 
# 切分数据
train_data = lines[:train_end]
dev_data = lines[train_end:dev_end]
test_data = lines[validation_end:]
 
# 将数据写入train.txt
with open('train.txt', 'w' ,encoding='utf-8') as file:
    file.writelines(train_data)
 
# 将数据写入validation.txt
with open('dev.txt', 'w' ,encoding='utf-8') as file:
    file.writelines(dev_data)
 
# 将数据写入test.txt
with open('test.txt', 'w' ,encoding='utf-8') as file:
    file.writelines(test_data)

# 导入DatasetBuilder
from paddlenlp.datasets import DatasetBuilder
 
 
class NewsData(DatasetBuilder):
    SPLITS = {
        'train': r'train.txt',  # 训练集
        'dev': r'dev.txt', # 验证集
        'test': r'test.txt' #测试集
    }
 
    def _get_data(self, mode, **kwargs):
        filename = self.SPLITS[mode]
        return filename
 
    def _read(self, filename):
        """读取数据"""
        with open(filename, 'r', encoding='utf-8',errors='ignore') as f:
            for line in f:
                if line == '\n':
                    continue
                data = line.strip().split("\t")    # 以'\t'分隔各列
                label, text_a = data
                text_a = text_a.replace(" ", "")
                if label in ['0', '1']:
                    yield {"text_a": text_a, "label": label}  # 此次设置数据的格式为：text_a,label，可以根据具体情况进行修改
 
    def get_labels(self):
        return label_list   # 类别标签
    
    from paddlenlp.datasets import load_dataset
 
-------------------------------------------
D:******lib\site-packages\_distutils_hack\__init__.py:33: UserWarning: Setuptools is replacing distutils.
  warnings.warn("Setuptools is replacing distutils.")

# 定义数据集加载函数
def load_dataset(name=None,
                 data_files=None,
                 splits=None,
                 lazy=None,
                 **kwargs):
   
    reader_cls = NewsData  # 加载定义的数据集格式
    print(reader_cls)
    if not name:
        reader_instance = reader_cls(lazy=lazy, **kwargs)
    else:
        reader_instance = reader_cls(lazy=lazy, name=name, **kwargs)
    datasets = reader_instance.read_datasets(data_files=data_files, splits=splits)
    return datasets

# 加载训练和验证集
label_list = ['0', '1']
train_ds, dev_ds, text_t = load_dataset(splits=['train', 'dev', 'test'])
 
 
-------------------------------------
<class '__main__.NewsData'>

#展示前五行数据
train_ds[:5]
 
--------------------------------------
[{'text_a': '[抓狂][抓狂][抓狂]起晚了[泪]', 'label': 0},
 {'text_a': '分享图片，不要啊~~~虽然我很喜欢周迅，可是八阿哥，你一定要等晴川啊~~[泪]', 'label': 0},
 {'text_a': '想shi的就上飞机吧???只见君去不见君还[泪]//@玉翠文章:不错很丰满，赛过杨贵妃，我喜欢，带着你超过盖茨4倍家财的嫁妆来吧[酷]请各路大仙作媒',
  'label': 0},
 {'text_a': '多谢支持！//@洪三水:[嘻嘻]画面不错，其他继续研究中。。//@曹欣Dyson:抢怪的那叫个多。-。-//@刘波BOB:钢铁侠，有没有？有没有？亮了！',
  'label': 1},
 {'text_a': '#周末节奏#美好的一天从早餐开始，黄金蛋炒饭，番茄牛尾汤[嘻嘻]', 'label': 1}]

import os
import paddle
import paddlenlp

from paddlenlp.transformers import AutoModelForSequenceClassification, AutoTokenizer
model_name = "ernie-3.0-medium-zh"
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_classes=2)
tokenizer = AutoTokenizer.from_pretrained(model_name)
 
---------------------------------------
[2024-01-26 13:15:04,015] [    INFO] - We are using <class 'paddlenlp.transformers.ernie.modeling.ErnieForSequenceClassification'> to load 'ernie-3.0-medium-zh'.
[2024-01-26 13:15:04,016] [    INFO] - Already cached *****\.paddlenlp\models\ernie-3.0-medium-zh\model_state.pdparams
[2024-01-26 13:15:04,017] [    INFO] - Loading weights file model_state.pdparams from cache at *****\.paddlenlp\models\ernie-3.0-medium-zh\model_state.pdparams
[2024-01-26 13:15:04,276] [    INFO] - Loaded weights file from disk, setting weights to model.
[2024-01-26 13:15:09,982] [ WARNING] - Some weights of the model checkpoint at ernie-3.0-medium-zh were not used when initializing ErnieForSequenceClassification: ['ernie.encoder.layers.6.self_attn.k_proj.weight', 'ernie.encoder.layers.6.self_attn.q_proj.bias', 'ernie.encoder.layers.6.linear1.weight', 'ernie.encoder.layers.6.norm2.bias', 'ernie.encoder.layers.6.self_attn.k_proj.bias', 'ernie.encoder.layers.6.self_attn.v_proj.bias', 'ernie.encoder.layers.6.self_attn.out_proj.weight', 'ernie.encoder.layers.6.self_attn.v_proj.weight', 'ernie.encoder.layers.6.norm1.bias', 'ernie.encoder.layers.6.norm1.weight', 'ernie.encoder.layers.6.linear2.bias', 'ernie.encoder.layers.6.linear1.bias', 'ernie.encoder.layers.6.self_attn.q_proj.weight', 'ernie.encoder.layers.6.linear2.weight', 'ernie.encoder.layers.6.self_attn.out_proj.bias', 'ernie.encoder.layers.6.norm2.weight']
- This IS expected if you are initializing ErnieForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing ErnieForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
[2024-01-26 13:15:09,982] [ WARNING] - Some weights of ErnieForSequenceClassification were not initialized from the model checkpoint at ernie-3.0-medium-zh and are newly initialized: ['classifier.bias', 'ernie.pooler.dense.bias', 'classifier.weight', 'ernie.pooler.dense.weight']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
[2024-01-26 13:15:10,009] [    INFO] - We are using (<class 'paddlenlp.transformers.ernie.tokenizer.ErnieTokenizer'>, False) to load 'ernie-3.0-medium-zh'.
[2024-01-26 13:15:10,010] [    INFO] - Already cached *****\.paddlenlp\models\ernie-3.0-medium-zh\ernie_3.0_medium_zh_vocab.txt
[2024-01-26 13:15:10,028] [    INFO] - tokenizer config file saved in C:\Users\徐金硕\.paddlenlp\models\ernie-3.0-medium-zh\tokenizer_config.json
[2024-01-26 13:15:10,030] [    INFO] - Special tokens file saved in *****\.paddlenlp\models\ernie-3.0-medium-zh\special_tokens_map.json

from functools import partial
from paddlenlp.data import Stack, Tuple, Pad
from utils import convert_example, create_dataloader
 
# 模型运行批处理大小
batch_size = 32
max_seq_length = 128
 
trans_func = partial(
    convert_example,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length)
batchify_fn = lambda samples, fn=Tuple(
    Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input
    Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # segment
    Stack(dtype="int64")  # label
): [data for data in fn(samples)]
train_data_loader = create_dataloader(
    train_ds,
    mode='train',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)
dev_data_loader = create_dataloader(
    dev_ds,
    mode='dev',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)