赞
踩
官方参考文档:https://huggingface.co/docs/transformers/training#additional-resources
文本分类实例解析:https://www.freesion.com/article/31511099215/
#transformer bert微调实例:以imdb数据集为基础(二分类),进行文本分类任务的微调 #进行下列步骤之前,要先安装好transformer和pytorch #导入数据,该数据集是一个具有三个键的字典:"train","test"和"unsupervised" 。我们使用"train"进行训练,使用 "test"进行验证。 from datasets import load_dataset raw_datasets = load_dataset("imdb") #分词器(用的bert基线模型) from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained("bert-base-cased") #文本截断:截断一个实例(非必须,默认最大长度为512) inputs = tokenizer(sentences, padding="max_length", truncation=True) #文本截断:批量处理 def tokenize_function(examples): return tokenizer(examples["text"], padding="max_length", truncation=True) tokenized_datasets = raw_datasets.map(tokenize_function, batched=True) #获取数据集中的一部分,进行训练(非必须,主要是快) small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000)) small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000)) full_train_dataset = tokenized_datasets["train"] full_eval_dataset = tokenized_datasets["test"] #进行微调共有三种方式,分别是用Pytorch API,Keras API 以及原生Pytorch '''Fine-tuning in PyTorch with the Trainer API''' #定义模型(会随机初始化) from transformers import AutoModelForSequenceClassification model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=2) #实例化一个 TrainingArguments。这个类包含我们可以为Trainer或标志调整的所有超参数 ,以激活它支持的不同训练选项。 from transformers import TrainingArguments training_args = TrainingArguments("test_trainer") #实例化一个Trainer from transformers import Trainer trainer = Trainer( model=model, args=training_args, train_dataset=small_train_dataset, eval_dataset=small_eval_dataset ) #微调(默认情况下,训练过程中没有评估) trainer.train() #计算过程中的指标 import numpy as np from datasets import load_metric metric = load_metric("accuracy") def compute_metrics(eval_pred): logits, labels = eval_pred predictions = np.argmax(logits, axis=-1) return metric.compute(predictions=predictions, references=labels) #验证 trainer = Trainer( model=model, args=training_args, train_dataset=small_train_dataset, eval_dataset=small_eval_dataset, compute_metrics=compute_metrics, ) trainer.evaluate() #定期输出评估指标(非必须) from transformers import TrainingArguments training_args = TrainingArguments("test_trainer", evaluation_strategy="epoch") '''Fine-tuning with Keras''' #使用 Keras API 在 TensorFlow 中进行本地训练。首先,定义模型 import tensorflow as tf from transformers import TFAutoModelForSequenceClassification model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=2) #数据集标准转换 tf_train_dataset = small_train_dataset.remove_columns(["text"]).with_format("tensorflow") tf_eval_dataset = small_eval_dataset.remove_columns(["text"]).with_format("tensorflow") #将所有内容转换为大张量并使用以下tf.data.Dataset.from_tensor_slices方法 train_features = {x: tf_train_dataset[x] for x in tokenizer.model_input_names} train_tf_dataset = tf.data.Dataset.from_tensor_slices((train_features, tf_train_dataset["label"])) train_tf_dataset = train_tf_dataset.shuffle(len(tf_train_dataset)).batch(8) eval_features = {x: tf_eval_dataset[x] for x in tokenizer.model_input_names} eval_tf_dataset = tf.data.Dataset.from_tensor_slices((eval_features, tf_eval_dataset["label"])) eval_tf_dataset = eval_tf_dataset.batch(8) #编译、训练 model.compile( optimizer=tf.keras.optimizers.Adam(learning_rate=5e-5), loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=tf.metrics.SparseCategoricalAccuracy(), ) model.fit(train_tf_dataset, validation_data=eval_tf_dataset, epochs=3) #保存模型,及重新加载为 PyTorch 模型(非必须) from transformers import AutoModelForSequenceClassification model.save_pretrained("my_imdb_model") pytorch_model = AutoModelForSequenceClassification.from_pretrained("my_imdb_model", from_tf=True) '''Fine-tuning in native PyTorch''' #前面操作会占用一定量的内存,可以先把其释放掉(非必须) del model del pytorch_model del trainer torch.cuda.empty_cache() #定义数据加载器,使用它来进行迭代批次。tokenized_datasets在执行此操作之前,需要对其进行一些后处理: #删除与模型不期望的值相对应的列(这里是"text"列) #将列重命名"label"为"labels"(因为模型期望参数被命名labels) #设置数据集的格式,以便它们返回 PyTorch 张量而不是列表。 tokenized_datasets = tokenized_datasets.remove_columns(["text"]) tokenized_datasets = tokenized_datasets.rename_column("label", "labels") tokenized_datasets.set_format("torch") small_train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(1000)) small_eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(1000)) #定义数据加载器 from torch.utils.data import DataLoader train_dataloader = DataLoader(small_train_dataset, shuffle=True, batch_size=8) eval_dataloader = DataLoader(small_eval_dataset, batch_size=8) #定义模型 from transformers import AutoModelForSequenceClassification model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=2) #优化器和学习率调度器 from transformers import AdamW optimizer = AdamW(model.parameters(), lr=5e-5) #学习率设置为从最大值(此处为 5e-5)到 0 的线性衰减 from transformers import get_scheduler num_epochs = 3 num_training_steps = num_epochs * len(train_dataloader) lr_scheduler = get_scheduler( "linear", optimizer=optimizer, num_warmup_steps=0, num_training_steps=num_training_steps ) #定义一个device放置模型(如果有GPU可用,会快很多) import torch device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") model.to(device) #开始训练,为直观展示,训练步骤中添加了一个进度条 from tqdm.auto import tqdm progress_bar = tqdm(range(num_training_steps)) model.train() for epoch in range(num_epochs): for batch in train_dataloader: batch = {k: v.to(device) for k, v in batch.items()} outputs = model(**batch) loss = outputs.loss loss.backward() optimizer.step() lr_scheduler.step() optimizer.zero_grad() progress_bar.update(1) #验证评估 metric= load_metric("accuracy") model.eval() for batch in eval_dataloader: batch = {k: v.to(device) for k, v in batch.items()} with torch.no_grad(): outputs = model(**batch) logits = outputs.logits predictions = torch.argmax(logits, dim=-1) metric.add_batch(predictions=predictions, references=batch["labels"]) metric.compute()
Copyright © 2003-2013 www.wpsshop.cn 版权所有,并保留所有权利。