devbox
小小林熬夜学编程
这个屌丝很懒，什么也没留下！
关注作者
热门标签
热门文章
当前位置:   article > 正文

预训练模型transformers综合总结（三）_automodelforcausallm.from_pretrained

作者：小小林熬夜学编程 | 2024-03-01 17:03:10

赞

踩

automodelforcausallm.from_pretrained

这就是最后的部分，关于语言建模、多选、问答系统、文本分类、命名实体识别等任务的模型微调，这一块我也是直接看人家笔记本总结的。按照这三个部分，这个库基本就弄熟了

语言建模

主要有这两种方式进行建模

  • 因果语言建模：模型必须预测句子中的下一个标记（因此标签与向右移动的输入相同）。为确保模型不作弊，当尝试预测句子中的标记i + 1时，它将获得一个注意掩码，以防止其访问标记i之后的标记。
  • 屏蔽语言建模：模型必须预测输入中被屏蔽的一些标记。它仍然可以访问整个句子，因此它可以在屏蔽标记之前和之后使用标记来预测其值。
  1. 数据加载
  2. from datasets import load_dataset
  3. datasets = load_dataset('wikitext', 'wikitext-2-raw-v1')
  4. #可以替换成如下的，取消注释即可，只需取消注释以下单元格，然后将路径替换为本地地址即可：
  5. # datasets = load_dataset("text", data_files={"train": path_to_train.txt, "validation": path_to_validation.txt}
  6. datasets["train"][10]
  7. from datasets import ClassLabel
  8. import random
  9. import pandas as pd
  10. from IPython.display import display, HTML
  11.  
  12. def show_random_elements(dataset, num_examples=10):
  13.     assert num_examples <= len(dataset), "Can't pick more elements than there are in the dataset."
  14.     picks = []
  15.     for _ in range(num_examples):
  16.         pick = random.randint(0, len(dataset)-1)
  17.         while pick in picks:
  18.             pick = random.randint(0, len(dataset)-1)
  19.         picks.append(pick)
  20.     
  21.     df = pd.DataFrame(dataset[picks])
  22.     for column, typ in dataset.features.items():
  23.         if isinstance(typ, ClassLabel):
  24.             df[column] = df[column].transform(lambda i: typ.names[i])
  25.     display(HTML(df.to_html()))
  26. show_random_elements(datasets["train"])
  27.

因果建模

  1. model_path="H:\\code\\Model\\distilgpt2\\"
  2. from transformers import AutoTokenizer
  3.     
  4. tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
  5. def tokenize_function(examples):
  6.     return tokenizer(examples["text"])
  7. tokenized_datasets = datasets.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"])
  8. tokenized_datasets["train"][1]
  9. # block_size = tokenizer.model_max_length
  10. block_size = 128
  11. #数据预处理
  12. def group_texts(examples):
  13.     # Concatenate all texts.
  14.     concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
  15.     total_length = len(concatenated_examples[list(examples.keys())[0]])
  16.     # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
  17.         # customize this part to your needs.
  18.     total_length = (total_length // block_size) * block_size
  19.     # Split by chunks of max_len.
  20.     result = {
  21.         k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
  22.         for k, t in concatenated_examples.items()
  23.     }
  24.     result["labels"] = result["input_ids"].copy()
  25.     return result
  26. lm_datasets = tokenized_datasets.map(
  27.     group_texts,
  28.     batched=True,
  29.     batch_size=1000,
  30.     num_proc=4,
  31. )
  32. tokenizer.decode(lm_datasets["train"][1]["input_ids"])
  33. from transformers import AutoModelForCausalLM
  34. model = AutoModelForCausalLM.from_pretrained(model_path)
  35. from transformers import Trainer, TrainingArguments
  36. training_args = TrainingArguments(
  37.     "test-clm",
  38.     evaluation_strategy = "epoch",
  39.     learning_rate=2e-5,
  40.     weight_decay=0.01,
  41. )
  42. trainer = Trainer(
  43.     model=model,
  44.     args=training_args,
  45.     train_dataset=lm_datasets["train"],
  46.     eval_dataset=lm_datasets["validation"],
  47. )
  48. trainer.train()
  49. import math
  50. eval_results = trainer.evaluate()
  51. print(f"Perplexity: {math.exp(eval_results['eval_loss']):.2f}")


掩盖语言建模

  1. model_path="H:\\code\\Model\\distilroberta-base\\"
  2. tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True)
  3. tokenized_datasets = datasets.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"])
  4. lm_datasets = tokenized_datasets.map(
  5.     group_texts,
  6.     batched=True,
  7.     batch_size=1000,
  8.     num_proc=4,
  9. )
  10. from transformers import AutoModelForMaskedLM
  11. model = AutoModelForMaskedLM.from_pretrained(model_path)
  12. from transformers import DataCollatorForLanguageModeling
  13. data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm_probability=0.15)
  14. trainer = Trainer(
  15.     model=model,
  16.     args=training_args,
  17.     train_dataset=lm_datasets["train"],
  18.     eval_dataset=lm_datasets["validation"],
  19.     data_collator=data_collator,
  20. )
  21. trainer.train()
  22. eval_results = trainer.evaluate()
  23. print(f"Perplexity: {math.exp(eval_results['eval_loss']):.2f}")

多项选择任务

  1. model_checkpoint ="H:\\code\\Model\\bert-base-uncased\\"
  2. batch_size = 16
  3. from datasets import load_dataset, load_metric
  4. datasets = load_dataset("swag", "regular")
  5. datasets
  6. datasets["train"][0]
  7. from datasets import ClassLabel
  8. import random
  9. import pandas as pd
  10. from IPython.display import display, HTML
  11.  
  12. def show_random_elements(dataset, num_examples=10):
  13.     assert num_examples <= len(dataset), "Can't pick more elements than there are in the dataset."
  14.     picks = []
  15.     for _ in range(num_examples):
  16.         pick = random.randint(0, len(dataset)-1)
  17.         while pick in picks:
  18.             pick = random.randint(0, len(dataset)-1)
  19.         picks.append(pick)
  20.     
  21.     df = pd.DataFrame(dataset[picks])
  22.     for column, typ in dataset.features.items():
  23.         if isinstance(typ, ClassLabel):
  24.             df[column] = df[column].transform(lambda i: typ.names[i])
  25.     display(HTML(df.to_html()))
  26. show_random_elements(datasets["train"])
  27. def show_one(example):
  28.     print(f"Context: {example['sent1']}")
  29.     print(f"  A - {example['sent2']} {example['ending0']}")
  30.     print(f"  B - {example['sent2']} {example['ending1']}")
  31.     print(f"  C - {example['sent2']} {example['ending2']}")
  32.     print(f"  D - {example['sent2']} {example['ending3']}")
  33.     print(f"\nGround truth: option {['A', 'B', 'C', 'D'][example['label']]}")
  34. #预处理数据
  35. from transformers import AutoTokenizer
  36.     
  37. tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)
  38. ending_names = ["ending0", "ending1", "ending2", "ending3"]
  39.  
  40. def preprocess_function(examples):
  41.     # Repeat each first sentence four times to go with the four possibilities of second sentences.
  42.     first_sentences = [[context] * 4 for context in examples["sent1"]]
  43.     # Grab all second sentences possible for each context.
  44.     question_headers = examples["sent2"]
  45.     second_sentences = [[f"{header} {examples[end][i]}" for end in ending_names] for i, header in enumerate(question_headers)]
  46.     
  47.     # Flatten everything
  48.     first_sentences = sum(first_sentences, [])
  49.     second_sentences = sum(second_sentences, [])
  50.     
  51.     # Tokenize
  52.     tokenized_examples = tokenizer(first_sentences, second_sentences, truncation=True)
  53.     # Un-flatten
  54.     return {k: [v[i:i+4] for i in range(0, len(v), 4)] for k, v in tokenized_examples.items()}
  55. examples = datasets["train"][:5]
  56. features = preprocess_function(examples)
  57. print(len(features["input_ids"]), len(features["input_ids"][0]), [len(x) for x in features["input_ids"][0]])
  58. idx = 3
  59. [tokenizer.decode(features["input_ids"][idx][i]) for i in range(4)]
  60. show_one(datasets["train"][3])
  61. encoded_datasets = datasets.map(preprocess_function, batched=True)
  62. #微调模型
  63. from transformers import AutoModelForMultipleChoice, TrainingArguments, Trainer
  64.  
  65. model = AutoModelForMultipleChoice.from_pretrained(model_checkpoint)
  66. args = TrainingArguments(
  67.     "test-glue",
  68.     evaluation_strategy = "epoch",
  69.     learning_rate=5e-5,
  70.     per_device_train_batch_size=batch_size,
  71.     per_device_eval_batch_size=batch_size,
  72.     num_train_epochs=3,
  73.     weight_decay=0.01,
  74. )
  75. from dataclasses import dataclass
  76. from transformers.tokenization_utils_base import PreTrainedTokenizerBase, PaddingStrategy
  77. from typing import Optional, Union
  78. import torch
  79.  
  80. @dataclass
  81. class DataCollatorForMultipleChoice:
  82.     """
  83.     Data collator that will dynamically pad the inputs for multiple choice received.
  84.     """
  85.  
  86.     tokenizer: PreTrainedTokenizerBase
  87.     padding: Union[bool, str, PaddingStrategy] = True
  88.     max_length: Optional[int] = None
  89.     pad_to_multiple_of: Optional[int] = None
  90.  
  91.     def __call__(self, features):
  92.         label_name = "label" if "label" in features[0].keys() else "labels"
  93.         labels = [feature.pop(label_name) for feature in features]
  94.         batch_size = len(features)
  95.         num_choices = len(features[0]["input_ids"])
  96.         flattened_features = [[{k: v[i] for k, v in feature.items()} for i in range(num_choices)] for feature in features]
  97.         flattened_features = sum(flattened_features, [])
  98.         
  99.         batch = self.tokenizer.pad(
  100.             flattened_features,
  101.             padding=self.padding,
  102.             max_length=self.max_length,
  103.             pad_to_multiple_of=self.pad_to_multiple_of,
  104.             return_tensors="pt",
  105.         )
  106.         
  107.         # Un-flatten
  108.         batch = {k: v.view(batch_size, num_choices, -1) for k, v in batch.items()}
  109.         # Add back labels
  110.         batch["labels"] = torch.tensor(labels, dtype=torch.int64)
  111.         return batch
  112. accepted_keys = ["input_ids", "attention_mask", "label"]
  113. features = [{k: v for k, v in encoded_datasets["train"][i].items() if k in accepted_keys} for i in range(10)]
  114. batch = DataCollatorForMultipleChoice(tokenizer)(features)
  115. import numpy as np
  116.  
  117. def compute_metrics(eval_predictions):
  118.     predictions, label_ids = eval_predictions
  119.     preds = np.argmax(predictions, axis=1)
  120.     return {"accuracy": (preds == label_ids).astype(np.float32).mean().item()}
  121. trainer = Trainer(
  122.     model,
  123.     args,
  124.     train_dataset=encoded_datasets["train"],
  125.     eval_dataset=encoded_datasets["validation"],
  126.     tokenizer=tokenizer,
  127.     data_collator=DataCollatorForMultipleChoice(tokenizer),
  128.     compute_metrics=compute_metrics,
  129. )
  130. trainer.train()
  131.

问答系统

  1. # This flag is the difference between SQUAD v1 or 2 (if you're using another dataset, it indicates if impossible
  2. # answers are allowed or not).
  3. squad_v2 = False
  4. model_checkpoint="H:\\code\\Model\\distilbert-base-cased\\"
  5. batch_size = 16
  6. from datasets import load_dataset, load_metric
  7. datasets = load_dataset("squad_v2" if squad_v2 else "squad")
  8. datasets
  9. from datasets import ClassLabel, Sequence
  10. import random
  11. import pandas as pd
  12. from IPython.display import display, HTML
  13.  
  14. def show_random_elements(dataset, num_examples=10):
  15.     assert num_examples <= len(dataset), "Can't pick more elements than there are in the dataset."
  16.     picks = []
  17.     for _ in range(num_examples):
  18.         pick = random.randint(0, len(dataset)-1)
  19.         while pick in picks:
  20.             pick = random.randint(0, len(dataset)-1)
  21.         picks.append(pick)
  22.     
  23.     df = pd.DataFrame(dataset[picks])
  24.     for column, typ in dataset.features.items():
  25.         if isinstance(typ, ClassLabel):
  26.             df[column] = df[column].transform(lambda i: typ.names[i])
  27.         elif isinstance(typ, Sequence) and isinstance(typ.feature, ClassLabel):
  28.             df[column] = df[column].transform(lambda x: [typ.feature.names[i] for i in x])
  29.     display(HTML(df.to_html()))
  30. show_random_elements(datasets["train"])
  31. #预处理训练数据
  32. from transformers import AutoTokenizer
  33.     
  34. tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
  35. import transformers
  36. assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)
  37. tokenizer("What is your name?", "My name is Sylvain.")
  38. max_length = 384 # The maximum length of a feature (question and context)
  39. doc_stride = 128 # The authorized overlap between two part of the context when splitting it is needed.
  40. for i, example in enumerate(datasets["train"]):
  41.     if len(tokenizer(example["question"], example["context"])["input_ids"]) > 384:
  42.         break
  43. example = datasets["train"][i]
  44. len(tokenizer(example["question"], example["context"])["input_ids"])
  45. len(tokenizer(example["question"], example["context"], max_length=max_length, truncation="only_second")["input_ids"])
  46. tokenized_example = tokenizer(
  47.     example["question"],
  48.     example["context"],
  49.     max_length=max_length,
  50.     truncation="only_second",
  51.     return_overflowing_tokens=True,
  52.     return_offsets_mapping=True,
  53.     stride=doc_stride
  54. )
  55. print(tokenized_example["offset_mapping"][0][:100])
  56. first_token_id = tokenized_example["input_ids"][0][1]
  57. offsets = tokenized_example["offset_mapping"][0][1]
  58. print(tokenizer.convert_ids_to_tokens([first_token_id])[0], example["question"][offsets[0]:offsets[1]])
  59. sequence_ids = tokenized_example.sequence_ids()
  60. print(sequence_ids)
  61. answers = example["answers"]
  62. start_char = answers["answer_start"][0]
  63. end_char = start_char + len(answers["text"][0])
  64.  
  65. # Start token index of the current span in the text.
  66. token_start_index = 0
  67. while sequence_ids[token_start_index] != 1:
  68.     token_start_index += 1
  69.  
  70. # End token index of the current span in the text.
  71. token_end_index = len(tokenized_example["input_ids"][0]) - 1
  72. while sequence_ids[token_end_index] != 1:
  73.     token_end_index -= 1
  74.  
  75. # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).
  76. offsets = tokenized_example["offset_mapping"][0]
  77. if (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):
  78.     # Move the token_start_index and token_end_index to the two ends of the answer.
  79.     # Note: we could go after the last offset if the answer is the last word (edge case).
  80.     while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:
  81.         token_start_index += 1
  82.     start_position = token_start_index - 1
  83.     while offsets[token_end_index][1] >= end_char:
  84.         token_end_index -= 1
  85.     end_position = token_end_index + 1
  86.     print(start_position, end_position)
  87. else:
  88.     print("The answer is not in this feature.")
  89. print(tokenizer.decode(tokenized_example["input_ids"][0][start_position: end_position+1]))
  90. print(answers["text"][0])
  91. pad_on_right = tokenizer.padding_side == "right"
  92. def prepare_train_features(examples):
  93.     # Tokenize our examples with truncation and padding, but keep the overflows using a stride. This results
  94.     # in one example possible giving several features when a context is long, each of those features having a
  95.     # context that overlaps a bit the context of the previous feature.
  96.     tokenized_examples = tokenizer(
  97.         examples["question" if pad_on_right else "context"],
  98.         examples["context" if pad_on_right else "question"],
  99.         truncation="only_second" if pad_on_right else "only_first",
  100.         max_length=max_length,
  101.         stride=doc_stride,
  102.         return_overflowing_tokens=True,
  103.         return_offsets_mapping=True,
  104.         padding="max_length",
  105.     )
  106.  
  107.     # Since one example might give us several features if it has a long context, we need a map from a feature to
  108.     # its corresponding example. This key gives us just that.
  109.     sample_mapping = tokenized_examples.pop("overflow_to_sample_mapping")
  110.     # The offset mappings will give us a map from token to character position in the original context. This will
  111.     # help us compute the start_positions and end_positions.
  112.     offset_mapping = tokenized_examples.pop("offset_mapping")
  113.  
  114.     # Let's label those examples!
  115.     tokenized_examples["start_positions"] = []
  116.     tokenized_examples["end_positions"] = []
  117.  
  118.     for i, offsets in enumerate(offset_mapping):
  119.         # We will label impossible answers with the index of the CLS token.
  120.         input_ids = tokenized_examples["input_ids"][i]
  121.         cls_index = input_ids.index(tokenizer.cls_token_id)
  122.  
  123.         # Grab the sequence corresponding to that example (to know what is the context and what is the question).
  124.         sequence_ids = tokenized_examples.sequence_ids(i)
  125.  
  126.         # One example can give several spans, this is the index of the example containing this span of text.
  127.         sample_index = sample_mapping[i]
  128.         answers = examples["answers"][sample_index]
  129.         # If no answers are given, set the cls_index as answer.
  130.         if len(answers["answer_start"]) == 0:
  131.             tokenized_examples["start_positions"].append(cls_index)
  132.             tokenized_examples["end_positions"].append(cls_index)
  133.         else:
  134.             # Start/end character index of the answer in the text.
  135.             start_char = answers["answer_start"][0]
  136.             end_char = start_char + len(answers["text"][0])
  137.  
  138.             # Start token index of the current span in the text.
  139.             token_start_index = 0
  140.             while sequence_ids[token_start_index] != (1 if pad_on_right else 0):
  141.                 token_start_index += 1
  142.  
  143.             # End token index of the current span in the text.
  144.             token_end_index = len(input_ids) - 1
  145.             while sequence_ids[token_end_index] != (1 if pad_on_right else 0):
  146.                 token_end_index -= 1
  147.  
  148.             # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).
  149.             if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):
  150.                 tokenized_examples["start_positions"].append(cls_index)
  151.                 tokenized_examples["end_positions"].append(cls_index)
  152.             else:
  153.                 # Otherwise move the token_start_index and token_end_index to the two ends of the answer.
  154.                 # Note: we could go after the last offset if the answer is the last word (edge case).
  155.                 while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:
  156.                     token_start_index += 1
  157.                 tokenized_examples["start_positions"].append(token_start_index - 1)
  158.                 while offsets[token_end_index][1] >= end_char:
  159.                     token_end_index -= 1
  160.                 tokenized_examples["end_positions"].append(token_end_index + 1)
  161.  
  162.     return tokenized_examples
  163. features = prepare_train_features(datasets['train'][:5])
  164. tokenized_datasets = datasets.map(prepare_train_features, batched=True, remove_columns=datasets["train"].column_names)
  165. #微调模型
  166. from transformers import AutoModelForQuestionAnswering, TrainingArguments, Trainer
  167.  
  168. model = AutoModelForQuestionAnswering.from_pretrained(model_checkpoint)
  169. args = TrainingArguments(
  170.     f"test-squad",
  171.     evaluation_strategy = "epoch",
  172.     learning_rate=2e-5,
  173.     per_device_train_batch_size=batch_size,
  174.     per_device_eval_batch_size=batch_size,
  175.     num_train_epochs=3,
  176.     weight_decay=0.01,
  177. )
  178. from transformers import default_data_collator
  179.  
  180. data_collator = default_data_collator
  181. trainer = Trainer(
  182.     model,
  183.     args,
  184.     train_dataset=tokenized_datasets["train"],
  185.     eval_dataset=tokenized_datasets["validation"],
  186.     data_collator=data_collator,
  187.     tokenizer=tokenizer,
  188. )
  189. trainer.train()
  190. trainer.save_model("test-squad-trained")
  191. #评估
  192. import torch
  193.  
  194. for batch in trainer.get_eval_dataloader():
  195.     break
  196. batch = {k: v.to(trainer.args.device) for k, v in batch.items()}
  197. with torch.no_grad():
  198.     output = trainer.model(**batch)
  199. output.keys()
  200. output.start_logits.shape, output.end_logits.shape
  201. output.start_logits.argmax(dim=-1), output.end_logits.argmax(dim=-1)
  202. n_best_size = 20
  203. import numpy as np
  204.  
  205. start_logits = output.start_logits[0].cpu().numpy()
  206. end_logits = output.end_logits[0].cpu().numpy()
  207. # Gather the indices the best start/end logits:
  208. start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()
  209. end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()
  210. valid_answers = []
  211. for start_index in start_indexes:
  212.     for end_index in end_indexes:
  213.         if start_index <= end_index: # We need to refine that test to check the answer is inside the context
  214.             valid_answers.append(
  215.                 {
  216.                     "score": start_logits[start_index] + end_logits[end_index],
  217.                     "text": "" # We need to find a way to get back the original substring corresponding to the answer in the context
  218.                 }
  219.             )
  220. def prepare_validation_features(examples):
  221.     # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results
  222.     # in one example possible giving several features when a context is long, each of those features having a
  223.     # context that overlaps a bit the context of the previous feature.
  224.     tokenized_examples = tokenizer(
  225.         examples["question" if pad_on_right else "context"],
  226.         examples["context" if pad_on_right else "question"],
  227.         truncation="only_second" if pad_on_right else "only_first",
  228.         max_length=max_length,
  229.         stride=doc_stride,
  230.         return_overflowing_tokens=True,
  231.         return_offsets_mapping=True,
  232.         padding="max_length",
  233.     )
  234.  
  235.     # Since one example might give us several features if it has a long context, we need a map from a feature to
  236.     # its corresponding example. This key gives us just that.
  237.     sample_mapping = tokenized_examples.pop("overflow_to_sample_mapping")
  238.  
  239.     # We keep the example_id that gave us this feature and we will store the offset mappings.
  240.     tokenized_examples["example_id"] = []
  241.  
  242.     for i in range(len(tokenized_examples["input_ids"])):
  243.         # Grab the sequence corresponding to that example (to know what is the context and what is the question).
  244.         sequence_ids = tokenized_examples.sequence_ids(i)
  245.         context_index = 1 if pad_on_right else 0
  246.  
  247.         # One example can give several spans, this is the index of the example containing this span of text.
  248.         sample_index = sample_mapping[i]
  249.         tokenized_examples["example_id"].append(examples["id"][sample_index])
  250.  
  251.         # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token
  252.         # position is part of the context or not.
  253.         tokenized_examples["offset_mapping"][i] = [
  254.             (o if sequence_ids[k] == context_index else None)
  255.             for k, o in enumerate(tokenized_examples["offset_mapping"][i])
  256.         ]
  257.  
  258.     return tokenized_examples
  259. validation_features = datasets["validation"].map(
  260.     prepare_validation_features,
  261.     batched=True,
  262.     remove_columns=datasets["validation"].column_names
  263. )
  264. raw_predictions = trainer.predict(validation_features)
  265. validation_features.set_format(type=validation_features.format["type"], columns=list(validation_features.features.keys()))
  266. max_answer_length = 30
  267. start_logits = output.start_logits[0].cpu().numpy()
  268. end_logits = output.end_logits[0].cpu().numpy()
  269. offset_mapping = validation_features[0]["offset_mapping"]
  270. # The first feature comes from the first example. For the more general case, we will need to be match the example_id to
  271. # an example index
  272. context = datasets["validation"][0]["context"]
  273.  
  274. # Gather the indices the best start/end logits:
  275. start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()
  276. end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()
  277. valid_answers = []
  278. for start_index in start_indexes:
  279.     for end_index in end_indexes:
  280.         # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond
  281.         # to part of the input_ids that are not in the context.
  282.         if (
  283.             start_index >= len(offset_mapping)
  284.             or end_index >= len(offset_mapping)
  285.             or offset_mapping[start_index] is None
  286.             or offset_mapping[end_index] is None
  287.         ):
  288.             continue
  289.         # Don't consider answers with a length that is either < 0 or > max_answer_length.
  290.         if end_index < start_index or end_index - start_index + 1 > max_answer_length:
  291.             continue
  292.         if start_index <= end_index: # We need to refine that test to check the answer is inside the context
  293.             start_char = offset_mapping[start_index][0]
  294.             end_char = offset_mapping[end_index][1]
  295.             valid_answers.append(
  296.                 {
  297.                     "score": start_logits[start_index] + end_logits[end_index],
  298.                     "text": context[start_char: end_char]
  299.                 }
  300.             )
  301.  
  302. valid_answers = sorted(valid_answers, key=lambda x: x["score"], reverse=True)[:n_best_size]
  303. valid_answers
  304. import collections
  305.  
  306. examples = datasets["validation"]
  307. features = validation_features
  308.  
  309. example_id_to_index = {k: i for i, k in enumerate(examples["id"])}
  310. features_per_example = collections.defaultdict(list)
  311. for i, feature in enumerate(features):
  312.     features_per_example[example_id_to_index[feature["example_id"]]].append(i)
  313. from tqdm.auto import tqdm
  314.  
  315. def postprocess_qa_predictions(examples, features, raw_predictions, n_best_size = 20, max_answer_length = 30):
  316.     all_start_logits, all_end_logits = raw_predictions
  317.     # Build a map example to its corresponding features.
  318.     example_id_to_index = {k: i for i, k in enumerate(examples["id"])}
  319.     features_per_example = collections.defaultdict(list)
  320.     for i, feature in enumerate(features):
  321.         features_per_example[example_id_to_index[feature["example_id"]]].append(i)
  322.  
  323.     # The dictionaries we have to fill.
  324.     predictions = collections.OrderedDict()
  325.  
  326.     # Logging.
  327.     print(f"Post-processing {len(examples)} example predictions split into {len(features)} features.")
  328.  
  329.     # Let's loop over all the examples!
  330.     for example_index, example in enumerate(tqdm(examples)):
  331.         # Those are the indices of the features associated to the current example.
  332.         feature_indices = features_per_example[example_index]
  333.  
  334.         min_null_score = None # Only used if squad_v2 is True.
  335.         valid_answers = []
  336.         
  337.         context = example["context"]
  338.         # Looping through all the features associated to the current example.
  339.         for feature_index in feature_indices:
  340.             # We grab the predictions of the model for this feature.
  341.             start_logits = all_start_logits[feature_index]
  342.             end_logits = all_end_logits[feature_index]
  343.             # This is what will allow us to map some the positions in our logits to span of texts in the original
  344.             # context.
  345.             offset_mapping = features[feature_index]["offset_mapping"]
  346.  
  347.             # Update minimum null prediction.
  348.             cls_index = features[feature_index]["input_ids"].index(tokenizer.cls_token_id)
  349.             feature_null_score = start_logits[cls_index] + end_logits[cls_index]
  350.             if min_null_score is None or min_null_score < feature_null_score:
  351.                 min_null_score = feature_null_score
  352.  
  353.             # Go through all possibilities for the `n_best_size` greater start and end logits.
  354.             start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()
  355.             end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()
  356.             for start_index in start_indexes:
  357.                 for end_index in end_indexes:
  358.                     # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond
  359.                     # to part of the input_ids that are not in the context.
  360.                     if (
  361.                         start_index >= len(offset_mapping)
  362.                         or end_index >= len(offset_mapping)
  363.                         or offset_mapping[start_index] is None
  364.                         or offset_mapping[end_index] is None
  365.                     ):
  366.                         continue
  367.                     # Don't consider answers with a length that is either < 0 or > max_answer_length.
  368.                     if end_index < start_index or end_index - start_index + 1 > max_answer_length:
  369.                         continue
  370.  
  371.                     start_char = offset_mapping[start_index][0]
  372.                     end_char = offset_mapping[end_index][1]
  373.                     valid_answers.append(
  374.                         {
  375.                             "score": start_logits[start_index] + end_logits[end_index],
  376.                             "text": context[start_char: end_char]
  377.                         }
  378.                     )
  379.         
  380.         if len(valid_answers) > 0:
  381.             best_answer = sorted(valid_answers, key=lambda x: x["score"], reverse=True)[0]
  382.         else:
  383.             # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid
  384.             # failure.
  385.             best_answer = {"text": "", "score": 0.0}
  386.         
  387.         # Let's pick our final answer: the best one or the null answer (only for squad_v2)
  388.         if not squad_v2:
  389.             predictions[example["id"]] = best_answer["text"]
  390.         else:
  391.             answer = best_answer["text"] if best_answer["score"] > min_null_score else ""
  392.             predictions[example["id"]] = answer
  393.  
  394.     return predictions
  395. final_predictions = postprocess_qa_predictions(datasets["validation"], validation_features, raw_predictions.predictions)
  396. metric = load_metric("squad_v2" if squad_v2 else "squad")
  397. if squad_v2:
  398.     formatted_predictions = [{"id": k, "prediction_text": v, "no_answer_probability": 0.0} for k, v in predictions.items()]
  399. else:
  400.     formatted_predictions = [{"id": k, "prediction_text": v} for k, v in final_predictions.items()]
  401. references = [{"id": ex["id"], "answers": ex["answers"]} for ex in datasets["validation"]]
  402. metric.compute(predictions=formatted_predictions, references=references)

文本分类

  1. GLUE_TASKS = ["cola", "mnli", "mnli-mm", "mrpc", "qnli", "qqp", "rte", "sst2", "stsb", "wnli"]
  2. task = "cola"
  3. model_checkpoint="H:\\code\\Model\\distilbert-base-cased\\"
  4. batch_size = 16
  5. from datasets import load_dataset, load_metric
  6. actual_task = "mnli" if task == "mnli-mm" else task
  7. dataset = load_dataset("glue", actual_task)
  8. metric = load_metric('glue', actual_task)
  9. import datasets
  10. import random
  11. import pandas as pd
  12. from IPython.display import display, HTML
  13.  
  14. def show_random_elements(dataset, num_examples=10):
  15.     assert num_examples <= len(dataset), "Can't pick more elements than there are in the dataset."
  16.     picks = []
  17.     for _ in range(num_examples):
  18.         pick = random.randint(0, len(dataset)-1)
  19.         while pick in picks:
  20.             pick = random.randint(0, len(dataset)-1)
  21.         picks.append(pick)
  22.     
  23.     df = pd.DataFrame(dataset[picks])
  24.     for column, typ in dataset.features.items():
  25.         if isinstance(typ, datasets.ClassLabel):
  26.             df[column] = df[column].transform(lambda i: typ.names[i])
  27.     display(HTML(df.to_html()))
  28. show_random_elements(dataset["train"])
  29. import numpy as np
  30.  
  31. fake_preds = np.random.randint(0, 2, size=(64,))
  32. fake_labels = np.random.randint(0, 2, size=(64,))
  33. metric.compute(predictions=fake_preds, references=fake_labels)
  34. #预处理数据
  35. from transformers import AutoTokenizer
  36.     
  37. tokenizer = AutoTokenizer.from_pretrained(model_checkpoint, use_fast=True)
  38. tokenizer("Hello, this one sentence!", "And this sentence goes with it.")
  39. task_to_keys = {
  40.     "cola": ("sentence", None),
  41.     "mnli": ("premise", "hypothesis"),
  42.     "mnli-mm": ("premise", "hypothesis"),
  43.     "mrpc": ("sentence1", "sentence2"),
  44.     "qnli": ("question", "sentence"),
  45.     "qqp": ("question1", "question2"),
  46.     "rte": ("sentence1", "sentence2"),
  47.     "sst2": ("sentence", None),
  48.     "stsb": ("sentence1", "sentence2"),
  49.     "wnli": ("sentence1", "sentence2"),
  50. }
  51. sentence1_key, sentence2_key = task_to_keys[task]
  52. if sentence2_key is None:
  53.     print(f"Sentence: {dataset['train'][0][sentence1_key]}")
  54. else:
  55.     print(f"Sentence 1: {dataset['train'][0][sentence1_key]}")
  56.     print(f"Sentence 2: {dataset['train'][0][sentence2_key]}")
  57. def preprocess_function(examples):
  58.     if sentence2_key is None:
  59.         return tokenizer(examples[sentence1_key], truncation=True)
  60.     return tokenizer(examples[sentence1_key], examples[sentence2_key], truncation=True)
  61. preprocess_function(dataset['train'][:5])
  62. encoded_dataset = dataset.map(preprocess_function, batched=True)
  63. #微调模型
  64. from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer
  65.  
  66. num_labels = 3 if task.startswith("mnli") else 1 if task=="stsb" else 2
  67. model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)
  68. metric_name = "pearson" if task == "stsb" else "matthews_correlation" if task == "cola" else "accuracy"
  69.  
  70. args = TrainingArguments(
  71.     "test-glue",
  72.     evaluation_strategy = "epoch",
  73.     learning_rate=2e-5,
  74.     per_device_train_batch_size=batch_size,
  75.     per_device_eval_batch_size=batch_size,
  76.     num_train_epochs=5,
  77.     weight_decay=0.01,
  78.     load_best_model_at_end=True,
  79.     metric_for_best_model=metric_name,
  80. )
  81. def compute_metrics(eval_pred):
  82.     predictions, labels = eval_pred
  83.     if task != "stsb":
  84.         predictions = np.argmax(predictions, axis=1)
  85.     else:
  86.         predictions = predictions[:, 0]
  87.     return metric.compute(predictions=predictions, references=labels)
  88. validation_key = "validation_mismatched" if task == "mnli-mm" else "validation_matched" if task == "mnli" else "validation"
  89. trainer = Trainer(
  90.     model,
  91.     args,
  92.     train_dataset=encoded_dataset["train"],
  93.     eval_dataset=encoded_dataset[validation_key],
  94.     tokenizer=tokenizer,
  95.     compute_metrics=compute_metrics
  96. )
  97. trainer.train()
  98. trainer.evaluate()
  99. #超参数搜索
  100. def model_init():
  101.     return AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)
  102. trainer = Trainer(
  103.     model_init=model_init,
  104.     args=args,
  105.     train_dataset=encoded_dataset["train"],
  106.     eval_dataset=encoded_dataset[validation_key],
  107.     tokenizer=tokenizer,
  108.     compute_metrics=compute_metrics
  109. )
  110. best_run = trainer.hyperparameter_search(n_trials=10, direction="maximize")
  111. best_run
  112. for n, v in best_run.hyperparameters.items():
  113.     setattr(trainer.args, n, v)
  114.  
  115. trainer.train()

命名实体识别

  1. task = "ner" # Should be one of "ner", "pos" or "chunk"
  2. model_checkpoint="H:\\code\\Model\\distilbert-base-cased\\"
  3. batch_size = 16
  4. from datasets import load_dataset, load_metric
  5. datasets = load_dataset("conll2003")
  6. datasets
  7. label_list = datasets["train"].features[f"{task}_tags"].feature.names
  8. label_list
  9. from datasets import ClassLabel, Sequence
  10. import random
  11. import pandas as pd
  12. from IPython.display import display, HTML
  13.  
  14. def show_random_elements(dataset, num_examples=10):
  15.     assert num_examples <= len(dataset), "Can't pick more elements than there are in the dataset."
  16.     picks = []
  17.     for _ in range(num_examples):
  18.         pick = random.randint(0, len(dataset)-1)
  19.         while pick in picks:
  20.             pick = random.randint(0, len(dataset)-1)
  21.         picks.append(pick)
  22.     
  23.     df = pd.DataFrame(dataset[picks])
  24.     for column, typ in dataset.features.items():
  25.         if isinstance(typ, ClassLabel):
  26.             df[column] = df[column].transform(lambda i: typ.names[i])
  27.         elif isinstance(typ, Sequence) and isinstance(typ.feature, ClassLabel):
  28.             df[column] = df[column].transform(lambda x: [typ.feature.names[i] for i in x])
  29.     display(HTML(df.to_html()))
  30. show_random_elements(datasets["train"])
  31. #预处理数据
  32. from transformers import AutoTokenizer
  33.     
  34. tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
  35. import transformers
  36. assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)
  37. tokenizer("Hello, this is one sentence!")
  38. example = datasets["train"][4]
  39. print(example["tokens"])
  40. tokenized_input = tokenizer(example["tokens"], is_split_into_words=True)
  41. tokens = tokenizer.convert_ids_to_tokens(tokenized_input["input_ids"])
  42. print(tokens)
  43. word_ids = tokenized_input.word_ids()
  44. aligned_labels = [-100 if i is None else example[f"{task}_tags"][i] for i in word_ids]
  45. print(len(aligned_labels), len(tokenized_input["input_ids"]))
  46. label_all_tokens = True
  47. def tokenize_and_align_labels(examples):
  48.     tokenized_inputs = tokenizer(examples["tokens"], truncation=True, is_split_into_words=True)
  49.  
  50.     labels = []
  51.     for i, label in enumerate(examples[f"{task}_tags"]):
  52.         word_ids = tokenized_inputs.word_ids(batch_index=i)
  53.         previous_word_idx = None
  54.         label_ids = []
  55.         for word_idx in word_ids:
  56.             # Special tokens have a word id that is None. We set the label to -100 so they are automatically
  57.             # ignored in the loss function.
  58.             if word_idx is None:
  59.                 label_ids.append(-100)
  60.             # We set the label for the first token of each word.
  61.             elif word_idx != previous_word_idx:
  62.                 label_ids.append(label[word_idx])
  63.             # For the other tokens in a word, we set the label to either the current label or -100, depending on
  64.             # the label_all_tokens flag.
  65.             else:
  66.                 label_ids.append(label[word_idx] if label_all_tokens else -100)
  67.             previous_word_idx = word_idx
  68.  
  69.         labels.append(label_ids)
  70.  
  71.     tokenized_inputs["labels"] = labels
  72.     return tokenized_inputs
  73. tokenize_and_align_labels(datasets['train'][:5])
  74. tokenized_datasets = datasets.map(tokenize_and_align_labels, batched=True)
  75. #微调模型
  76. from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer
  77.  
  78. model = AutoModelForTokenClassification.from_pretrained(model_checkpoint, num_labels=len(label_list))
  79. args = TrainingArguments(
  80.     f"test-{task}",
  81.     evaluation_strategy = "epoch",
  82.     learning_rate=2e-5,
  83.     per_device_train_batch_size=batch_size,
  84.     per_device_eval_batch_size=batch_size,
  85.     num_train_epochs=3,
  86.     weight_decay=0.01,
  87. )
  88. from transformers import DataCollatorForTokenClassification
  89.  
  90. data_collator = DataCollatorForTokenClassification(tokenizer)
  91. metric = load_metric("seqeval")
  92. labels = [label_list[i] for i in example[f"{task}_tags"]]
  93. metric.compute(predictions=[labels], references=[labels])
  94. import numpy as np
  95.  
  96. def compute_metrics(p):
  97.     predictions, labels = p
  98.     predictions = np.argmax(predictions, axis=2)
  99.  
  100.     # Remove ignored index (special tokens)
  101.     true_predictions = [
  102.         [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
  103.         for prediction, label in zip(predictions, labels)
  104.     ]
  105.     true_labels = [
  106.         [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
  107.         for prediction, label in zip(predictions, labels)
  108.     ]
  109.  
  110.     results = metric.compute(predictions=true_predictions, references=true_labels)
  111.     return {
  112.         "precision": results["overall_precision"],
  113.         "recall": results["overall_recall"],
  114.         "f1": results["overall_f1"],
  115.         "accuracy": results["overall_accuracy"],
  116.     }
  117. trainer = Trainer(
  118.     model,
  119.     args,
  120.     train_dataset=tokenized_datasets["train"],
  121.     eval_dataset=tokenized_datasets["validation"],
  122.     data_collator=data_collator,
  123.     tokenizer=tokenizer,
  124.     compute_metrics=compute_metrics
  125. )
  126. trainer.train()
  127. trainer.evaluate()
  128. predictions, labels, _ = trainer.predict(tokenized_datasets["validation"])
  129. predictions = np.argmax(predictions, axis=2)
  130.  
  131. # Remove ignored index (special tokens)
  132. true_predictions = [
  133.     [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
  134.     for prediction, label in zip(predictions, labels)
  135. ]
  136. true_labels = [
  137.     [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
  138.     for prediction, label in zip(predictions, labels)
  139. ]
  140.  
  141. results = metric.compute(predictions=true_predictions, references=true_labels)
  142. results

 

声明：本文内容由网友自发贡献，不代表【wpsshop博客】立场，版权归原作者所有，本站不承担相应法律责任。如您发现有侵权的内容，请联系我们。转载请注明出处：https://www.wpsshop.cn/w/小小林熬夜学编程/article/detail/175488
推荐阅读
相关标签

Copyright © 2003-2013 www.wpsshop.cn 版权所有，并保留所有权利。

  

闽ICP备14008679号