python垃圾分类源代码_【自然语言处理】的迁移学习：微调BERT来对垃圾邮件进行分类【含Python演示】...

!pip install transformers

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
import transformers
from transformers import AutoModel, BertTokenizerFast
 
# specify GPU
device = torch.device("cuda")

df = pd.read_csv("spamdata_v2.csv")
df.head()

# split train dataset into train, validation and test sets
 

# import BERT-base pretrained model
bert = AutoModel.from_pretrained('bert-base-uncased')
 
# Load the BERT tokenizer
tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')

# sample data
text = ["this is a bert model tutorial", "we will fine-tune a bert model"]
 
# encode text
sent_id = tokenizer.batch_encode_plus(text, padding=True)
 
# output
print(sent_id)

# get length of all the messages in the train set
seq_len = [len(i.split()) for i in train_text]
 
pd.Series(seq_len).hist(bins = 30)

# tokenize and encode sequences in the training set
tokens_train = tokenizer.batch_encode_plus(
    train_text.tolist(),
    max_length = 25,
    pad_to_max_length=True,
    truncation=True
)
 
# tokenize and encode sequences in the validation set
tokens_val = tokenizer.batch_encode_plus(
    val_text.tolist(),
    max_length = 25,
    pad_to_max_length=True,
    truncation=True
)
 
# tokenize and encode sequences in the test set
tokens_test = tokenizer.batch_encode_plus(
    test_text.tolist(),
    max_length = 25,
    pad_to_max_length=True,
    truncation=True

## convert lists to tensors
 
train_seq = torch.tensor(tokens_train['input_ids'])
train_mask = torch.tensor(tokens_train['attention_mask'])
train_y = torch.tensor(train_labels.tolist())
 
val_seq = torch.tensor(tokens_val['input_ids'])
val_mask = torch.tensor(tokens_val['attention_mask'])
val_y = torch.tensor(val_labels.tolist())
 
test_seq = torch.tensor(tokens_test['input_ids'])
test_mask = torch.tensor(tokens_test['attention_mask'])
test_y = torch.tensor(test_labels.tolist())

from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
 
#define a batch size
batch_size = 32
 
# wrap tensors
train_data = TensorDataset(train_seq, train_mask, train_y)
 
# sampler for sampling the data during training
train_sampler = RandomSampler(train_data)
 
# dataLoader for train set
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)
 
# wrap tensors
val_data = TensorDataset(val_seq, val_mask, val_y)
 
# sampler for sampling the data during training
val_sampler = SequentialSampler(val_data)
 
# dataLoader for validation set
val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)

# freeze all the parameters
for param in bert.parameters():
    param.requires_grad = False

class BERT_Arch(nn.Module):
 
    def __init__(self, bert):
      
      super(BERT_Arch, self).__init__()
 
      self.bert = bert 
      
      # dropout layer
      self.dropout = nn.Dropout(0.1)
      
      # relu activation function
      self.relu =  nn.ReLU()
 
      # dense layer 1
      self.fc1 = nn.Linear(768,512)
      
      # dense layer 2 (Output layer)
      self.fc2 = nn.Linear(512,2)
 
      #softmax activation function
      self.softmax = nn.LogSoftmax(dim=1)
 
    #define the forward pass
    def forward(self, sent_id, mask):
 
      #pass the inputs to the model  
      _, cls_hs = self.bert(sent_id, attention_mask=mask)
      
      x = self.fc1(cls_hs)
 
      x = self.relu(x)
 
      x = self.dropout(x)
 
      # output layer
      x = self.fc2(x)
      
      # apply softmax activation
      x = self.softmax(x)
 
      return x

# pass the pre-trained BERT to our define architecture
model = BERT_Arch(bert)
 
# push the model to GPU
model = model.to(device)

# optimizer from hugging face transformers
from transformers import AdamW
 
# define the optimizer
optimizer = AdamW(model.parameters(),
                  lr = 1e-5)          # learning rate

from sklearn.utils.class_weight import compute_class_weight
 
#compute the class weights
class_weights = compute_class_weight('balanced', np.unique(train_labels), train_labels)
 
print("Class Weights:",class_weights)

# converting list of class weights to a tensor
weights= torch.tensor(class_weights,dtype=torch.float)
 
# push to GPU
weights = weights.to(device)
 
# define the loss function
cross_entropy  = nn.NLLLoss(weight=weights) 
 
# number of training epochs
epochs = 10

# function to train the model
def train():
  
  model.train()
 
  total_loss, total_accuracy = 0, 0
  
  # empty list to save model predictions
  total_preds=[]
  
  # iterate over batches
  for step,batch in enumerate(train_dataloader):
    
    # progress update after every 50 batches.
    if step % 50 == 0 and not step == 0:
      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))
 
    # push the batch to gpu
    batch = [r.to(device) for r in batch]
 
    sent_id, mask, labels = batch
 
    # clear previously calculated gradients 
    model.zero_grad()        
 
    # get model predictions for the current batch
    preds = model(sent_id, mask)
 
    # compute the loss between actual and predicted values
    loss = cross_entropy(preds, labels)
 
    # add on to the total loss
    total_loss = total_loss + loss.item()
 
    # backward pass to calculate the gradients
    loss.backward()
 
    # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
 
    # update parameters
    optimizer.step()
 
    # model predictions are stored on GPU. So, push it to CPU
    preds=preds.detach().cpu().numpy()
 
    # append the model predictions
    total_preds.append(preds)
 
  # compute the training loss of the epoch
  avg_loss = total_loss / len(train_dataloader)
  
  # predictions are in the form of (no. of batches, size of batch, no. of classes).
  # reshape the predictions in form of (number of samples, no. of classes)
  total_preds  = np.concatenate(total_preds, axis=0)
 
  #returns the loss and predictions
  return avg_loss, total_preds

# function for evaluating the model
def evaluate():
  
  print("nEvaluating...")
  
  # deactivate dropout layers
  model.eval()
 
  total_loss, total_accuracy = 0, 0
  
  # empty list to save the model predictions
  total_preds = []
 
  # iterate over batches
  for step,batch in enumerate(val_dataloader):
    
    # Progress update every 50 batches.
    if step % 50 == 0 and not step == 0:
      
      # Calculate elapsed time in minutes.
      elapsed = format_time(time.time() - t0)
            
      # Report progress.
      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))
 
    # push the batch to gpu
    batch = [t.to(device) for t in batch]
 
    sent_id, mask, labels = batch
 
    # deactivate autograd
    with torch.no_grad():
      
      # model predictions
      preds = model(sent_id, mask)
 
      # compute the validation loss between actual and predicted values
      loss = cross_entropy(preds,labels)
 
      total_loss = total_loss + loss.item()
 
      preds = preds.detach().cpu().numpy()
 
      total_preds.append(preds)
 
  # compute the validation loss of the epoch
  avg_loss = total_loss / len(val_dataloader) 
 
  # reshape the predictions in form of (number of samples, no. of classes)
  total_preds  = np.concatenate(total_preds, axis=0)
 
  return avg_loss, total_preds

Training Loss: 0.592
Validation Loss: 0.567
 
Epoch 5 / 10
Batch 50 of 122.
Batch 100 of 122.
 
Evaluating...
 
Training Loss: 0.566
Validation Loss: 0.543
 
Epoch 6 / 10
Batch 50 of 122.
Batch 100 of 122.
 
Evaluating...
 
Training Loss: 0.552
Validation Loss: 0.525
 
Epoch 7 / 10
Batch 50 of 122.
Batch 100 of 122.
 
Evaluating...
 
Training Loss: 0.525
Validation Loss: 0.498
 
Epoch 8 / 10
Batch 50 of 122.
Batch 100 of 122.
 
Evaluating...
 
Training Loss: 0.507
Validation Loss: 0.477
 
Epoch 9 / 10
Batch 50 of 122.
Batch 100 of 122.
 
Evaluating...
 
Training Loss: 0.488
Validation Loss: 0.461
 
Epoch 10 / 10
Batch 50 of 122.
Batch 100 of 122.
 
Evaluating...
 
Training Loss: 0.474
Validation Loss: 0.454

#load weights of best model
path = 'saved_weights.pt'
model.load_state_dict(torch.load(path))

# get predictions for test data
with torch.no_grad():
  preds = model(test_seq.to(device), test_mask.to(device))
  preds = preds.detach().cpu().numpy()

preds = np.argmax(preds, axis = 1)
print(classification_report(test_y, preds))