第14章 用BERT实现中文语句分类_14.1 背景说明 本章用到预训练模型库transformers为自然语言理解(nlu)和自然语言

1

pip install transformers

1

2

3

4

5

6

7

8

import torch

from transformers import BertTokenizer

from IPython.display import clear_output

# 指定繁简中文 BERT-BASE预训练模型

PRETRAINED_MODEL_NAME = "bert-base-chinese"  

# 获取预测模型所使用的tokenizer

tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)

1

2

vocab = tokenizer.vocab

print("字典大小：", len(vocab))

1

2

3

4

5

6

7

8

import random

random_tokens = random.sample(list(vocab), 5)

random_ids = [vocab[t] for t in random_tokens]

print("{0:20}{1:15}".format("token", "index"))

print("-" * 30)

for t, id in zip(random_tokens, random_ids):

    print("{0:15}{1:10}".format(t, id))

1

2

3

4

5

6

7

text = "[CLS] 他移开这[MASK]桌子，就看到他的手表了。"

tokens = tokenizer.tokenize(text)

ids = tokenizer.convert_tokens_to_ids(tokens)

print(text)

print(tokens[:10], '...')

print(ids[:10], '...')

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

"""

导入已训练好的masked语言模型并对有[MASK]的句子做预测

"""

from transformers import BertForMaskedLM

# 除了tokens 以外我们还需要辨別句子的segment ids

tokens_tensor = torch.tensor([ids])  # (1, seq_len)

segments_tensors = torch.zeros_like(tokens_tensor)  # (1, seq_len)

maskedLM_model = BertForMaskedLM.from_pretrained(PRETRAINED_MODEL_NAME)

clear_output()

# 使用masked LM 估计[MASK]位置所代表的实际标识符(token)

maskedLM_model.eval()

with torch.no_grad():

    outputs = maskedLM_model(tokens_tensor, segments_tensors)

    predictions = outputs[0]

    # (1, seq_len, num_hidden_units)

del maskedLM_model

# 将[MASK]位置的概率分布取前k个最有可能的标识符出来

masked_index = 5

k = 3

probs, indices = torch.topk(torch.softmax(predictions[0, masked_index], -1), k)

predicted_tokens = tokenizer.convert_ids_to_tokens(indices.tolist())

# 显示前k个最可能的字。一般取第一个作为预测值

print("輸入 tokens ：", tokens[:10], '...')

print('-' * 50)

for i, (t, p) in enumerate(zip(predicted_tokens, probs), 1):

    tokens[masked_index] = t

    print("Top {} ({:2}%)：{}".format(i, int(p.item() * 100), tokens[:10]), '...')

1

2

from transformers import BertTokenizer, BertModel

from bertv_master.bertviz import head_view

1

2

3

4

5

6

# 在 jupyter notebook 显示visualzation

def call_html():

  import IPython

  display(IPython.core.display.HTML('''<script src="/static/components/requirejs/require.js"></script><script>// <![CDATA[

requirejs.config({ paths: { base: '/static/base', "d3": "https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.8/d3.min", jquery: '//ajax.googleapis.com/ajax/libs/jquery/2.0.0/jquery.min', }, });

// ]]></script>'''))

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

# 記得我们是使用中文 BERT

model_version = 'bert-base-chinese'

model = BertModel.from_pretrained(model_version, output_attentions=True)

tokenizer = BertTokenizer.from_pretrained(model_version)

# 情境 1 的句子

sentence_a = "老爸叫小宏去买酱油，"

sentence_b = "回来慢了就骂他。"

# 得到tokens后输入BERT模型获取注意力权重(attention)

inputs = tokenizer.encode_plus(sentence_a,sentence_b,return_tensors='pt', add_special_tokens=True)

token_type_ids = inputs['token_type_ids']

input_ids = inputs['input_ids']

attention = model(input_ids, token_type_ids=token_type_ids)[-1]

input_id_list = input_ids[0].tolist() # Batch index 0

tokens = tokenizer.convert_ids_to_tokens(input_id_list)

call_html()

# 用BertViz可视化

head_view(attention, tokens)