spaCy：处理流程_nlp = spacy.load('zh_core_web_sm')

1

doc = nlp("This is a sentence.")

1

print(nlp.pipe_names)

1

['tok2vec', 'tagger', 'parser', 'ner', 'attribute_ruler', 'lemmatizer']

1

print(nlp.pipeline)

1

2

3

4

5

6

[('tok2vec', <spacy.pipeline.Tok2Vec>),

('tagger', <spacy.pipeline.Tagger>),

('parser', <spacy.pipeline.DependencyParser>),

('ner', <spacy.pipeline.EntityRecognizer>),

('attribute_ruler', <spacy.pipeline.AttributeRuler>),

('lemmatizer', <spacy.pipeline.Lemmatizer>)]

1

2

3

4

5

6

7

8

from spacy.language import Language

@Language.component("custom_component")

def custom_component_function(doc):

    # 对doc做一些处理

    return doc

nlp.add_pipe("custom_component")

1

2

3

4

5

6

@Language.component("custom_component")

def custom_component_function(doc):

    # 对doc做一些处理

    return doc

nlp.add_pipe("custom_component")

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

import spacy

from spacy.language import Language

# 定义定制化组件

@Language.component("length_component")

def length_component_function(doc):

    # 获取doc的长度

    doc_length = len(doc)

    print(f"This document is {doc_length} tokens long.")

    # 返回这个doc

    return doc

# 读取小规模的中文流程

nlp = spacy.load("zh_core_web_sm")

# 将组件加入到流程的最前面，打印流程组件名

nlp.add_pipe("length_component", first=True)

print(nlp.pipe_names)

# 处理一段文本

doc = nlp("这是一个句子。")

1

2

['length_component', 'tok2vec', 'tagger', 'parser', 'attribute_ruler', 'ner']

This document is 4 tokens long.

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

import spacy

from spacy.language import Language

from spacy.matcher import PhraseMatcher

from spacy.tokens import Span

nlp = spacy.load("zh_core_web_sm")

animals = ["金毛犬", "猫", "乌龟", "老鼠"]

animal_patterns = list(nlp.pipe(animals))

print("animal_patterns:", animal_patterns)

matcher = PhraseMatcher(nlp.vocab)

matcher.add("ANIMAL", animal_patterns)

# 定义定制化组件

@Language.component("animal_component")

def animal_component_function(doc):

    # 把matcher应用到doc上

    matches = matcher(doc)

    # 为每一个匹配结果生成一个Span并赋予标签"ANIMAL"

    spans = [Span(doc, start, end, label="ANIMAL") for match_id, start, end in matches]

    # 用匹配到的span覆盖doc.ents

    doc.ents = spans

    return doc

# 把组件加入到流程中，紧跟在"ner"组件后面

nlp.add_pipe("animal_component", after="ner")

print(nlp.pipe_names)

# 处理文本，打印doc.ents的文本和标签

doc = nlp("我养了一只猫和一条金毛犬。")

print([(ent.text, ent.label_) for ent in doc.ents])

1

2

3

animal_patterns: [金毛犬, 猫, 乌龟, 老鼠]

['tok2vec', 'tagger', 'parser', 'attribute_ruler', 'ner', 'animal_component']

[('猫', 'ANIMAL'), ('金毛犬', 'ANIMAL')]

1

2

3

doc._.title = "My document"

token._.is_color = True

span._.has_color = False

1

2

3

4

5

6

7

# 导入全局类

from spacy.tokens import Doc, Token, Span

# 在Doc、Token和Span上设置扩展属性

Doc.set_extension("title", default=None)

Token.set_extension("is_color", default=False)

Span.set_extension("has_color", default=False)

1

2

3

4

5

6

7

8

9

from spacy.tokens import Token

# 为Token设置一个有默认值的扩展

Token.set_extension("is_color", default=False)

doc = nlp("天空是蓝色的。")

# 覆盖默认扩展特性的值

doc[2]._.is_color = True

1

2

3

4

5

6

7

8

9

10

11

12

from spacy.tokens import Token

# 定义取值器函数

def get_is_color(token):

    colors = ["红色", "黄色", "蓝色"]

    return token.text in colors

# 为词符设置有取值器的扩展

Token.set_extension("is_color", getter=get_is_color)

doc = nlp("天空是蓝色的。")

print(doc[2]._.is_color, "-", doc[2].text)

1

True - 蓝色

1

2

3

4

5

6

7

8

9

10

11

12

13

from spacy.tokens import Span

# 定义取值器函数

def get_has_color(span):

    colors = ["红色", "黄色", "蓝色"]

    return any(token.text in colors for token in span)

# 为Span设置一个带有取值器getter的扩展

Span.set_extension("has_color", getter=get_has_color)

doc = nlp("天空是蓝色的")

print(doc[1:4]._.has_color, "-", doc[1:4].text)

print(doc[0:2]._.has_color, "-", doc[0:2].text)

1

2

True - 是蓝色的

False - 天空是

1

2

3

4

5

6

7

8

9

10

11

12

13

from spacy.tokens import Doc

# 定义含有参数的方法

def has_token(doc, token_text):

    in_doc = token_text in [token.text for token in doc]

    return in_doc

# 在doc上设置方法扩展

Doc.set_extension("has_token", method=has_token)

doc = nlp("天空是蓝色的。")

print(doc._.has_token("蓝色"), "- 蓝色")

print(doc._.has_token("云朵"), "- 云朵")

1

2

True - 蓝色

False - 云朵

1

2

3

4

5

6

7

8

9

10

11

12

13

14

import spacy

from spacy.tokens import Token

nlp = spacy.load("zh_core_web_sm")

# 注册词符的扩展属性"is_country"，其默认值是False

Token.set_extension("is_country", default=False)

# 处理文本，将词符"新加坡"的is_country属性设置为True

doc = nlp("我住在新加坡。")

doc[2]._.is_country = True

# 对所有词符打印词符文本及is_country属性

print([(token.text, token._.is_country) for token in doc])

1

[('我', False), ('住在', False), ('新加坡', True), ('。', False)]

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

import spacy

from spacy.tokens import Token

​

nlp = spacy.blank("zh")

​

# 定义取值器函数，读入一个词符并返回其逆序的文本

def get_reversed(token):

    return token.text[::-1]

​

​

# 注册词符的扩展属性get_reversed及其取值器get_reversed

Token.set_extension("reversed", getter=get_reversed)

​

# 处理文本，打印没一个词符的逆序属性

doc = nlp("我说的所有话都是假的，包括这一句。")

for token in doc:

    print("reversed:", token._.reversed)

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

reversed: 我

reversed: 说

reversed: 的

reversed: 所

reversed: 有

reversed: 话

reversed: 都

reversed: 是

reversed: 假

reversed: 的

reversed: ，

reversed: 包

reversed: 括

reversed: 这

reversed: 一

reversed: 句

reversed: 。

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

import spacy

from spacy.tokens import Doc

nlp = spacy.blank("zh")

# 定义取值器函数

def get_has_number(doc):

    # 返回是否doc中的任一个词符的token.like_num返回True

    return any(token.like_num for token in doc)

# 注册Doc的扩展属性"has_number"及其取值器get_has_number

Doc.set_extension("has_number", getter=get_has_number)

# 处理文本，检查定制化的has_number属性

doc = nlp("这家博物馆在2012年关了五个月。")

print("has_number:", doc._.has_number)

1

has_number: True

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

import spacy

from spacy.tokens import Span

nlp = spacy.blank("zh")

# 定义这个方法

def to_html(span, tag):

    # 将span文本包在HTML标签中并返回

    return f"<{tag}>{span.text}</{tag}>"

# 注册这个Span方法扩展名"to_html"及其方法to_html

Span.set_extension("to_html", method=to_html)

# 处理文本，在span上调用to_html方法及其标签名"strong"

doc = nlp("大家好，这是一个句子。")

span = doc[0:3]

print(span._.to_html("strong"))

1

<strong>大家好</strong>

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

import spacy

from spacy.tokens import Span

nlp = spacy.load("zh_core_web_sm")

def get_wikipedia_url(span):

    # 如果span有其中一个标签则获取其维基百科URL

    if span.label_ in ("PERSON", "ORG", "GPE", "LOCATION"):

        entity_text = span.text.replace(" ", "_")

        return "https://zh.wikipedia.org/w/index.php?search=" + entity_text

# 设置Span的扩展wikipedia_url及其取值器get_wikipedia_url

Span.set_extension("wikipedia_url", getter=get_wikipedia_url)

doc = nlp(

    "出道这么多年，周杰伦已经成为几代年轻人共同的偶像。"

)

for ent in doc.ents:

    # 打印实体的文本和其维基百科URL

    print(ent.text, ent._.wikipedia_url)

1

周杰伦 https://zh.wikipedia.org/w/index.php?search=周杰伦

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

import json

import spacy

from spacy.language import Language

from spacy.tokens import Span

from spacy.matcher import PhraseMatcher

with open("exercises/zh/countries.json", encoding="utf8") as f:

    COUNTRIES = json.loads(f.read())

with open("exercises/zh/capitals.json", encoding="utf8") as f:

    CAPITALS = json.loads(f.read())

nlp = spacy.blank("zh")

matcher = PhraseMatcher(nlp.vocab)

matcher.add("COUNTRY", list(nlp.pipe(COUNTRIES)))

@Language.component("countries_component")

def countries_component_function(doc):

    # 对所有匹配结果创建一个标签为"GPE"的实体Span

    matches = matcher(doc)

    doc.ents = [Span(doc, start, end, label="GPE") for match_id, start, end in matches]

    return doc

# 把这个组件加入到流程中

nlp.add_pipe("countries_component")

print(nlp.pipe_names)

# 取值器，在国家首都的字典中寻找span的文本

get_capital = lambda span: CAPITALS.get(span.text)

# 用这个取值器注册Span的扩展属性"capital"

Span.set_extension("capital", getter=get_capital, force=True)

# 处理文本，打印实体文本、标签和首都属性

doc = nlp("新加坡可能会和马来西亚一起建造高铁。")

print([(ent.text, ent.label_, ent._.capital) for ent in doc.ents])

1

2

['countries_component']

[('新加坡', 'GPE', '新加坡'), ('马来西亚', 'GPE', '吉隆坡')]

1

docs = [nlp(text) for text in LOTS_OF_TEXTS]

1

docs = list(nlp.pipe(LOTS_OF_TEXTS))

1

2

3

4

5

6

7

data = [

    ("这是一段文本", {"id": 1, "page_number": 15}),

    ("以及另一段文本", {"id": 2, "page_number": 16}),

]

for doc, context in nlp.pipe(data, as_tuples=True):

    print(doc.text, context["page_number"])

1

2

这是一段文本 15

以及另一段文本 16

1

2

3

4

5

6

7

8

9

10

11

12

13

from spacy.tokens import Doc

Doc.set_extension("id", default=None)

Doc.set_extension("page_number", default=None)

data = [

    ("这是一段文本", {"id": 1, "page_number": 15}),

    ("以及另一段文本", {"id": 2, "page_number": 16}),

]

for doc, context in nlp.pipe(data, as_tuples=True):

    doc._.id = context["id"]

    doc._.page_number = context["page_number"]

1

doc = nlp("Hello world")

1

doc = nlp.make_doc("Hello world!")

1

2

3

4

5

# 关闭词性标注器tagger和依存关系标注器parser

with nlp.select_pipes(disable=["tagger", "parser"]):

    # 处理文本并打印实体结果

    doc = nlp(text)

    print(doc.ents)

1

2

3

4

5

6

7

8

9

10

11

12

import json

import spacy

​

nlp = spacy.load("zh_core_web_sm")

​

with open("exercises/zh/weibo.json", encoding="utf8") as f:

    TEXTS = json.loads(f.read())

​

# 处理文本，打印形容词

for text in TEXTS:

    doc = nlp(text)

    print([token.text for token in doc if token.pos_ == "ADJ"])

1

2

3

4

5

6

[]

[]

[]

['老']

[]

[]

1

2

3

4

5

6

7

8

9

10

11

12

import json

import spacy

nlp = spacy.load("zh_core_web_sm")

with open("exercises/zh/weibo.json", encoding="utf8") as f:

    TEXTS = json.loads(f.read())

# 处理文本，打印实体

docs = list(nlp.pipe(TEXTS))

entities = [doc.ents for doc in docs]

print(*entities)

1

(麦当劳,) (麦当劳, 汉堡, 汉堡) (麦当劳,) (中国, 麦当劳, 北京) (麦当劳,) (今天, 早上, 麦当劳, 一整天)

1

2

3

4

5

6

7

8

import spacy

nlp = spacy.blank("zh")

people = ["周杰伦", "庞麦郎", "诸葛亮"]

# 为PhraseMatcher创建一个模板列表

patterns = [nlp(person) for person in people]

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

import json

import spacy

from spacy.tokens import Doc

with open("exercises/en/bookquotes.json", encoding="utf8") as f:

    DATA = json.loads(f.read())

nlp = spacy.blank("en")

# 注册Doc的扩展"author"（默认值为None）

Doc.set_extension("author", default=None)

# 注册Doc的扩展"book"（默认值为None）

Doc.set_extension("book", default=None)

for doc, context in nlp.pipe(DATA, as_tuples=True):

    # 从context中设置属性doc._.book和doc._.author

    doc._.book = context["book"]

    doc._.author = context["author"]

    # 打印文本和定制化的属性数据

    print(f"{doc.text}\n — '{doc._.book}' by {doc._.author}\n")

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

One morning, when Gregor Samsa woke from troubled dreams, he found himself transformed in his bed into a horrible vermin.

— 'Metamorphosis' by Franz Kafka

I know not all that may be coming, but be it what it will, I'll go to it laughing.

— 'Moby-Dick or, The Whale' by Herman Melville

It was the best of times, it was the worst of times.

— 'A Tale of Two Cities' by Charles Dickens

The only people for me are the mad ones, the ones who are mad to live, mad to talk, mad to be saved, desirous of everything at the same time, the ones who never yawn or say a commonplace thing, but burn, burn, burn like fabulous yellow roman candles exploding like spiders across the stars.

— 'On the Road' by Jack Kerouac

It was a bright cold day in April, and the clocks were striking thirteen.

— '1984' by George Orwell

Nowadays people know the price of everything and the value of nothing.

— 'The Picture Of Dorian Gray' by Oscar Wilde

1

2

3

4

5

6

7

8

9

10

11

import spacy

nlp = spacy.load("zh_core_web_sm")

text = (

    "在300多年的风雨历程中，历代同仁堂人始终恪守“炮制虽繁必不敢省人工，品味虽贵必不敢减物力”的古训，"

    "树立“修合无人见，存心有天知”的自律意识，造就了制药过程中兢兢小心、精益求精的严细精神。"

)

# 仅对文本做分词

doc = nlp.make_doc(text)

print([token.text for token in doc])

1

['在', '300多', '年', '的', '风雨', '历程', '中', '，', '历代', '同仁', '堂人', '始终', '恪守', '“', '炮制', '虽', '繁必', '不', '敢', '省', '人工', '，', '品味', '虽', '贵必', '不', '敢', '减物力', '”', '的', '古训', '，', '树立', '“', '修合', '无', '人', '见', '，', '存心', '有', '天知', '”', '的', '自律', '意识', '，', '造就', '了', '制药', '过程', '中', '兢兢小心', '、', '精益求精', '的', '严细', '精神', '。']

1

2

3

4

5

6

7

8

9

10

11

12

13

14

import spacy

nlp = spacy.load("zh_core_web_sm")

text = (

    "在300多年的风雨历程中，历代同仁堂人始终恪守“炮制虽繁必不敢省人工，品味虽贵必不敢减物力”的古训，"

    "树立“修合无人见，存心有天知”的自律意识，造就了制药过程中兢兢小心、精益求精的严细精神。"

)

# 关闭tagger和parser

with nlp.select_pipes(disable=["tagger", "parser"]):

    # 处理文本

    doc = nlp(text)

    # 打印doc中的实体

    print(doc.ents)

1

(300多年,)