nltk中文分句_利用NLTK进行分句分词

1.输入一个段落，分成句子(Punkt句子分割器)

import nltk

import nltk.data

def splitSentence(paragraph):

tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')

sentences = tokenizer.tokenize(paragraph)

return sentences

if __name__ == '__main__':

print splitSentence("My name is Tom. I am a boy. I like soccer!")

结果为['My name is Tom.', 'I am a boy.', 'I like soccer!']

2.输入一个句子，分成词组

from nltk.tokenize import WordPunctTokenizer

def wordtokenizer(sentence):

#分段

words = WordPunctTokenizer().tokenize(sentence)

return words

if __name__ == '__main__':

print wordtokenizer("My name is Tom.")结果为['My', 'name', 'is', 'Tom', '.']