OpenNLP：驾驭文本，分词那些事

$ opennlp ToolName lang-model-name.bin < input.txt > output.txt

//英文切词：空格或者换行符
	public static void ENSplit(String str)
	{
		String[] result=str.split("\\s+");
		for(String s:result){
			System.out.println(s+" ");
		}
		System.out.println();
	}

/**
	 * 1.句子探测器：Sentence Detector 
	 * @deprecated Sentence detector is for detecting sentence boundaries. Given the following paragraph:
	 * Hi. How are you? This is Mike.
	 * 如：Hi. How are you? This is Mike.将返回如下：
	 * Hi. How are you? 
	 * This is Mike.
	 * @throws IOException 
	 * @throws InvalidFormatException 
	 */
	public static void SentenceDetector(String str) throws InvalidFormatException, IOException
	{
		//always start with a model, a model is learned from training data
		InputStream is = new FileInputStream("./nlpbin/en-sent.bin");
		SentenceModel model = new SentenceModel(is);
		SentenceDetectorME sdetector = new SentenceDetectorME(model);
	 
		String sentences[] = sdetector.sentDetect(str);
	 
		System.out.println(sentences[0]);
		System.out.println(sentences[1]);
		is.close();
		System.out.println("---------------1------------");
	}

/**
	 * 2.标记生成器:Tokenizer
	 * @deprecated Tokens are usually words which are separated by space, but there are exceptions.  For example, "isn't" gets split into "is" and "n't, since it is a a brief format of "is not". Our sentence is separated into the following tokens:
	 * @param str
	 */
	public static void Tokenize(String str) throws InvalidFormatException, IOException {
		InputStream is = new FileInputStream("./nlpbin/en-token.bin");
		TokenizerModel model = new TokenizerModel(is);
		Tokenizer tokenizer = new TokenizerME(model);
		String tokens[] = tokenizer.tokenize(str);
		for (String a : tokens)
			System.out.println(a);
		is.close();
		System.out.println("--------------2-------------");
	}

/**
	 * 3.名称搜索:Name Finder
	 * @deprecated By its name, name finder just finds names in the context. Check out the following example to see what name finder can do. It accepts an array of strings, and find the names inside.
	 * @param str
	 */
	public static void findName() throws IOException {
		InputStream is = new FileInputStream("./nlpbin/en-ner-person.bin");
		TokenNameFinderModel model = new TokenNameFinderModel(is);
		is.close();
		NameFinderME nameFinder = new NameFinderME(model);
		String[] sentence = new String[]{
			    "Mike",
			    "Tom",
			    "Smith",
			    "is",
			    "a",
			    "good",
			    "person"
			    };
		Span nameSpans[] = nameFinder.find(sentence);
		for(Span s: nameSpans)
			System.out.println(s.toString());	
		System.out.println("--------------3-------------");
	}

/**
	 * 4.POS标注器:POS Tagger
	 * @deprecated Hi._NNP How_WRB are_VBP you?_JJ This_DT is_VBZ Mike._NNP
	 * @param str
	 */
	public static void POSTag(String str) throws IOException {
		POSModel model = new POSModelLoader().load(new File("./nlpbin/en-pos-maxent.bin"));
		PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");//显示加载时间
		POSTaggerME tagger = new POSTaggerME(model);
		ObjectStream<String> lineStream = new PlainTextByLineStream(new StringReader(str));
		perfMon.start();
		String line;
		while ((line = lineStream.read()) != null) {
			String whitespaceTokenizerLine[] = WhitespaceTokenizer.INSTANCE.tokenize(line);
			String[] tags = tagger.tag(whitespaceTokenizerLine);
			POSSample sample = new POSSample(whitespaceTokenizerLine, tags);
			System.out.println(sample.toString());
			perfMon.incrementCounter();
		}
		perfMon.stopAndPrintFinalResult();
		System.out.println("--------------4-------------");
	}

/**
	 * OpenNLP词性标注工具的例子：最大熵词性标注器pos-maxent
	 * JJ形容词、JJS形容词最高级、JJR形容词比较级
	 * RB副词、RBR副词最高级、RBS副词比较级
	 * DT限定词
	 * NN名称、NNS名称复试、NNP专有名词、NNPS专有名词复数：
	 * PRP：人称代词、PRP$：物主代词
	 * VB动词不定式、VBD过去式、VBN过去分词、VBZ现在人称第三人称单数、VBP现在非第三人称、VBG动名词或现在分词
	 */
	public static void POSMaxent(String str) throws InvalidFormatException, IOException
	{
		//给出词性模型所在的路径
		File posModeFile=new File("./nlpbin/en-pos-maxent.bin");
		FileInputStream posModeStream=new FileInputStream(posModeFile);
		POSModel model=new POSModel(posModeStream);
		//将句子切分成词
		POSTaggerME tagger=new POSTaggerME(model);
		String[] words=SimpleTokenizer.INSTANCE.tokenize(str);
		//将切好的词的句子传递给标注器
		String[] result=tagger.tag(words);
		for (int i=0; i<words.length;i++){
			System.out.print(words[i]+"/"+result[i]+" ");
		}
		System.out.println();
	}

/**
	 * 5.序列标注:Chunker
	 * @deprecated 通过使用标记生成器生成的tokens分为一个句子划分为一组块。What chunker does is to partition a sentence to a set of chunks by using the tokens generated by tokenizer.
	 * @param str
	 */
	public static void chunk(String str) throws IOException {
		POSModel model = new POSModelLoader().load(new File("./nlpbin/en-pos-maxent.bin"));
		//PerformanceMonitor perfMon = new PerformanceMonitor(System.err, "sent");
		POSTaggerME tagger = new POSTaggerME(model);
		ObjectStream<String> lineStream = new PlainTextByLineStream(new StringReader(str));
		//perfMon.start();
		String line;
		String whitespaceTokenizerLine[] = null;
		String[] tags = null;
		while ((line = lineStream.read()) != null) {
			whitespaceTokenizerLine = WhitespaceTokenizer.INSTANCE.tokenize(line);
			tags = tagger.tag(whitespaceTokenizerLine);
			POSSample sample = new POSSample(whitespaceTokenizerLine, tags);
			System.out.println(sample.toString());
				//perfMon.incrementCounter();
		}
		//perfMon.stopAndPrintFinalResult();
		
		// chunker
		InputStream is = new FileInputStream("./nlpbin/en-chunker.bin");
		ChunkerModel cModel = new ChunkerModel(is);
		ChunkerME chunkerME = new ChunkerME(cModel);
		String result[] = chunkerME.chunk(whitespaceTokenizerLine, tags);
		for (String s : result)
			System.out.println(s);
		Span[] span = chunkerME.chunkAsSpans(whitespaceTokenizerLine, tags);
		for (Span s : span)
			System.out.println(s.toString());
		System.out.println("--------------5-------------");
	}

/**
	 * 6.分析器: Parser
	 * @deprecated Given this sentence: "Programcreek is a very huge and useful website.", parser can return the following:
	 * (TOP (S (NP (NN Programcreek) ) (VP (VBZ is) (NP (DT a) (ADJP (RB very) (JJ huge) (CC and) (JJ useful) ) ) ) (. website.) ) )
	 * (TOP
	 *     (S
	 *        (NP
	 *           (NN Programcreek)
	 *        )
	 *        (VP
	 *           (VBZ is)
	 *           (NP
	 *              (DT a)
	 *              (ADJP
	 *                   (RB very)
	 *                   (JJ huge)
	 *                   (CC and)
	 *                   (JJ userful)
	 *              )
	 *           )
	 *        )
	 *        (. website.)
	 *     )
	 * )
	 * @param str
	 */
	public static void Parse() throws InvalidFormatException, IOException {
		// http://sourceforge.net/apps/mediawiki/opennlp/index.php?title=Parser#Training_Tool
		InputStream is = new FileInputStream("./nlpbin/en-parser-chunking.bin");
		ParserModel model = new ParserModel(is);
		Parser parser = ParserFactory.create(model);
		String sentence = "Programcreek is a very huge and useful website.";
		opennlp.tools.parser.Parse topParses[] = ParserTool.parseLine(sentence, parser, 1);
		for (opennlp.tools.parser.Parse p : topParses)
			p.show();
		is.close();
	}