java中文分词_java 汉字分词

最近由于工作原因,接触到了JAVA的中文分词, 现在来讲讲具体操作.

java的分词器都是基于 lucene 核心的.

我选择的是

ikanalyzer2.0.2的最新版本

具体调用过程如下:

import java.io.Reader; import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.cjk.CJKAnalyzer; import org.apache.lucene.analysis.cn.ChineseAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.mira.lucene.analysis.MIK_CAnalyzer; public class JeAnalyzer { //private static String testString1 = "/u4E2D/u56FD/u4EBA/u52A0/u6CB9"; private static String testString1 = "中国人加油"; public static void testStandard(String testString) { try { Analyzer analyzer = new StandardAnalyzer(); Reader r = new StringReader(testString); StopFilter sf = (StopFilter) analyzer.tokenStream("", r); System.err.println("=====standard analyzer===="); Token t; while ((t = sf.next()) != null) { System.out.println(t.termText()); } } catch(Exception e) { e.printStackTrace(); } } public static void testCJK(String testString) { try { Analyzer analyzer = new CJKAnalyzer(); Reader r = new StringReader(testString); StopFilter sf = (StopFilter) analyzer.tokenStream("", r); System.err.println("=====cjk analyzer===="); Token t; while ((t = sf.next()) != null) { System.out.println(t.termText()); } } catch(Exception e) { e.printStackTrace(); } } public static void testChiniese(String testString) { try { Analyzer analyzer = new ChineseAnalyzer(); Reader r = new StringReader(testString); TokenFilter tf = (TokenFilter) analyzer.tokenStream("", r); System.err.println("=====chinese analyzer===="); Token t; while ((t = tf.next()) != null) { System.out.println(t.termText()); } } catch(Exception e) { e.printStackTrace(); } } public static String transJe(String testString,String c1,String c2) { String result = ""; try { Analyzer analyzer = new MIK_CAnalyzer(); Reader r = new StringReader(testString); TokenStream ts = (TokenStream)analyzer.tokenStream("", r); //System.out.println("=====je analyzer===="); Token t; while ((t = ts.next()) != null) { result += t.termText()+","; } } catch(Exception e) { e.printStackTrace(); } return result; } public static void main(String[] args) { try { //String testString = testString1; String testString = testString1; System.out.println(testString); //testStandard(testString); //testCJK(testString); //testPaoding(testString); //testChiniese(testString); //testString = new TestT().convert(testString); //testString = "/u636E/u8DEF/u900F/u793E/u62A5/u9053/uff0C/u5370/u5EA6/u5C3C/u897F/u4E9A/u793E/u4f1A/u4E8B/u52A1/u90E8/u4E00/u5B98/u5458/u661F/u671F/u4E8C"; String sResult[] = transJe(testString,"gb2312","utf-8").split(","); for(int i = 0 ; i< sResult.length ; i++) { System.out.println(sResult[i]); } } catch(Exception e) { e.printStackTrace(); } } }

需要 的JAR如下:

lucene-analyzers-2.4.1.jar

lucene-core-2.4.1.jar

当然少不了

IKAnalyzer2.0.2OBF.jar

在UNIX下使用的话,注意对中文的编码问题.