[NLP]OpenNLP语言检测器的使用_opennlp 中文模型

目录

Language Detector

模型训练

语言类型预测

Language Detector

语言检测器，属于分类范畴。即OpenNLP语言检测器根据模型的能力用ISO-639-3（国际语种代号标准）语言对文档进行分类。模型可以用最大熵模型（Maxent）、感知器（Perceptron）或朴素贝叶斯算法（Naive Bayes algorithms）训练。通过用n-gram算法在规范化文本的上下文中提取词汇，默认情况下，大小为1、2或3个字。通过扩展LanguageDetectorFactory可以定制n-gram的大小、规范化和上下文生成器。

默认的标准化器：

模型训练

import java.io.File;
 
import java.io.IOException;
 
import java.nio.charset.StandardCharsets;
 
import opennlp.tools.langdetect.LanguageDetectorModel;
 
import opennlp.tools.langdetect.LanguageDetectorSampleStream;
 
import opennlp.tools.langdetect.LanguageSample;
 
import opennlp.tools.ml.perceptron.PerceptronTrainer;
 
import opennlp.tools.util.InputStreamFactory;
 
import opennlp.tools.util.MarkableFileInputStreamFactory;
 
import opennlp.tools.util.ObjectStream;
 
import opennlp.tools.util.PlainTextByLineStream;
 
import opennlp.tools.util.TrainingParameters;
 
import opennlp.tools.util.model.ModelUtil;
 
import opennlp.tools.langdetect.LanguageDetectorFactory;
 
import opennlp.tools.langdetect.LanguageDetectorME;
 
 
 
public class LangdetectTrain {
 
 
 
    public static void main(String[] args) throws IOException {
 
       // TODO Auto-generated method stub
 
       String rootDir = System.getProperty("user.dir") + File.separator;
 
      
 
       String fileResourcesDir = rootDir + "resources" + File.separator;
 
       String modelResourcesDir = rootDir + "opennlpmodel" + File.separator;
 
      
 
       String filePath = fileResourcesDir + "languageDetector.txt";
 
       System.out.println(filePath);
 
     
 
        //创建一个可标注的输入数据流
 
       InputStreamFactory inputStreamFactory = new MarkableFileInputStreamFactory(new File(filePath));
 
        //按行输入文档，一行代表一个文档
 
       ObjectStream<String> lineStream = new PlainTextByLineStream(inputStreamFactory, StandardCharsets.UTF_8);
 
        
 
       System.out.println(lineStream.read());
 
       ObjectStream<LanguageSample> sampleStream = new LanguageDetectorSampleStream(lineStream);
 
        
 
      
 
       //确定训练参数
 
       TrainingParameters params = ModelUtil.createDefaultTrainingParameters();
 
       //采用感知器算法训练
 
        params.put(TrainingParameters.ALGORITHM_PARAM,PerceptronTrainer.PERCEPTRON_VALUE);
 
       params.put(TrainingParameters.CUTOFF_PARAM, 0);
 
      
 
       LanguageDetectorFactory factory = new LanguageDetectorFactory();
 
        //训练模型
 
       LanguageDetectorModel model = LanguageDetectorME.train(sampleStream, params, factory);
 
       //保存模型
 
       model.serialize(new File(modelResourcesDir+"langdetect.bin"));
 
 
 
    }
 
}

语言类型预测

import java.io.File;
 
import java.io.IOException;
 
import opennlp.tools.langdetect.LanguageDetectorModel;
 
import opennlp.tools.langdetect.Language;
 
import opennlp.tools.langdetect.LanguageDetectorME;
 
 
 
public class LangdetectPredit {
 
 
 
    public static void main(String[] args) throws IOException {
 
       // TODO Auto-generated method stub
 
       String rootDir=System.getProperty("user.dir")+File.separator;
 
       String fileResourcesDir=rootDir+"resources"+File.separator;
 
       String modelResourcesDir=rootDir+"opennlpmodel"+File.separator;
 
       String filePath=fileResourcesDir+"languageDetector.txt";
 
       String inputTest_en="A la fecha tres calles bonaerenses recuerdan";
 
       //模型文件保存目录
 
       String LDMFile =modelResourcesDir+ "langdetect.bin";
 
       File modelFile = new File("");
 
       //加载模型
 
       LanguageDetectorModel LDM = new LanguageDetectorModel(modelFile);
 
       LanguageDetectorME myCategorizer = new LanguageDetectorME(LDM);
 
       //预测语言类型
 
       Language bestLanguage = myCategorizer.predictLanguage(inputTest_en);
 
       System.out.println("语言类型为: " + bestLanguage.getLang());
 
       System.out.println("置信度: " + bestLanguage.getConfidence());
 
       //预测语言类型列表
 
       Language[] languages = myCategorizer.predictLanguages(inputTest_en);
 
    }
 
}