基于Spark上的中文分词算法的实现_spark 分词

import org.lionsoul.jcseg.ASegment;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import org.lionsoul.jcseg.core.ADictionary;
import org.lionsoul.jcseg.core.DictionaryFactory;
import org.lionsoul.jcseg.core.IWord;
import org.lionsoul.jcseg.core.JcsegException;
import org.lionsoul.jcseg.core.JcsegTaskConfig;
import org.lionsoul.jcseg.core.SegmentFactory;

/**
 * Created by Administrator on 2016/4/6.
 */
public class AnaylyzerTools {

public static ArrayList<String> anaylyzerWords(String str){
        JcsegTaskConfig config =new JcsegTaskConfig(AnaylyzerTools.class.getResource("").getPath()+"jcseg.properties");
        ADictionary dic = DictionaryFactory.createDefaultDictionary(config);
        ArrayList<String> list=new ArrayList<String>();
        ASegment seg=null;
        try {
            seg = (ASegment) SegmentFactory.createJcseg(JcsegTaskConfig.COMPLEX_MODE, new Object[]{config, dic});
        } catch (JcsegException e1) {
            e1.printStackTrace();
        }
        try {
            seg.reset(new StringReader(str));
            IWord word = null;
            while ( (word = seg.next()) != null ) {
                list.add(word.getValue());
            }
        } catch (IOException e) {
            e.printStackTrace();
    }
    return list;
}

    public static void main(String[] args) {
        String str="HBase中通过row和columns确定的为一个存贮单元称为cell。显示每个元素，每个 cell都保存着同一份数据的多个版本。版本通过时间戳来索引。迎泽区是繁华的地方,营业厅营业";
        List<String> list=AnaylyzerTools.anaylyzerWords(str);
        for(String word:list){
            System.out.println(word);
        }
        System.out.println(list.size());
    }
}

import org.apache.spark._
import utils.AnaylyzerTools

/**
 * Created by Administrator on 2016/4/6.
 */
object AnalyzerDemo {

//分词排序后取出词频最高的前10个
def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("my app").setMaster("local")
    val sc = new SparkContext(conf)
    val data=sc.textFile("D:\\aaab.txt").map(x => {
    val list=AnaylyzerTools.anaylyzerWords(x) //分词处理
        list.toString.replace("[", "").replace("]", "").split(",")
    }).flatMap(x => x.toList).map(x => (x.trim(),1)).reduceByKey(_+_).top(10)(Ord.reverse).foreach(println)
  }

//分词排序
    object Ord extends Ordering[(String,Int)]{
        def compare(a:(String,Int), b:(String,Int))=a._2 compare (b._2)
     }
}