新闻文本分类之旅 Word2Vec_Corpus_python nlp 新闻分类word2vec

新闻文本分类

预训练Word2vec语料

导入相关库

import numpy as np
import pandas as pd
from gensim.models import word2vec
1
2
3

读取数据

train_df = pd.read_csv('../data/train_set.csv', sep='\t')
test_df = pd.read_csv('../data/test_a.csv', sep='\t')
1
2

加载语料

list_data = list(pd.concat((train_df['text'], test_df['text']), axis=0, ignore_index=True).map(lambda x:x.split(' ')))
1

训练语料

model = word2vec.Word2Vec(list_data, min_count=1, window=10, size=120, seed=7)
model.init_sims(replace=True)
1
2

保存模型

# 保存模型，供日後使用
model.save("../emb/word2vec.h5")
model = word2vec.Word2Vec.load("../emb/word2vec.h5")
#输入与“3370”相近的10个词
for key in model.wv.similar_by_word('3370', topn =10):
    print(key)
1
2
3
4
5
6