机器学习从入门到创业手记-数据分析实战_lda = latentdirichletallocation(n_components=no_to

字段

说明

Train_id

商品ID

Name

商品名称

item_condition_id

销售者提供的型号

category_name

商品所属类别

brand_name

商品所属品牌

Price

商品售价，这个字段也就是我们要预测的字段

Shipping

1 ： 包邮 0 ：不包邮

item_description

商品的完整描述

# 商品价格统计代码：

full_data = pd.read_csv('train.tsv','\t')

def showpricehist():

       full_data.hist('price',bins=50,range=[0,250],label='price')

       plt.title('Price distribution')

       plt.xlabel('Price')

       plt.ylabel('Countss')

       plt.show()

#产品分类

def split_cat(text):

    try: return text.split("/")

    except: return ("No Label", "No Label", "No Label")

#重组产品分类，按主分类/二级分类/三级分类重构数据集

full_data['general_cat'], full_data['subcat_1'], full_data['subcat_2'] = zip(*full_data['category_name'].apply(lambda x: split_cat(x)))

def  CatelogGoodsStats():

       brand_goods = full_data.groupby(['brand_name','subcat_2']).size().to_frame("Counts").sort_values("Counts",ascending=False).reset_index()

       print(brand_goods)

CatelogGoodsStats()

from wordcloud import WordCloud,STOPWORDS

def findVSBrasDesc():

       vs_data = full_data[full_data['item_description']!='No description yet']

       vs_data = vs_data[vs_data['subcat_2'] == 'Bras']

       vs_data = vs_data[vs_data['brand_name'] == "Victoria's Secret"]

       #包邮

       ship_vs_data_desc = vs_data[vs_data.shipping == 1].item_description

       ship_data_text = ','.join(str(i) for i in ship_vs_data_desc.values)

       ship_wordcloud = WordCloud(background_color='black',stopwords = STOPWORDS , max_font_size=110,  min_font_size=10, mode='RGBA', font_path='source/simhei.ttf').generate(ship_data_text)

       #不包邮

       noship_vs_data_desc = vs_data[vs_data.shipping == 0].item_description

       noship_data_text = ','.join(str(i) for i in noship_vs_data_desc.values)

       noship_wordcloud = WordCloud(background_color='black',stopwords =STOPWORDS , max_font_size=110,  min_font_size=10, mode='RGBA', font_path='source/simhei.ttf').generate(noship_data_text)

       fig = plt.figure()

       ship_ax = fig.add_subplot(121)

       ship_ax.imshow(ship_wordcloud)

       ship_ax.set_title("包邮词频云图")

       noship_ax = fig.add_subplot(122)

       noship_ax.imshow(noship_wordcloud)

       noship_ax.set_title("不包邮词频云图")

       ship_ax.axis('off')

       noship_ax.axis('off')

       plt.show()

def findVSBrasShipStat():

       vs_data = full_data[full_data['item_description']!='No description yet']

       vs_data = vs_data[vs_data['subcat_2'] == 'Bras']

       vs_data = vs_data[vs_data['brand_name'] == "Victoria's Secret"]

       ship_data = vs_data.groupby(["shipping","price"]).size().to_frame("Counts").sort_values("Counts",ascending=False).reset_index()

       sns.lineplot(x = 'price',y="Counts",hue = 'shipping',data = ship_data)

       plt.xticks(rotation=90)

       plt.show()

from sklearn.feature_extraction.text import CountVectorizer

import collections

from pprint import pprint

def DescCountVectorizer():

       vs_data = full_data[full_data['item_description']!='No description yet']

       vs_data = vs_data[vs_data['subcat_2'] == 'Bras']

       vs_data = vs_data[vs_data['brand_name'] == "Victoria's Secret"]

       vs_data_desc = vs_data.item_description

       vs_data_text =[]

       for value in vs_data_desc.values:

              vs_data_text.append(value)

       c_vec_s = CountVectorizer(analyzer='word',stop_words='english')

       c_vec_s.fit_transform(np.array(vs_data_text))

       sort_dict = collections.OrderedDict(sorted(c_vec_s.vocabulary_.items(),key=lambda t:t[1]))

       pprint(sort_dict)

from sklearn.feature_extraction.text import TfidfVectorizer

def DescTFidfVectorizer():

       vs_data_desc = full_data.item_description

       vs_data_text =[]

       for value in vs_data_desc.values:

              vs_data_text.append(value)

       t_vec_s = TfidfVectorizer(analyzer='word',stop_words='english')

       weight = t_vec_s.fit_transform(np.array(vs_data_text)).toarray()

       word = t_vec_s.get_feature_names()

       sort_dict = collections.OrderedDict(sorted(t_vec_s.vocabulary_.items(),key=lambda t:t[1]))

       for i in range(len(weight)):

              for j in range(len(word)):

                     pprint (word[i] +" : " + str(weight[i][j]))

图3-6 CountVectorizer统计图

图3-6所示的是左边是单词，右边是单词在描述中出现的次数，这里我截取的最多的一些词汇，比如Zoomed在数据集最长出现的写法是：In the last pic is the 32c bra zoomed in to try to show you that it really is a 32c bra.而剩下的词汇都是一些品牌的名称，而这些品牌名称为什么会大量的出现呢？举个例子

They are all Small 2nd pic: from zumies or pacsun 3rd pic: from Hollister Last pic: white - from zumies or pacsun Pink- from VS Worn a couple times In good condition,Women,Underwear,Bras。

这里就能明白了之前描述了二手交易网站都是一些家庭妇女在用，而她们并不会修图，所以晒的图片都是一些从其它网站找到的样例。

def display_topics(model, feature_names, no_top_words):

       for topic_idx, topic in enumerate(model.components_):

              print (u"Topic %d:" % (topic_idx))

              print (" ".join([feature_names[i] for i in topic.argsort()[:-no_top_words - 1:-1]]))

#NMF & LDA分布

from sklearn.decomposition import NMF, LatentDirichletAllocation

#显示20个主题

no_topics = 10

topic_len = 15

def NMFLDATopic():

       vs_data_desc = full_data.item_description

       vs_data_text =[]

       for value in vs_data_desc.values:

              vs_data_text.append(value)

       #tfidf&nmf

       t_vec_s = TfidfVectorizer(analyzer='word',stop_words='english')

       tfidf = t_vec_s.fit_transform(np.array(vs_data_text)).toarray()

       nmf = NMF(n_components=no_topics, random_state=1, alpha=.1, l1_ratio=.5, init='nndsvd').fit(tfidf)

       print (u"=====NMF TOPIC====")

       display_topics(nmf,t_vec_s.get_feature_names(),topic_len)

       print ("\n")

       #count&lda

       c_vec_s = CountVectorizer(analyzer='word',stop_words='english')

       tf = c_vec_s.fit_transform(np.array(vs_data_text))

       lda = LatentDirichletAllocation(n_topics=no_topics, max_iter=5, learning_method='online', learning_offset=50.,random_state=0).fit(tf)

       print (u"====LDA TOPIC====")

       display_topics(lda,c_vec_s.get_feature_names(),topic_len)

图3-7 NMF主题抽象

图3-8 LDA 主题抽象

from sklearn.ensemble import GradientBoostingRegressor

def vsdesctoprice():

       vs_data_desc = full_data.item_description

       vs_data_text =[]

       for value in vs_data_desc.values:

              vs_data_text.append(value)

       #tfidf&nmf

       t_vec_s = TfidfVectorizer(analyzer='word',stop_words='english')

       tfidf = t_vec_s.fit_transform(np.array(vs_data_text)).toarray()

       gbr = GradientBoostingRegressor()

       gbr.fit(tfidf,full_data.price)

       gbr_y_pred = gbr.predict(item_desc)

       full_data['item_description'] = pd.Series(gbr_y_pred)

       full_data['item_description'].fillna(full_data['item_description'].mean(),inplace=True)

def vsdesctoprice():

                full_data = pd.read_csv('vsdata.csv',encoding = "ISO-8859-1")

                vs_data_desc = full_data.item_description

                vs_data_text =[]

                for value in vs_data_desc.values:

                     vs_data_text.append(value)

                #tfidf&nmf

                t_vec_s = TfidfVectorizer(analyzer='word',stop_words='english')

                tfidf = t_vec_s.fit_transform(np.array(vs_data_text)).toarray()

                gbr = GradientBoostingRegressor()

                gbr.fit(tfidf,full_data.price)

                gbr_y_pred = gbr.predict(tfidf)

                full_data['item_description'] = pd.Series(gbr_y_pred)

                full_data['item_description'].fillna(full_data['item_description'].mean(),inplace=True)

                #高维：区分训练字段与目标字段

                train_col = ['name','item_condition_id','brand_name','shipping','item_description','general_cat','subcat_1','subcat_2']

                target_col = ['price']

                full_data_col = train_col + target_col

                full_data = full_data[full_data_col]

                #高维：区分训练集与测试集

                train_x,test_x,train_y,test_y = train_test_split(full_data[train_col],full_data[target_col].astype(float))

                #高维：使用LabelEncoder编码，将字符串字段转为0~nClass之间唯一编码

                le = preprocessing.LabelEncoder()

                #高维：将所有字符字段进行转化

                train_x['name'] = le.fit_transform(train_x['name']).astype(float)

                train_x['brand_name'] = le.fit_transform(train_x['brand_name']).astype(float)

                train_x['general_cat'] = le.fit_transform(train_x['general_cat']).astype(float)

                train_x['subcat_1'] = le.fit_transform(train_x['subcat_1']).astype(float)

                train_x['subcat_2'] = le.fit_transform(train_x['subcat_2']).astype(float)

                train_x['item_condition_id'] = le.fit_transform(train_x['item_condition_id']).astype(float)

                #高维：使用XGB提升回归数算法进行预测

                exported_pipeline = XGBRegressor(learning_rate=0.1, max_depth=2, min_child_weight=6, n_estimators=100, nthread=1, subsample=0.9000000000000001)

                exported_pipeline.fit(train_x, train_y)

                #高维：打印影响价格权重

                print("---------------------------------------")

                pprint(eli5.explain_weights_xgboost(exported_pipeline,top=10))

                print(eli5.show_weights(exported_pipeline,vec=train_col,top=10))

                results = exported_pipeline.predict(test_x)

                print (results)

                #高维：将预测结果输出并保存

                pd.to_csv(results)

from tpot import TPOTRegressor

tpot = TPOTRegressor(generations=5, population_size=50, verbosity=2)

tpot.fit(train_x, train_y)

tpot.export('tpotopt.py')