python数据挖掘分析案例_基于Python的Titanic【案例分析】

#导入基础模块

#导入数据

#缺失值处理
 
#数据备份并装入清洗列表中
data_cleaner = []
data_test = test.copy()
data_train = train.copy()
data_cleaner.append(data_train)
data_cleaner.append(data_test)
 
#查看数据集中的数据分布状况，提供填充值选择思路
for data in data_cleaner:
    data[['Age']].hist()
    print(data[['Age']].mean())

#填充缺失值

#同样查看test中的Fare
test[['Fare']].hist()
print(test[['Fare']].mean())

#填充中位数
data_test[['Fare']] = data_test[['Fare']].fillna(data_test['Fare'].median())
data_test[['Fare']].info()

#查看Embarked 字段
print(data_train[['Embarked']].head(20))
#Embarked字段主要有S、C、Q三种取值，我们用众数填充
data_train[['Embarked']] = data_train[['Embarked']].fillna(data_train['Embarked'].mode())
data_train[['Embarked']].info()

#查看Cabin字段数据
for data in data_cleaner:
    print(data[['Cabin']].head(50))

#填充Unknown
for data in data_cleaner:
    data[['Cabin']] = data_test[['Cabin']].fillna('Unknown')
    data[['Cabin']].info()

#查看特征的分布状况
for data in data_cleaner:
    print(data.describe())

#使用matplotlib 画箱型图
 
#先看Age字段箱型图
plt.figure(figsize = (10,6))
plt.subplot(121)
data_train[['Age']].boxplot()
plt.title('train Age boxplot')
plt.subplot(122)
data_test[['Age']].boxplot()
plt.title('test Age boxplot')
plt.grid(alpha = 0.3)
plt.show()

#为了保证数据的可靠性，我们还是查看小于Age下界的记录数据
for data in data_cleaner:
    IQR = data[['Age']].quantile(0.75)-data[['Age']].quantile(0.25)
    DB = data[['Age']].quantile(0.25) - 1.5*IQR
    print(data.loc[data['Age']< DB ,:])

#Fare数据箱型图与异常值查询

#为了观察Age是否分布也存在一定规律，决定画出Age散点图
# Age散点图
i = 0
plt.figure(figsize=(10,6))
namelist = ['train','test']
for data in data_cleaner:
    i += 1
    plt.subplot(1,2,i)
    plt.scatter(data['PassengerId'],data['Age'])
    plt.title('{} scatter'.format(namelist[i-1]))
    plt.grid(alpha = 0.3)
plt.show()

#离散化
 
for data in data_cleaner:
    data['Agebin'] = pd.qcut(data['Age'],4)
    data['Farebin'] = pd.cut(data['Fare'],[0,50,100,200,300,600],labels=['A','B','C','D','E'])

#删除 Ticket
for data in data_cleaner:
    data = data.drop(columns=['Ticket'])

#Name字段的常规结果为：'名字,头衔.姓氏'，有效特征为头衔
for data in data_cleaner:
    data['Head'] = data['Name'].str.split(',').apply(lambda x:x[1].split('.')[0])
#查询Head包含的值
for data in data_cleaner:
    print(data['Head'].value_counts())
#存在量很少的统计值，将其替换成‘N’
for data in data_cleaner:
    H_values = data['Head'].value_counts()
    for i in H_values[H_values<10].index:
        data.loc[data['Head'].str.contains(i),'Head'] = 'N'

for data in data_cleaner:
    data['Cabin_c'] = data['Cabin'].str[0:1]

#进行onehot编码，但数据编码后放入模型会很麻烦，因此建立列表进行管理
x_dummy_columns = ['Agebin','Farebin','Head','Embarked','Sex','Cabin_c','Pclass']
data_train_dummy = pd.get_dummies(data_train[x_dummy_columns])
data_test_dummy = pd.get_dummies(data_test[x_dummy_columns])

#相关性分析
data_train.set_index(['PassengerId'])
 
# 设置标签
lable = preprocessing.LabelEncoder()
data_train['Sex_Code'] = lable.fit_transform(data_train['Sex'])
data_train['Agebin_Code'] = lable.fit_transform(data_train['Agebin'])
data_train['Farebin_Code'] = lable.fit_transform(data_train['Farebin'])
data_train['Head_Code'] = lable.fit_transform(data_train['Head'])
data_train['Embarked_Code'] = lable.fit_transform(data_train['Embarked'])
 
print(data_train.corr()[['Survived']].sort_values(by = 'Survived',ascending= False))

#Survived hist
plt.figure(figsize=(18,12))
 
#Fare
plt.subplot(221)
plt.hist(x = [data_train.loc[data_train['Survived'] == 0,'Fare'],data_train.loc[data_train['Survived'] == 1,'Fare']],label = ['died','Survived'],color = ['red','green'])
plt.grid(alpha = 0.3)
plt.title('Fare Survived hist')
plt.xlabel('Fare')
plt.ylabel('Survived')
plt.legend()
 
#Age
plt.subplot(222)
plt.hist(x = [data_train.loc[data_train['Survived'] == 0,'Age'],data_train.loc[data_train['Survived'] == 1,'Age']],label = ['died','Survived'],color = ['red','green'])
plt.grid(alpha = 0.3)
plt.title('Age Survived hist')
plt.xlabel('Age')
plt.ylabel('Survived')
plt.legend()
 
 
#Pclass
plt.subplot(223)
plt.hist(x = [data_train.loc[data_train['Survived'] == 0,'Pclass'],data_train.loc[data_train['Survived'] == 1,'Pclass']],label = ['died','Survived'],color = ['red','green'])
plt.grid(alpha = 0.3)
plt.title('Pclass Survived hist')
plt.xlabel('Pclass')
plt.ylabel('Survived')
plt.legend()
 
 
#Embarked
plt.subplot(224)
plt.hist(x = [data_train.loc[data_train['Survived'] == 0,'Embarked'],data_train.loc[data_train['Survived'] == 1,'Embarked']],label = ['died','Survived'],color = ['red','green'])
plt.grid(alpha = 0.3)
plt.title('Embarked Survived hist')
plt.xlabel('Embarked')
plt.ylabel('Survived')
plt.legend()
 
plt.show()

#将需要使用的模型及需要网格搜索的参数提前设定
# LogisticRegression
LR_clf = LogisticRegression(max_iter= 3000)
param_dict1 = {
    'penalty' : ['l1','l2'],
    'C':[0.1,0.5,1]
}
# DecisionTreeClassifier
DT_clf = DecisionTreeClassifier(random_state=0)
param_dict2 = {
    'max_depth':[3,4,5],
    'min_samples_leaf' : [1,2]
}
# MultinomialNB
NBC_clf = MultinomialNB()
para_dict3 ={
    'alpha' : [1.0]
}
# SVC
SVC_clf = SVC()
param_dict4 = {
    'C':[2,2.5,3],
    'kernel':['rbf','linear','poly']
}
# RandomForestClassifier
RF_clf = RandomForestClassifier()
param_dict5 = {
    'n_estimators' :range(80,200,4),
    'max_features':[2,4,6,8]
}
# # XGBClassifier  (带入网格搜索，优化参数时，一直报错：feature_names must be string，没找到合理的原因，先注释掉了)
# XGB_clf = XGBClassifier()
# param_dict6 = {
#     'n_estimators':[50,100,200],
#     'max_depth':[2,5,8],
#     'learning_rate':np.linspace(0.01,2,20),
#     'binary':['hinge']
# }
# KNeighborsClassifier
KNN_clf = KNeighborsClassifier()
param_dict6 = {
    'n_neighbors':[5]
}

Model_list = [LR_clf,DT_clf,NBC_clf,SVC_clf,RF_clf,KNN_clf]
param_list = [param_dict1,param_dict2,para_dict3,param_dict4,param_dict5,param_dict6]
estimator = []
params = []
score = []
scoring = make_scorer(accuracy_score)
for i in range(6):
    clf_R = Model_list[i]
    parameters = param_list[i]
    grid = GridSearchCV(clf_R,parameters,cv = 5,scoring= scoring,n_jobs = 4)
    grid.fit(data_train_dummy,Y)
    temp = grid.best_estimator_
    estimator.append(temp)
    temp = grid.best_params_
    params.append(temp)
    temp = grid.best_score_
    score.append(temp)

for i in range(6):
    model = Model_list[i]
    model.fit(data_train_dummy,Y)
    y_predict = model.predict(data_train_dummy)
    roc = roc_auc_score(y_true=Y,y_score=y_predict)
    recall = recall_score(y_true=Y,y_pred=y_predict)
    print('算法模型：',estimator[i],'n')
    print('模型的roc：',roc,'n')
    print('模型的召回率：',recall,'n')
    print('模型测试精确率：',score[i],'n')
    print('-'*20)

CLF = Model_list[3]
results= CLF.predict(data_test_dummy)
results = pd.DataFrame(results,columns = ['Survived'])
test = pd.concat([test,results],axis = 1)
results = test[['PassengerId','Survived']]
results = results.set_index(['PassengerId'])
results.to_csv('./Results.csv')