【数据挖掘】心跳信号分类预测 之 建模调参 —— 学习笔记（四）_lgb.score

import numpy as np
import pandas as pd
from sklearn.metrics import f1_score
 
import os
import seaborn as sns
import matplotlib.pyplot as plt
 
import warnings
warnings.filterwarnings("ignore")

train = pd.read_csv('./data/train.csv')
test = pd.read_csv('./data/testA.csv')

def reduce_mem_usage(df):
    # 处理前 数据集总内存计算
    start_mem = df.memory_usage().sum() / 1024**2 
    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
    
    # 遍历特征列
    for col in df.columns:
        # 当前特征类型
        col_type = df[col].dtype
        # 处理 numeric 型数据
        if col_type != object:
            c_min = df[col].min()  # 最小值
            c_max = df[col].max()  # 最大值
            # int 型数据 精度转换
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            # float 型数据 精度转换
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        # 处理 object 型数据
        else:
            df[col] = df[col].astype('category')  # object 转 category
    
    # 处理后 数据集总内存计算
    end_mem = df.memory_usage().sum() / 1024**2 
    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
    
    return df

train_list = [] # 训练集列表
 
# 遍历每一条样本，分离信号数据作为独立特征 (id + 205 1-lead ECG records + label)
for items in train.values:
    train_list.append([items[0]] + [float(i) for i in items[1].split(',')] + [items[2]])
 
# 数据类型转换 list -> numpy -> DataFrame
train = pd.DataFrame(np.array(train_list))
 
# 特征名构造 id + s_0 ~ s_205 + label
train.columns = ['id'] + ['s_' + str(i) for i in range(len(train_list[0])-2)] + ['label']
 
# 内存优化
train = reduce_mem_usage(train)
 
# 查看
train.head()

# 同理，对测试集量化压缩
test_list=[]
for items in test.values:
    test_list.append([items[0]] + [float(i) for i in items[1].split(',')])
test = pd.DataFrame(np.array(test_list))
test.columns = ['id'] + ['s_'+str(i) for i in range(len(test_list[0])-1)]
test = reduce_mem_usage(test)
test.head()  # without labels

4.5.2 简单建模

from sklearn.model_selection import KFold  
 
# 训练-验证集标签与数据分离，方便进行 K-fold 交叉验证
X_train = train.drop(['id', 'label'], axis=1)
y_train = train['label']
 
# 随机数种子
seed = 2021  
 
# 5-fold 交叉验证
folds = 5  
kf = KFold(n_splits=folds, shuffle=True, random_state=seed)

def f1_score_vali(preds, data_vali):
    labels = data_vali.get_label()
    preds = np.argmax(preds.reshape(4, -1), axis=0)
    score_vali = f1_score(y_true=labels, y_pred=preds, average='macro')
    return 'f1_score', score_vali, True

import lightgbm as lgb
from sklearn.model_selection import train_test_split
 
# 留出法划分数据集
X_train_split, X_val, y_train_split, y_val = train_test_split(X_train, y_train, test_size=0.2)
train_matrix = lgb.Dataset(X_train_split, label=y_train_split)  # 训练集
valid_matrix = lgb.Dataset(X_val, label=y_val)  # 验证集

# 参数定义
params = {
    "learning_rate": 0.1,
    "boosting": 'gbdt',  
    "lambda_l2": 0.1,
    "max_depth": -1,
    "num_leaves": 128,
    "bagging_fraction": 0.8,
    "feature_fraction": 0.8,
    "metric": None,
    "objective": "multiclass",
    "num_class": 4,
    "nthread": 10,
    "verbose": -1,
}
 
# 模型训练
model = lgb.train(params, 
                  train_set=train_matrix, 
                  valid_sets=valid_matrix, 
                  num_boost_round=2000, 
                  verbose_eval=100, 
                  early_stopping_rounds=200,
                  feval=f1_score_vali)  # 使用了自定义指标

​

# 使用模型迭代最佳参数进行验证集预测
val_pre_lgb = model.predict(X_val, num_iteration=model.best_iteration)
# 取最大值作为预测值
preds = np.argmax(val_pre_lgb, axis=1)
# 计算验证集 F1-score
score = f1_score(y_true=y_val, y_pred=preds, average='macro')
 
print(f'未调参前lightgbm单模型在验证集上的f1：{score}')

# 每 fold 的验证得分
cv_scores = []
# 5-fold CV
for i, (train_index, valid_index) in enumerate(kf.split(X_train, y_train)):
    print('************************************ {} ************************************'.format(str(i+1)))
    X_train_split, y_train_split, X_val, y_val = X_train.iloc[train_index], y_train[train_index], X_train.iloc[valid_index], y_train[valid_index]
    # 构造数据格式
    train_matrix = lgb.Dataset(X_train_split, label=y_train_split)
    valid_matrix = lgb.Dataset(X_val, label=y_val)
    # 模型参数
    params = {
                "learning_rate": 0.1,
                "boosting": 'gbdt',  
                "lambda_l2": 0.1,
                "max_depth": -1,
                "num_leaves": 128,
                "bagging_fraction": 0.8,
                "feature_fraction": 0.8,
                "metric": None,
                "objective": "multiclass",
                "num_class": 4,
                "nthread": 10,
                "verbose": -1,
            }
    # 模型训练
    model = lgb.train(params, 
                      train_set=train_matrix, 
                      valid_sets=valid_matrix, 
                      num_boost_round=2000, 
                      verbose_eval=100, 
                      early_stopping_rounds=200,
                      feval=f1_score_vali)
    # 验证集预测并计分
    val_pred = model.predict(X_val, num_iteration=model.best_iteration)
    val_pred = np.argmax(val_pred, axis=1)
    cv_scores.append(f1_score(y_true=y_val, y_pred=val_pred, average='macro'))
    print(cv_scores)
 
print("lgb_scotrainre_list:{}".format(cv_scores))
print("lgb_score_mean:{}".format(np.mean(cv_scores)))
print("lgb_score_std:{}".format(np.std(cv_scores)))

""" 通过贪心法确定最优参数 (注意是伪代码, 不要直接运行) """
 
from sklearn.model_selection import cross_val_score
 
# objective 调优
best_obj = dict()     # 表示用于记录最佳结果
objects = [...]       # 表示存储待调参选项
for obj in objects:
    model = LGBMRegressor(objective=obj)
    # 预测并计算 roc 的相关指标 
    score = cross_val_score(model, X_train, y_train, cv=5, scoring='f1').mean()
    best_obj[obj] = score  # 记录
 
# num_leaves 调优
best_leaves = dict()  # 表示用于记录最佳结果
num_leaves = [...]    # 表示存储待调参选项
for leaves in num_leaves:
    model = LGBMRegressor(objective=min(best_obj.items(), key=lambda x:x[1])[0], 
                          num_leaves=leaves)         
    # 预测并计算 roc 的相关指标
    score = cross_val_score(model, X_train, y_train, cv=5, scoring='f1').mean()
    best_leaves[leaves] = score  # 记录
 
# max_depth 调优
best_depth = dict()  # 表示用于记录最佳结果
max_depth = [...]    # 表示存储待调参选项
for depth in max_depth:
    model = LGBMRegressor(objective=min(best_obj.items(), key=lambda x:x[1])[0],  
                          num_leaves=min(best_leaves.items(), key=lambda x:x[1])[0],
                          max_depth=depth)
    # 预测并计算 roc 的相关指标
    score = cross_val_score(model, X_train, y_train, cv=5, scoring='f1').mean()
    best_depth[depth] = score  # 记录

""" 通过网格搜索确定最优参数 """
 
from sklearn.model_selection import GridSearchCV
 
def get_best_cv_params(learning_rate=0.1, n_estimators=581, num_leaves=31, max_depth=-1, 
                       bagging_fraction=1.0, feature_fraction=1.0, bagging_freq=0, min_data_in_leaf=20,
                       min_child_weight=0.001, min_split_gain=0, reg_lambda=0, reg_alpha=0, param_grid=None):
                      
    # 设置 5-fold 交叉验证
    cv_fold = KFold(n_splits=5, shuffle=True, random_state=2021)
 
    # 模型构建
    model_lgb = lgb.LGBMClassifier(learning_rate=learning_rate,
                                   n_estimators=n_estimators,
                                   num_leaves=num_leaves,
                                   max_depth=max_depth,
                                   bagging_fraction=bagging_fraction,
                                   feature_fraction=feature_fraction,
                                   bagging_freq=bagging_freq,
                                   min_data_in_leaf=min_data_in_leaf,
                                   min_child_weight=min_child_weight,
                                   min_split_gain=min_split_gain,
                                   reg_lambda=reg_lambda,
                                   reg_alpha=reg_alpha,
                                   n_jobs=-1
                                   )
    # 封装用于 GridSearchCV 和 cross_val_score 的 f1 评分函数
    f1 = make_scorer(f1_score, average='micro')
    
    # 网格搜索设置
    grid_search = GridSearchCV(estimator=model_lgb, 
                               cv=cv_fold,
                               param_grid=param_grid,
                               scoring=f1
                               )
    
    # 开始拟合
    grid_search.fit(X_train, y_train)
    
    # 输出当前 fold 最优参数
    print('模型当前最优参数为:{}'.format(grid_search.best_params_))
    print('模型当前最优得分为:{}'.format(grid_search.best_score_))

# 以下代码耗时较长，应谨慎运行，且每一步的最优参数需要在下一步进行手动更新
  
"""
注意：除上面获取 num_boost_round 时用的是原生的 lightgbm（因为要用自带的 cv）
     以下配合 GridSearchCV 时必须使用 sklearn 接口的 lightgbm
"""
 
# 设置n_estimators 为 581，调整num_leaves和max_depth，这里选择先粗调再细调
lgb_params = {'num_leaves': range(10, 80, 5), 
              'max_depth': range(3,10,2)}
get_best_cv_params(learning_rate=0.1, n_estimators=581, num_leaves=None, max_depth=None, min_data_in_leaf=20, 
                   min_child_weight=0.001,bagging_fraction=1.0, feature_fraction=1.0, bagging_freq=0, 
                   min_split_gain=0, reg_lambda=0, reg_alpha=0, param_grid=lgb_params)
  
# num_leaves为30，max_depth为7，进一步细调num_leaves和max_depth
lgb_params = {'num_leaves': range(25, 35, 1), 
              'max_depth': range(5,9,1)}
get_best_cv_params(learning_rate=0.1, n_estimators=85, num_leaves=None, max_depth=None, min_data_in_leaf=20, 
                     min_child_weight=0.001,bagging_fraction=1.0, feature_fraction=1.0, bagging_freq=0, 
                     min_split_gain=0, reg_lambda=0, reg_alpha=0, param_grid=lgb_params)
  
# 确定min_data_in_leaf为45，min_child_weight为0.001 ，下面进行bagging_fraction、feature_fraction和bagging_freq的调参
lgb_params = {'bagging_fraction': [i/10 for i in range(5,10,1)], 
              'feature_fraction': [i/10 for i in range(5,10,1)],
              'bagging_freq': range(0,81,10)
             }
get_best_cv_params(learning_rate=0.1, n_estimators=85, num_leaves=29, max_depth=7, min_data_in_leaf=45, 
                     min_child_weight=0.001,bagging_fraction=None, feature_fraction=None, bagging_freq=None, 
                     min_split_gain=0, reg_lambda=0, reg_alpha=0, param_grid=lgb_params)
  
# 确定bagging_fraction为0.4、feature_fraction为0.6、bagging_freq为 ，下面进行reg_lambda、reg_alpha的调参
lgb_params = {'reg_lambda': [0,0.001,0.01,0.03,0.08,0.3,0.5], 
              'reg_alpha': [0,0.001,0.01,0.03,0.08,0.3,0.5]}
get_best_cv_params(learning_rate=0.1, n_estimators=85, num_leaves=29, max_depth=7, min_data_in_leaf=45, 
                     min_child_weight=0.001,bagging_fraction=0.9, feature_fraction=0.9, bagging_freq=40, 
                     min_split_gain=0, reg_lambda=None, reg_alpha=None, param_grid=lgb_params)
  
# 确定reg_lambda、reg_alpha都为0，下面进行min_split_gain的调参
lgb_params = {'min_split_gain': [i/10 for i in range(0,11,1)]}
get_best_cv_params(learning_rate=0.1, n_estimators=85, num_leaves=29, max_depth=7, min_data_in_leaf=45, 
                   min_child_weight=0.001,bagging_fraction=0.9, feature_fraction=0.9, bagging_freq=40, 
                   min_split_gain=None, reg_lambda=0, reg_alpha=0, param_grid=lgb_params)

# 参数确定好以后，设置一个比较小的 learning_rate=0.005，来确定最终的 num_boost_round
 
final_params = {
                'boosting_type': 'gbdt',
                'learning_rate': 0.01,
                'num_leaves': 29,
                'max_depth': 7,
                'objective': 'multiclass',
                'num_class': 4,
                'min_data_in_leaf':45,
                'min_child_weight':0.001,
                'bagging_fraction': 0.9,
                'feature_fraction': 0.9,
                'bagging_freq': 40,
                'min_split_gain': 0,
                'reg_lambda':0,
                'reg_alpha':0,
                'nthread': 6
                }
 
cv_result = lgb.cv(train_set=lgb_train,
                   early_stopping_rounds=20,
                   num_boost_round=5000,
                   nfold=5,
                   stratified=True,
                   shuffle=True,
                   params=final_params,
                   feval=f1_score_vali,
                   seed=0,
                   )

# 官方网站 https://github.com/fmfn/BayesianOptimization
# 相关论文：https://proceedings.neurips.cc/paper/2012/file/05311655a15b75fab86956663e1819cd-Paper.pdf
 
!pip install bayesian-optimization

from sklearn.model_selection import cross_val_score
 
# 定义贝叶斯优化函数
def rf_cv_lgb(num_leaves, max_depth, bagging_fraction, feature_fraction, bagging_freq, 
              min_data_in_leaf, min_child_weight, min_split_gain, reg_lambda, reg_alpha):
    # 建立模型
    model_lgb = lgb.LGBMClassifier(boosting_type='gbdt', 
                                   objective='multiclass', 
                                   num_class=4,
                                   learning_rate=0.1, 
                                   n_estimators=5000,
                                   num_leaves=int(num_leaves), 
                                   max_depth=int(max_depth), 
                                   bagging_fraction=round(bagging_fraction, 2), 
                                   feature_fraction=round(feature_fraction, 2),
                                   bagging_freq=int(bagging_freq), 
                                   min_data_in_leaf=int(min_data_in_leaf),
                                   min_child_weight=min_child_weight, 
                                   min_split_gain=min_split_gain,
                                   reg_lambda=reg_lambda, 
                                   reg_alpha=reg_alpha,
                                   n_jobs= 8
                                   )
    # 封装指标
    f1 = make_scorer(f1_score, average='micro')
    
    # 验证得分
    val = cross_val_score(model_lgb, X_train_split, y_train_split, cv=5, scoring=f1).mean()
 
    return val

from bayes_opt import BayesianOptimization
 
# 定义贝叶斯优化参数
bayes_lgb = BayesianOptimization(
      rf_cv_lgb, 
      {
          'num_leaves':(10, 200),
          'max_depth':(3, 20),
          'bagging_fraction':(0.5, 1.0),
          'feature_fraction':(0.5, 1.0),
          'bagging_freq':(0, 100),
          'min_data_in_leaf':(10,100),
          'min_child_weight':(0, 10),
          'min_split_gain':(0.0, 1.0),
          'reg_alpha':(0.0, 10),
          'reg_lambda':(0.0, 10),
      }
)

# 开始优化
bayes_lgb.maximize(n_iter=10)  # 迭代次数

# 显示最优参数
bayes_lgb.max

# 使用贝叶斯调优得到的新参数
base_params_lgb = {
                      'boosting_type': 'gbdt',
                      'objective': 'multiclass',
                      'num_class': 4,
                      'learning_rate': 0.01,  # 减小
                      'num_leaves': 138,
                      'max_depth': 11,
                      'min_data_in_leaf': 43,
                      'min_child_weight':6.5,
                      'bagging_fraction': 0.64,
                      'feature_fraction': 0.93,
                      'bagging_freq': 49,
                      'reg_lambda': 7,
                      'reg_alpha': 0.21,
                      'min_split_gain': 0.288,
                      'nthread': 10,
                      'verbose': -1,
}

# 使用新参数训练
cv_result_lgb = lgb.cv(
      train_set=train_matrix,
      early_stopping_rounds=1000, 
      num_boost_round=20000,
      nfold=5,
      stratified=True,
      shuffle=True,
      params=base_params_lgb,
      feval=f1_score_vali,
      seed=0
)
 
print('迭代次数{}'.format(len(cv_result_lgb['f1_score-mean'])))
print('最终模型的f1为{}'.format(max(cv_result_lgb['f1_score-mean'])))

# 使用lightgbm 5折交叉验证进行建模预测
 
import lightgbm as lgb
 
cv_scores = []
for i, (train_index, valid_index) in enumerate(kf.split(X_train, y_train)):
    print('************************************ {} ************************************'.format(str(i+1)))
    X_train_split, y_train_split, X_val, y_val = X_train.iloc[train_index], y_train[train_index], X_train.iloc[valid_index], y_train[valid_index]
 
    train_matrix = lgb.Dataset(X_train_split, label=y_train_split)
    valid_matrix = lgb.Dataset(X_val, label=y_val)
 
    params = {
                  'boosting_type': 'gbdt',
                  'objective': 'multiclass',
                  'num_class': 4,
                  'learning_rate': 0.01,
                  'num_leaves': 138,
                  'max_depth': 11,
                  'min_data_in_leaf': 43,
                  'min_child_weight':6.5,
                  'bagging_fraction': 0.64,
                  'feature_fraction': 0.93,
                  'bagging_freq': 49,
                  'reg_lambda': 7,
                  'reg_alpha': 0.21,
                  'min_split_gain': 0.288,
                  'nthread': 10,
                  'verbose': -1,
    }
 
    model = lgb.train(params, 
                      train_set=train_matrix, 
                      num_boost_round=4833, 
                      valid_sets=valid_matrix, 
                      verbose_eval=1000, 
                      early_stopping_rounds=200, 
                      feval=f1_score_vali)
    
    val_pred = model.predict(X_val, num_iteration=model.best_iteration)
    
    val_pred = np.argmax(val_pred, axis=1)
    
    cv_scores.append(f1_score(y_true=y_val, y_pred=val_pred, average='macro'))
    print(cv_scores)
 
print("lgb_scotrainre_list:{}".format(cv_scores))
print("lgb_score_mean:{}".format(np.mean(cv_scores)))
print("lgb_score_std:{}".format(np.std(cv_scores)))