保险反欺诈预测学习_umbrella_limit

前言

保险是重要的金融体系，对社会发展，民生保障起到重要作用。保险欺诈近些年层出不穷，在某些险种上保险欺诈的金额已经占到了理赔金额的20%甚至更多。对保险欺诈的识别成为保险行业中的关键应用场景。

数据集

阿里云天池里面可以找到数据集

一，数据加载

import pandas as pd
 
# 数据加载
train = pd.read_csv('train.csv')
train

test = pd.read_csv('./test.csv')
test

 合并train, test

data = pd.concat([train, test], axis=0)
data

data.index = range(len(data))
data

1000 rows × 38 columns

二，数据探索

data.isnull().sum()

policy_id 0 age 0 customer_months 0 policy_bind_date 0 policy_state 0 policy_csl 0 policy_deductable 0 policy_annual_premium 0 umbrella_limit 0 insured_zip 0 insured_sex 0 insured_education_level 0 insured_occupation 0 insured_hobbies 0 insured_relationship 0 capital-gains 0 capital-loss 0 incident_date 0 incident_type 0 collision_type 0 incident_severity 0 authorities_contacted 0 incident_state 0 incident_city 0 incident_hour_of_the_day 0...

auto_make 0 auto_model 0 auto_year 0 fraud 300 dtype: int64

唯一值个数

# 唯一值个数
for col in data.columns:
    print(col, data[col].nunique())

policy_id 1000 age 45 customer_months 385 policy_bind_date 955 policy_state 3 policy_csl 3 policy_deductable 3 policy_annual_premium 996 umbrella_limit 11 insured_zip 999 insured_sex 2 insured_education_level 7 insured_occupation 14 insured_hobbies 20 insured_relationship 6 capital-gains 490 capital-loss 525 incident_date 113 incident_type 4 collision_type 4 incident_severity 4 authorities_contacted 5 incident_state 7 incident_city 7 incident_hour_of_the_day 24

auto_make 14 auto_model 39 auto_year 21 fraud 2

cat_columns = data.select_dtypes(include='O').columns
cat_columns

Index(['policy_bind_date', 'policy_state', 'policy_csl', 'insured_sex', 'insured_education_level', 'insured_occupation', 'insured_hobbies', 'insured_relationship', 'incident_date', 'incident_type', 'collision_type', 'incident_severity', 'authorities_contacted', 'incident_state', 'incident_city', 'property_damage', 'police_report_available', 'auto_make', 'auto_model'], dtype='object')

column_name = []
 
unique_value = []
 
for col in cat_columns:
 
    column_name.append(col)
 
    unique_value.append(data[col].nunique())
 
 
 
df = pd.DataFrame()
 
df['col_name'] = column_name
 
df['value'] = unique_value
 
df = df.sort_values('value', ascending=False)
 
df

data[cat_columns]

1000 rows × 19 columns

单独看某个字段

# 单独看某个字段
data['property_damage'].value_counts()
data['property_damage'] = data['property_damage'].map({'NO': 0, 'YES': 1, '?': 2})
data['property_damage'].value_counts()

2 360

0 338

1 302

Name: property_damage, dtype: int64

data['police_report_available'].value_counts()
data['police_report_available'] = data['police_report_available'].map({'NO': 0, 'YES': 1, '?': 2})
data['police_report_available'].value_counts()

2 343

0 343

1 314

Name: police_report_available, dtype: int64

查看最大日期，最小日期

# policy_bind_date, incident_date
data['policy_bind_date'] = pd.to_datetime(data['policy_bind_date'])
data['incident_date'] = pd.to_datetime(data['incident_date'])
 
# 查看最大日期，最小日期
data['policy_bind_date'].min() # 1990-01-08
data['policy_bind_date'].max() # 2015-02-22
 
data['incident_date'].min() # 2015-01-01
data['incident_date'].max() # 2015-03-01

Timestamp('2015-03-29 00:00:00')

base_date = data['policy_bind_date'].min()
# 转换为date_diff
data['policy_bind_date_diff'] = (data['policy_bind_date'] - base_date).dt.days

data['incident_date_diff'] = (data['incident_date'] - base_date).dt.days
data['incident_date_policy_bind_date_diff'] = data['incident_date_diff'] - data['policy_bind_date_diff']
data[['policy_bind_date', 'incident_date', 'policy_bind_date_diff', 'incident_date_diff', 'incident_date_policy_bind_date_diff']]

1000 rows × 5 columns

去掉原始日期字段 policy_bind_date    incident_date

data.drop(['policy_bind_date', 'incident_date'], axis=1, inplace=True)
data

1000 rows × 39 columns

data.drop(['policy_id'], axis=1, inplace=True)
data.columns

Index(['age', 'customer_months', 'policy_state', 'policy_csl', 'policy_deductable', 'policy_annual_premium', 'umbrella_limit', 'insured_zip', 'insured_sex', 'insured_education_level', 'insured_occupation', 'insured_hobbies', 'insured_relationship', 'capital-gains', 'capital-loss', 'incident_type', 'collision_type', 'incident_severity', 'authorities_contacted', 'incident_state', 'incident_city', 'incident_hour_of_the_day', 'number_of_vehicles_involved', 'property_damage', 'bodily_injuries', 'witnesses', 'police_report_available', 'total_claim_amount', 'injury_claim', 'property_claim', 'vehicle_claim', 'auto_make', 'auto_model', 'auto_year', 'fraud', 'policy_bind_date_diff', 'incident_date_diff', 'incident_date_policy_bind_date_diff'], dtype='object')

## 标签编码
from sklearn.preprocessing import LabelEncoder
cat_columns = data.select_dtypes(include='O').columns
for col in cat_columns:
    le = LabelEncoder()
    data[col] = le.fit_transform(data[col])
data[cat_columns]

1000 rows × 15 columns

三， 数据集切分

train = data[data['fraud'].notnull()]
test = data[data['fraud'].isnull()]

四，模型训练 

import lightgbm as lgb
model_lgb = lgb.LGBMClassifier(
            num_leaves=2**5-1, reg_alpha=0.25, reg_lambda=0.25, objective='binary',
            max_depth=-1, learning_rate=0.005, min_child_samples=3, random_state=2022,
            n_estimators=2000, subsample=1, colsample_bytree=1,
        )
# 模型训练
model_lgb.fit(train.drop(['fraud'], axis=1), train['fraud'])
# AUC评测： 以proba进行提交，结果会更好
y_pred = model_lgb.predict_proba(test.drop(['fraud'], axis=1))
y_pred

array([[9.45512572e-01, 5.44874283e-02], [2.82473773e-01, 7.17526227e-01], [9.93965667e-01, 6.03433310e-03], [9.76151564e-01, 2.38484363e-02], [9.80256195e-01, 1.97438052e-02], [9.14403023e-01, 8.55969772e-02], [9.92182848e-01, 7.81715179e-03], [9.95752551e-01, 4.24744905e-03], [9.97857734e-01, 2.14226648e-03], [9.87760328e-01, 1.22396721e-02], [9.82583675e-01, 1.74163246e-02], [9.94944970e-01, 5.05503008e-03], [2.40580578e-01, 7.59419422e-01], [9.94618068e-01, 5.38193197e-03], [9.93408223e-01, 6.59177664e-03], [6.73835883e-01, 3.26164117e-01], [9.77552932e-01, 2.24470682e-02], [9.90247497e-01, 9.75250283e-03], [4.04842173e-02, 9.59515783e-01], [9.90730834e-01, 9.26916620e-03], [9.88710210e-01, 1.12897903e-02], [9.40698501e-01, 5.93014990e-02], [1.80777338e-01, 8.19222662e-01], [9.69550179e-01, 3.04498213e-02], [9.88685801e-01, 1.13141994e-02],

[9.94965319e-01, 5.03468063e-03], [9.94962679e-01, 5.03732092e-03], [9.96830618e-01, 3.16938159e-03], [9.90193503e-01, 9.80649715e-03], [9.91544053e-01, 8.45594716e-03]])

测评分数 

train['fraud'].mean()

0.25857142857142856

y_pred[:, 1]

array([5.44874283e-02, 7.17526227e-01, 6.03433310e-03, 2.38484363e-02, 1.97438052e-02, 8.55969772e-02, 7.81715179e-03, 4.24744905e-03, 2.14226648e-03, 1.22396721e-02, 1.74163246e-02, 5.05503008e-03, 7.59419422e-01, 5.38193197e-03, 6.59177664e-03, 3.26164117e-01, 2.24470682e-02, 9.75250283e-03, 9.59515783e-01, 9.26916620e-03, 1.12897903e-02, 5.93014990e-02, 8.19222662e-01, 3.04498213e-02, 1.13141994e-02, 7.87110802e-01, 3.51231514e-03, 3.11553853e-03, 5.00394301e-01, 3.87089331e-03, 5.79284039e-01, 1.03762342e-02, 9.74583399e-01, 1.50181609e-02, 1.26499281e-02, 1.66637544e-02, 8.67847939e-01, 3.41162415e-03, 8.04672111e-03, 2.10217393e-03, 9.10874722e-01, 5.99570004e-03, 2.42292836e-03, 3.68146008e-03, 9.37376096e-02, 6.16000904e-03, 1.61912989e-02, 7.34289768e-03, 9.09509256e-01, 5.45276866e-03, 6.32876633e-03, 5.47020132e-03, 4.88930674e-03, 5.57701967e-01, 3.34273498e-01, 5.34055926e-03, 1.25786465e-02, 8.42825670e-01, 2.30038169e-02, 5.59174107e-01, 6.38115178e-03, 1.22456277e-02, 2.25640586e-02, 8.64430205e-01, 6.02454975e-03, 1.80564536e-02, 8.72487291e-01, 9.59318359e-01, 8.09359944e-03, 9.75164716e-03, 7.31212517e-03, 2.44207611e-02, 6.63915914e-01, 7.02252967e-01, 1.08551058e-03, 2.71298989e-03, 1.26591748e-02, 7.70278791e-04, 8.35599276e-01, 8.27508089e-03, 1.27339206e-02, 2.43397773e-02, 2.56317959e-01, 4.77122460e-02, 7.70369021e-03, 2.48156544e-03, 1.51776195e-02, 3.29396922e-03, 1.24885745e-02, 7.51917783e-01, 4.29885247e-03, 8.24166104e-01, 9.22399316e-01, 7.32952970e-03, 2.87178506e-03, 2.00915417e-02, 1.29787033e-02, 1.06394187e-02, 2.23851715e-01, 9.45179658e-03,

1.64958505e-02, 6.79121674e-02, 1.30053897e-02, 7.09693541e-01, 5.58447882e-01, 1.05058832e-03, 7.19235810e-01, 4.89357407e-02, 1.95654359e-01, 9.78073382e-01, 4.08296287e-02, 9.41766778e-01, 4.98860263e-01, 4.33861786e-03, 3.91737138e-01, 5.03468063e-03, 5.03732092e-03, 3.16938159e-03, 9.80649715e-03, 8.45594716e-03])

result = pd.read_csv('submission.csv')
result['fraud'] = y_pred[:, 1]
result.to_csv('baseline.csv', index=False)