Jupyter Notebook 代码连接：
machine_learning_demo
machine_learning_ensembles
Step 1: Imports and Configuration
import pandas as pd
import numpy as np
import copy
import json
import pickle
import joblib
import lightgbm as lgb
import optuna
import warnings
import gc
from sklearn.metrics import roc_curve, roc_auc_score, recall_score, accuracy_score, fbeta_score, precision_score
from sklearn.model_selection import train_test_split, cross_val_score, KFold
from sklearn.base import clone
import matplotlib.pyplot as plt
import seaborn as sns
# Setting configuration.
pd.set_option('display.float_format', lambda x: '%.5f' %x)
warnings.filterwarnings('ignore')
sns.set_style('whitegrid')
optuna.logging.set_verbosity(optuna.logging.WARNING)
SEED = 42
Step 2: Load the datasets
print('Loading data...')
path = '../datasets/Home-Credit-Default-Risk/'
df = pd.read_csv(path + 'selected_data.csv', index_col='SK_ID_CURR')
Loading data...
定义帮助节省内存的函数
def convert_dtypes(df, verbose=True):
original_memory = df.memory_usage().sum()
df = df.apply(pd.to_numeric, errors='ignore')
# Convert booleans to integers
boolean_features = df.select_dtypes(bool).columns.tolist()
df[boolean_features] = df[boolean_features].astype(np.int32)
# Convert objects to category
object_features = df.select_dtypes(object).columns.tolist()
df[object_features] = df[object_features].astype('category')
# Float64 to float32
float_features = df.select_dtypes(float).columns.tolist()
df[float_features] = df[float_features].astype(np.float32)
# Int64 to int32
int_features = df.select_dtypes(int).columns.tolist()
df[int_features] = df[int_features].astype(np.int32)
new_memory = df.memory_usage().sum()
if verbose:
print(f'Original Memory Usage: {round(original_memory / 1e9, 2)} gb.')
print(f'New Memory Usage: {round(new_memory / 1e9, 2)} gb.')
return df
print("Training dataset shape: ", df.shape)
Training dataset shape: (307511, 836)
df = convert_dtypes(df)
Original Memory Usage: 2.06 gb.
New Memory Usage: 1.0 gb.
df.dtypes.value_counts()
float32 796
int32 7
category 3
category 3
category 3
category 3
category 3
category 2
category 2
category 2
category 1
category 1
category 1
category 1
category 1
category 1
category 1
category 1
category 1
category 1
category 1
category 1
Name: count, dtype: int64
Step 3: Data preprocessing
# Check if the data is unbalanced
df["TARGET"].value_counts()
TARGET
0 282686
1 24825
Name: count, dtype: int64
数据集存在轻微的样本不平衡，我们接下来测试几种处理方法，来提高模型表现。
先定义评估函数
def timer(func):
import time
import functools
def strfdelta(tdelta, fmt):
hours, remainder = divmod(tdelta, 3600)
minutes, seconds = divmod(remainder, 60)
return fmt.format(hours, minutes, seconds)
@functools.wraps(func)
def wrapper(*args, **kwargs):
click = time.time()
result = func(*args, **kwargs)
delta = strfdelta(time.time() - click, "{:.0f} hours {:.0f} minutes {:.0f} seconds")
print(f"{func.__name__} cost time {delta}")
return result
return wrapper
# Define a cross validation strategy
# We use the cross_val_score function of Sklearn.
# However this function has not a shuffle attribute, we add then one line of code,
# in order to shuffle the dataset prior to cross-validation
@timer
def evaluate(model, X, y, n_folds = 5, params=None):
kf = KFold(n_folds, shuffle=True, random_state=SEED).get_n_splits(X)
scores = cross_val_score(
model,
X,
y,
scoring="roc_auc",
cv = kf,
verbose=1,
params=params
)
print(f"valid auc: {scores.mean():.3f} +/- {scores.std():.3f}")
return scores.mean()
Split data
留25%作为模型的验证集
# Split data into training and testing sets
X_train, X_valid, y_train, y_valid = train_test_split(
df.drop(columns="TARGET"),
df["TARGET"],
test_size=0.25,
random_state=SEED
)
print("X_train shape:", X_train.shape)
print('train:', y_train.value_counts(), sep='\n')
print('valid:', y_valid.value_counts(), sep='\n')
X_train shape: (230633, 835)
train:
TARGET
0 211999
1 18634
Name: count, dtype: int64
valid:
TARGET
0 70687
1 6191
Name: count, dtype: int64
del df
gc.collect()
# Specific feature names and categorical features
feature_name = X_train.columns.tolist()
categorical_feature = X_train.select_dtypes('category').columns.tolist()
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import make_column_transformer
# Encode categorical features
encoder = make_column_transformer(
(OneHotEncoder(
drop='if_binary',
min_frequency=0.02,
max_categories=20,
sparse_output=False,
handle_unknown='ignore'
), categorical_feature),
remainder='passthrough',
verbose_feature_names_out=False,
verbose=True
)
print('fitting...')
encoder.fit(X_train)
print('encoding...')
train_dummies = encoder.transform(X_train)
valid_dummies = encoder.transform(X_valid)
print('train data shape:', X_train.shape)
fitting...
[ColumnTransformer] . (1 of 2) Processing onehotencoder, total= 4.2s
[ColumnTransformer] ..... (2 of 2) Processing remainder, total= 0.0s
encoding...
train data shape: (230633, 835)
model 1: Use default parameters
model = lgb.LGBMClassifier(
boosting_type='gbdt',
objective='binary',
metric='auc',
n_estimators=500,
random_state=SEED,
verbose=0
)
fit_params = dict(
callbacks = [lgb.early_stopping(20)],
eval_set = [(train_dummies, y_train), (valid_dummies, y_valid)]
)
score = evaluate(model, train_dummies, y_train, params=fit_params)
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[147] valid_0's auc: 0.860844 valid_1's auc: 0.778985
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[99] valid_0's auc: 0.836905 valid_1's auc: 0.777066
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[121] valid_0's auc: 0.846901 valid_1's auc: 0.777927
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[118] valid_0's auc: 0.846341 valid_1's auc: 0.778487
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[114] valid_0's auc: 0.845653 valid_1's auc: 0.776624
valid auc: 0.779 +/- 0.001
evaluate cost time 0 hours 1 minutes 57 seconds
model 2: Set class weight
model2 = clone(model) # Construct a new unfitted estimator with the same parameters.
model2.set_params(class_weight='balanced')
score = evaluate(model2, train_dummies, y_train, params=fit_params)
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[122] valid_0's auc: 0.843105 valid_1's auc: 0.780157
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[95] valid_0's auc: 0.831016 valid_1's auc: 0.780049
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[107] valid_0's auc: 0.835709 valid_1's auc: 0.779769
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[159] valid_0's auc: 0.856821 valid_1's auc: 0.781057
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[138] valid_0's auc: 0.848312 valid_1's auc: 0.779905
valid auc: 0.780 +/- 0.002
evaluate cost time 0 hours 2 minutes 20 seconds
设置 is_unbalance=True 后，模型有所改善。
model 3: SMOTE
from imblearn.over_sampling import SMOTE
import imblearn
X_balanced, y_balanced = SMOTE(random_state=SEED).fit_resample(train_dummies, y_train)
print('balanced train data shape:', X_balanced.shape)
score = evaluate(clone(model), X_balanced, y_balanced, params=fit_params)
balanced train data shape: (423998, 990)
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[64] valid_0's auc: 0.726936 valid_1's auc: 0.7216
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[138] valid_0's auc: 0.834743 valid_1's auc: 0.780546
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[167] valid_0's auc: 0.849441 valid_1's auc: 0.782093
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[140] valid_0's auc: 0.834219 valid_1's auc: 0.780796
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[166] valid_0's auc: 0.848353 valid_1's auc: 0.780799
valid auc: 0.976 +/- 0.048
evaluate cost time 0 hours 5 minutes 46 seconds
model 4: Ensemble method
from imblearn.ensemble import BalancedRandomForestClassifier
model4 = BalancedRandomForestClassifier(
n_estimators=100,
max_depth=5,
random_state=SEED,
verbose=1
)
score = evaluate(model4, train_dummies, y_train)
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 7.3s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 0.1s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 7.2s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 0.1s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 7.2s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 0.2s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 7.3s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 0.2s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 7.1s
[Parallel(n_jobs=1)]: Done 49 tasks | elapsed: 0.1s
valid auc: 0.738 +/- 0.002
evaluate cost time 0 hours 1 minutes 21 seconds
model 5: FocalLoss
from focal_loss import BinaryFocalLoss # self-define loss function
focalloss = BinaryFocalLoss(alpha=0.9, gamma=0.05)
model5 = clone(model)
model5.set_params(objective = focalloss.objective)
fit_params['eval_metric'] = focalloss.evaluate
score = evaluate(model5, train_dummies, y_train, params=fit_params)
[LightGBM] [Info] Using self-defined objective function
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[116] valid_0's auc: 0.840709 valid_0's focal_loss: 0.0792912 valid_1's auc: 0.780966 valid_1's focal_loss: 0.0886921
[LightGBM] [Info] Using self-defined objective function
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[87] valid_0's auc: 0.82691 valid_0's focal_loss: 0.0816416 valid_1's auc: 0.779874 valid_1's focal_loss: 0.0888508
[LightGBM] [Info] Using self-defined objective function
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[101] valid_0's auc: 0.832985 valid_0's focal_loss: 0.0805644 valid_1's auc: 0.779294 valid_1's focal_loss: 0.0889485
[LightGBM] [Info] Using self-defined objective function
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[87] valid_0's auc: 0.827538 valid_0's focal_loss: 0.0816012 valid_1's auc: 0.78189 valid_1's focal_loss: 0.0885146
[LightGBM] [Info] Using self-defined objective function
Training until validation scores don't improve for 20 rounds
Early stopping, best iteration is:
[119] valid_0's auc: 0.840904 valid_0's focal_loss: 0.0792486 valid_1's auc: 0.781548 valid_1's focal_loss: 0.0886565
valid auc: nan +/- nan
evaluate cost time 0 hours 2 minutes 16 seconds
自定义FocalLoss损失函数后，表现不错
del train_dummies, valid_dummies
gc.collect()
Step 4: Hyperparameter Tuning
超参数调优算法主要有网格搜索(Grid Search)，随机搜索(Randomized Search)和贝叶斯优化(Bayesian Optimization)，本文采用贝叶斯优化。
本章准备使用LightGBM原生接口，需要创建 lightgbm 原生数据集
# Create Dataset object for lightgbm
dtrain = lgb.Dataset(
X_train, label=y_train,
free_raw_data=True
)
# In LightGBM, the validation data should be aligned with training data.
# if you want to re-use data, remember to set free_raw_data=False
dvalid = lgb.Dataset(
X_valid, label=y_valid,
reference=dtrain,
free_raw_data=True
)
超参数和目标函数设置
# Here we use Optuna
# define the search space and the objecive function
def objective(trial):
# LightGBM can use a dictionary to set Parameters.
params = dict(
boosting_type = 'gbdt',
objective = 'binary',
metric = 'auc',
is_unbalance = True,
num_boost_round = trial.suggest_int("num_boost_round", 50, 2000, step=50),
learning_rate = trial.suggest_float("learning_rate", 1e-4, 10, log=True),
max_depth = trial.suggest_int("max_depth", 2, 10),
feature_fraction = trial.suggest_float("feature_fraction", 0.2, 1.0),
bagging_fraction = trial.suggest_float("bagging_fraction", 0.2, 1.0),
bagging_freq = 5,
lambda_l1 = trial.suggest_float("lambda_l1", 1e-4, 1e2, log=True),
lambda_l2 = trial.suggest_float("lambda_l2", 1e-4, 1e2, log=True),
random_state = SEED,
verbosity = -1
)
# Perform the cross-validation with given parameters.
eval_results = lgb.cv(
params,
dtrain,
nfold = 5,
shuffle = True,
feature_name = feature_name,
categorical_feature = categorical_feature,
callbacks=[lgb.early_stopping(20)]
)
return eval_results['valid auc-mean'][-1]
贝叶斯优化
# Bayesian optimization
# create a study object.
study = optuna.create_study(
study_name = 'lightgbm-study', # Unique identifier of the study.
direction = 'maximize'
)
# Invoke optimization of the objective function.
study.optimize(
objective,
n_trials = 100,
timeout = 7200,
gc_after_trial = True,
show_progress_bar = True
)
joblib.dump(study, path + "lightgbm-study.pkl")
study = joblib.load(path + "lightgbm-study.pkl")
print("Best trial until now:")
print(" Value: ", study.best_trial.value)
print(" Params: ")
for key, value in study.best_trial.params.items():
print(f" {key}: {value}")
Best trial until now:
Value: 0.785777090367696
Params:
num_boost_round: 1000
learning_rate: 0.029182324488925142
max_depth: 8
feature_fraction: 0.902981862669475
bagging_fraction: 0.9853966386414182
lambda_l1: 73.55650874339202
lambda_l2: 6.572289325673235
# Continue to study
study.optimize(
objective,
n_trials = 100,
timeout = 7200,
gc_after_trial = True,
show_progress_bar = True
)
print("Number of finished trials: ", len(study.trials))
print("Best trial until now:")
print(" Best value: ", study.best_trial.value)
print(" Best params: ")
for key, value in study.best_trial.params.items():
print(f" {key}: {value}")
Number of finished trials: 135
Best trial until now:
Best value: 0.7865747325768904
Best params:
num_boost_round: 1300
learning_rate: 0.015480784915810246
max_depth: 8
feature_fraction: 0.3519165350962246
bagging_fraction: 0.9999568798413535
lambda_l1: 65.08840723355036
lambda_l2: 15.024421566966097
可视化
绘制优化过程曲线
optuna.visualization.plot_optimization_history(study)
绘制study目标值的edf
optuna.visualization.plot_edf(study)
Step 5: Training
训练
本节准备使用LightGBM原生接口，需要创建 lightgbm 原生数据集
# Create Dataset object for lightgbm
dtrain = lgb.Dataset(
X_train, label=y_train,
free_raw_data=True
)
# In LightGBM, the validation data should be aligned with training data.
# if you want to re-use data, remember to set free_raw_data=False
dvalid = lgb.Dataset(
X_valid, label=y_valid,
reference=dtrain,
free_raw_data=True
)
print('Starting training...')
best_params = dict(
boosting_type = 'gbdt',
objective = 'binary',
metric = 'auc',
is_unbalance = True,
num_boost_round = 1300,
learning_rate = 0.015480784915810246,
max_depth = 8,
feature_fraction = 0.3519165350962246,
bagging_fraction = 0.9999568798413535,
lambda_l1 = 65.08840723355036,
lambda_l2 = 15.024421566966097,
subsample_freq = 5,
random_state = SEED,
verbosity = 0
)
eval_results = {} # to record eval results for plotting
callbacks = [
lgb.log_evaluation(period=100),
lgb.early_stopping(stopping_rounds=20),
lgb.record_evaluation(eval_results)
]
# Training
bst = lgb.train(
best_params,
dtrain,
feature_name = feature_name,
categorical_feature = categorical_feature,
valid_sets = [dtrain, dvalid],
callbacks = callbacks
)
Starting training...
[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).
[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).
[LightGBM] [Warning] Found whitespace in feature_names, replace with underlines
[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).
[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).
[LightGBM] [Warning] Found whitespace in feature_names, replace with underlines
[LightGBM] [Warning] Accuracy may be bad since you didn't explicitly set num_leaves OR 2^max_depth > num_leaves. (num_leaves=31).
Training until validation scores don't improve for 20 rounds
[100] training's auc: 0.77831 valid_1's auc: 0.760952
[200] training's auc: 0.793115 valid_1's auc: 0.770076
[300] training's auc: 0.803729 valid_1's auc: 0.775631
[400] training's auc: 0.811797 valid_1's auc: 0.778893
[500] training's auc: 0.818789 valid_1's auc: 0.78126
[600] training's auc: 0.825071 valid_1's auc: 0.782986
[700] training's auc: 0.830958 valid_1's auc: 0.784242
[800] training's auc: 0.836567 valid_1's auc: 0.785216
[900] training's auc: 0.841761 valid_1's auc: 0.785837
[1000] training's auc: 0.846603 valid_1's auc: 0.786335
[1100] training's auc: 0.851281 valid_1's auc: 0.786744
Early stopping, best iteration is:
[1118] training's auc: 0.852112 valid_1's auc: 0.786804
可视化
# Plotting metrics recorded during training
ax = lgb.plot_metric(eval_results, metric='auc')
plt.show()
Step 6: Evaluating
模型得分
def get_adjusted_prediction(y_score, threshold=0.5):
y_pred = y_score.copy()
y_pred[y_score>=threshold] = 1
y_pred[y_score< threshold] = 0
return y_pred
def classification_report(model, X, y):
from sklearn.metrics import balanced_accuracy_score
report = {}
y_true = y
y_score = model.predict(X)
if y_score.ndim >= 2:
y_pred = np.argmax(y_score)
else:
y_pred = (y_score > 0.5).astype(int)
fpr, tpr, thresholds = roc_curve(y_true, y_score)
idx = (tpr - fpr).argmax()
adjusted_threshold = thresholds[idx]
adjusted_y_pred = (y_score > adjusted_threshold).astype(int)
return {
'y_pred': y_pred,
'y_score': y_score,
'fpr': fpr,
'tpr': tpr,
'thresholds': thresholds,
'ks': (tpr - fpr).max(),
'auc': roc_auc_score(y_true, y_score),
'accuracy': accuracy_score(y_true, y_pred),
'balanced_accuracy_score': balanced_accuracy_score(y_true, y_pred),
'precision': precision_score(y_true, y_pred),
'recall': recall_score(y_true, y_pred),
'f1-score': fbeta_score(y_true, y_pred, beta=1),
'adjusted_threshold': adjusted_threshold,
'adjusted_accuracy': accuracy_score(y_true, adjusted_y_pred)
}
# the model performance
train_report = classification_report(bst, X_train, y_train)
valid_report = classification_report(bst, X_valid, y_valid)
for label, stats in [('train data', train_report), ('valid data', valid_report)]:
print(label, ":")
print(
f"auc: {stats['auc']:.5f}",
f"accuracy: {stats['accuracy']:.5f}",
f"balanced_accuracy_score: {stats['balanced_accuracy_score']:.5f}",
f"adjusted_accuracy(threshold = {stats['adjusted_threshold']:.4f}): {stats['adjusted_accuracy']:.5f}",
f"recall: {stats['recall']:.5f}",
sep = '\n\t'
)
train data :
auc: 0.85211
accuracy: 0.75527
balanced_accuracy_score: 0.77060
adjusted_accuracy(threshold = 0.4885): 0.74530
recall: 0.78888
valid data :
auc: 0.78680
accuracy: 0.73706
balanced_accuracy_score: 0.71237
adjusted_accuracy(threshold = 0.4526): 0.69454
recall: 0.68293
ROC曲线
# Plot ROC curve
def plot_roc_curve(fprs, tprs, labels):
from sklearn import metrics
plt.figure()
plt.title('Receiver Operating Characteristic')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.plot([0, 1], [0, 1],'r--')
plt.xlim([0, 1])
plt.ylim([0, 1])
# Plotting ROC and computing AUC scores
for fpr, tpr, label in zip(fprs, tprs, labels):
auc = metrics.auc(fpr, tpr)
plt.plot(fpr, tpr, label = f"{label} ROC(auc={auc:.4f})")
plt.legend(loc = 'lower right')
plot_roc_curve(
fprs = (train_report['fpr'], valid_report['fpr']),
tprs = (train_report['tpr'], valid_report['tpr']),
labels = ('train', 'valid')
)
模型稳定性
PSI(Population Stability Index)指标反映了实际分布(actual)与预期分布(expected)的差异。在建模中，我们常用来筛选特征变量、评估模型稳定性。其中，在建模时通常以训练样本(In the Sample, INS)作为预期分布，而验证样本在各分数段的分布通常作为实际分布。验证样本一般包括样本外(Out of Sample, OOS)和跨时间样本(Out of Time, OOT)。
风控模型常用PSI衡量模型的稳定性。
def calc_psi(expected, actual, n_bins=10):
'''
Calculate the PSI (Population Stability Index) for two vectors.
Args:
expected: array-like, represents the expected distribution.
actual: array-like, represents the actual distribution.
bins: int, the number of bins to use in the histogram.
Returns:
float, the PSI value.
'''
# Calculate the expected frequencies in each bin
buckets, bins = pd.qcut(expected, n_bins, retbins=True, duplicates='drop')
expected_freq = buckets.value_counts()
expected_freq = expected_freq / expected_freq.sum()
# Calculate the actual frequencies in each bin
bins = [-np.inf] + list(bins)[1: -1] + [np.inf]
actual_freq = pd.cut(actual, bins).value_counts()
actual_freq = actual_freq / actual_freq.sum()
# Calculate PSI
psi = (actual_freq - expected_freq) * np.log(actual_freq / expected_freq)
return psi.sum()
psi = calc_psi(train_report['y_score'], valid_report['y_score'])
print("PSI:", psi)
PSI: 0.00019890720303521737
绘制实际分布与预期分布曲线
plt.figure(figsize=(8, 4))
sns.kdeplot(x=train_report['y_score'], label='train')
sns.kdeplot(x=valid_report['y_score'], label='valid')
plt.legend(loc='best')
plt.title(label = 'Frequency', loc ='center')
验证集正负样本分布曲线
valid_pred = pd.DataFrame({'score': valid_report['y_score'], 'target': y_valid})
plt.figure(figsize=(8, 4))
sns.kdeplot(data=valid_pred, x='score', hue='target', common_norm=False)
plt.title(label = 'Frequency', loc ='center')
验证集正负样本累积分布
plt.figure(figsize=(8, 4))
sns.kdeplot(data=valid_pred, x='score', hue='target', common_norm=False, cumulative=True)
plt.title(label = 'Cumulative', loc ='center')
Step 7: Show feature importance
feature_imp = pd.Series(
bst.feature_importance(),
index=bst.feature_name()
).sort_values(ascending=False)
print(feature_imp.head(20))
feature_imp.to_excel(path + 'feature_importance.xlsx')
AMT_ANNUITY_/_AMT_CREDIT 776
MODE(previous.PRODUCT_COMBINATION) 590
MODE(installments.previous.PRODUCT_COMBINATION) 475
MODE(cash.previous.PRODUCT_COMBINATION) 355
EXT_SOURCE_2_+_EXT_SOURCE_3 312
MAX(bureau.DAYS_CREDIT_ENDDATE) 296
MAX(bureau.DAYS_CREDIT) 281
MODE(previous.NAME_GOODS_CATEGORY) 274
MODE(installments.previous.NAME_GOODS_CATEGORY) 270
MEAN(bureau.AMT_CREDIT_SUM_DEBT) 252
MODE(cash.previous.NAME_GOODS_CATEGORY) 248
AMT_GOODS_PRICE_/_AMT_ANNUITY 232
MEAN(previous.MEAN(cash.CNT_INSTALMENT_FUTURE)) 210
frequency(CODE_GENDER_M)_by(EXT_SOURCE_1) 196
AMT_CREDIT_-_AMT_GOODS_PRICE 195
SUM(bureau.AMT_CREDIT_SUM) 192
SUM(bureau.AMT_CREDIT_MAX_OVERDUE) 191
EXT_SOURCE_1_/_DAYS_BIRTH 182
MAX(cash.previous.DAYS_LAST_DUE_1ST_VERSION) 178
DAYS_BIRTH_/_EXT_SOURCE_1 176
dtype: int32
# Plotting feature importances
ax = lgb.plot_importance(bst, max_num_features=20)
plt.show()
观察重点特征的分布
X_valid.columns = X_valid.columns.str.replace(' ', '_')
for col in feature_imp.index[:10]:
table = pd.DataFrame({col: X_valid[col], 'label': y_valid})
if table[col].dtype in [np.float32, np.int32]:
table[f'{col}_binned'] = pd.qcut(table[col], 5, duplicates='drop')
else:
table[f'{col}_binned'] = table[col]
print(table.pivot_table(
index=f'{col}_binned',
columns='label',
values='label',
aggfunc='count')
)
if table[f'{col}_binned'].nunique() <= 5:
sns.violinplot(
data=table,
x=f'{col}_binned',
y=valid_report['y_score'],
hue='label',
split=True
)
plt.show()
label 0 1
AMT_ANNUITY_/_AMT_CREDIT_binned
(0.015799999999999998, 0.0332] 14353 1039
(0.0332, 0.0463] 14392 974
(0.0463, 0.0512] 13906 1499
(0.0512, 0.0682] 13890 1450
(0.0682, 0.124] 14146 1229
label 0 1
MODE(previous.PRODUCT_COMBINATION)_binned
Card Street 6497 729
Card X-Sell 3161 274
Cash 12431 1234
Cash Street: high 2147 240
Cash Street: low 918 81
Cash Street: middle 910 91
Cash X-Sell: high 1695 207
Cash X-Sell: low 2944 169
Cash X-Sell: middle 3851 323
POS household with interest 17801 1333
POS household without interest 3204 205
POS industry with interest 4379 282
POS industry without interest 474 17
POS mobile with interest 8690 875
POS mobile without interest 685 54
POS other with interest 816 73
POS others without interest 84 4
label 0 1
MODE(installments.previous.PRODUCT_COMBINATION)...
Card Street 4039 472
Card X-Sell 5686 628
Cash Street: high 2265 259
Cash Street: low 903 77
Cash Street: middle 1300 128
Cash X-Sell: high 2132 251
Cash X-Sell: low 4010 193
Cash X-Sell: middle 5394 394
POS household with interest 19706 1657
POS household without interest 5942 412
POS industry with interest 6274 424
POS industry without interest 828 31
POS mobile with interest 9593 1032
POS mobile without interest 1086 97
POS other with interest 1369 127
POS others without interest 160 9
label 0 1
MODE(cash.previous.PRODUCT_COMBINATION)_binned
Cash Street: high 2695 340
Cash Street: low 1053 95
Cash Street: middle 1554 171
Cash X-Sell: high 2544 315
Cash X-Sell: low 4653 258
Cash X-Sell: middle 6635 496
POS household with interest 22600 1994
POS household without interest 6767 494
POS industry with interest 6970 493
POS industry without interest 923 35
POS mobile with interest 11399 1241
POS mobile without interest 1219 112
POS other with interest 1498 136
POS others without interest 177 11
label 0 1
EXT_SOURCE_2_+_EXT_SOURCE_3_binned
(0.00013999999999999993, 0.799] 12583 2793
(0.799, 0.987] 13994 1381
(0.987, 1.132] 14411 965
(1.132, 1.264] 14688 687
(1.264, 1.681] 15011 365
label 0 1
MAX(bureau.DAYS_CREDIT_ENDDATE)_binned
(-41875.001, 80.0] 14445 941
(80.0, 823.0] 14358 1014
(823.0, 983.0] 13918 1453
(983.0, 1735.0] 13997 1399
(1735.0, 31199.0] 13969 1384
label 0 1
MAX(bureau.DAYS_CREDIT)_binned
(-2922.001, -661.0] 14554 825
(-661.0, -327.0] 14371 1031
(-327.0, -273.0] 13949 1408
(-273.0, -134.0] 14068 1326
(-134.0, -1.0] 13745 1601
label 0 1
MODE(previous.NAME_GOODS_CATEGORY)_binned
Additional Service 10 0
Audio/Video 6458 469
Auto Accessories 467 42
Clothing and Accessories 1541 89
Computers 5827 482
Construction Materials 1432 104
Consumer Electronics 5977 422
Direct Sales 29 5
Education 13 1
Fitness 17 1
Furniture 2578 143
Gardening 129 5
Homewares 242 15
Insurance 0 0
Jewelry 247 24
Medical Supplies 256 10
Medicine 112 7
Mobile 9794 935
Office Appliances 50 2
Other 49 5
Photo / Cinema Equipment 550 56
Sport and Leisure 79 13
Tourism 78 3
Vehicles 140 12
Weapon 5 0
XNA 34607 3346
label 0 1
MODE(installments.previous.NAME_GOODS_CATEGORY)...
Additional Service 9 0
Animals 1 0
Audio/Video 6092 497
Auto Accessories 347 51
Clothing and Accessories 1604 88
Computers 6324 542
Construction Materials 1571 121
Consumer Electronics 7279 562
Direct Sales 13 4
Education 14 1
Fitness 22 1
Furniture 3235 208
Gardening 175 7
Homewares 363 29
Insurance 0 0
Jewelry 238 29
Medical Supplies 400 17
Medicine 162 9
Mobile 10152 1046
Office Appliances 90 9
Other 100 7
Photo / Cinema Equipment 1115 103
Sport and Leisure 150 18
Tourism 106 4
Vehicles 261 28
Weapon 6 0
XNA 30858 2810
label 0 1
MEAN(bureau.AMT_CREDIT_SUM_DEBT)_binned
(-220213.42299999998, 0.0] 16805 994
(0.0, 39052.254] 12066 886
(39052.254, 49487.143] 13928 1448
(49487.143, 148125.9] 13904 1471
(148125.9, 43650000.0] 13984 1392
Step 8: Visualize the model
# Plotting split value histogram
ax = lgb.plot_split_value_histogram(bst, feature='AMT_ANNUITY_/_AMT_CREDIT', bins='auto')
plt.show()
# Plotting 54th tree (one tree use categorical feature to split)
# ax = lgb.plot_tree(bst, tree_index=53, figsize=(15, 15), show_info=['split_gain'])
# plt.show()
# Plotting 54th tree with graphviz
# graph = lgb.create_tree_digraph(bst, tree_index=53, name='Tree54')
# graph.render(view=True)
Step 9: Model persistence
# Save model to file
print('Saving model...')
bst.save_model(path + 'lgb_model.txt')
Saving model...
<lightgbm.basic.Booster at 0x2c457d3a0>
Step 10: Predict
# Perform predictions
# If early stopping is enabled during training, you can get predictions from the best iteration with bst.best_iteration.
predictions = bst.predict(X_valid, num_iteration=bst.best_iteration)
# Load a saved model to predict
print('Loading model to predict...')
bst = lgb.Booster(model_file=path + 'lgb_model.txt')
predictions = bst.predict(X_valid)
Loading model to predict...
# Save predictions
# predictions.to_csv('valid_predictions.csv', index=True)
Appendices: FocalLoss
import numpy as np
from scipy import optimize, special
class BinaryFocalLoss:
def __init__(self, gamma, alpha=None):
# 使用FocalLoss只需要设定以上两个参数,如果alpha=None,默认取值为1
self.alpha = alpha
self.gamma = gamma
def at(self, y):
# alpha 参数, 根据FL的定义函数,正样本权重为self.alpha,负样本权重为1 - self.alpha
if self.alpha is None:
return np.ones_like(y)
return np.where(y, self.alpha, 1 - self.alpha)
def pt(self, y, p):
# pt和p的关系
p = np.clip(p, 1e-15, 1 - 1e-15)
return np.where(y, p, 1 - p)
def __call__(self, y_true, y_pred):
# 即FL的计算公式
at = self.at(y_true)
pt = self.pt(y_true, y_pred)
return -at * (1 - pt) ** self.gamma * np.log(pt)
def grad(self, y_true, y_pred):
# 一阶导数
y = 2 * y_true - 1 # {0, 1} -> {-1, 1}
at = self.at(y_true)
pt = self.pt(y_true, y_pred)
g = self.gamma
return at * y * (1 - pt) ** g * (g * pt * np.log(pt) + pt - 1)
def hess(self, y_true, y_pred):
# 二阶导数
y = 2 * y_true - 1 # {0, 1} -> {-1, 1}
at = self.at(y_true)
pt = self.pt(y_true, y_pred)
g = self.gamma
u = at * y * (1 - pt) ** g
du = -at * y * g * (1 - pt) ** (g - 1)
v = g * pt * np.log(pt) + pt - 1
dv = g * np.log(pt) + g + 1
return (du * v + u * dv) * y * (pt * (1 - pt))
def init_score(self, y_true):
# 样本初始值寻找过程
res = optimize.minimize_scalar(
lambda p: self(y_true, p).sum(),
bounds=(0, 1),
method='bounded'
)
p = res.x
log_odds = np.log(p / (1 - p))
return log_odds
def objective(self, y_true, y_pred):
y = y_true
p = special.expit(y_pred)
return self.grad(y, p), self.hess(y, p)
def evaluate(self, y_true, y_pred):
y = y_true
p = special.expit(y_pred)
is_higher_better = False
return 'focal_loss', self(y, p).mean(), is_higher_better
def fobj(self, preds, train_data):
'''lightgbm'''
y = train_data.get_label()
p = special.expit(preds)
return self.grad(y, p), self.hess(y, p)
def feval(self, preds, train_data):
'''lightgbm'''
y = train_data.get_label()
p = special.expit(preds)
is_higher_better = False
return 'focal_loss', self(y, p).mean(), is_higher_better
class SparseCategoricalFocalLoss:
pass
Ensembles
有时候模型集成可以取得不错的效果。常用的模型集成包括：
- Votting：简单投票或加权平均
- Stacking：简单来说就是学习各个基本模型的预测值来预测最终的结果
我们初步选用 Stacking 集成学习器，采用 LogisticRegression、SVC、GaussianNB、SGDClassifier 、RandomForestClassifier、HistGradientBoostingClassifier作为基分类器。
导入必要的包
import pandas as pd
import numpy as np
import copy
import json
import pickle
import joblib
import lightgbm as lgb
import optuna
import warnings
import gc
from sklearn.ensemble import StackingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import SGDClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.model_selection import train_test_split, cross_val_score, KFold
from sklearn.metrics import roc_auc_score
from sklearn.base import clone
import matplotlib.pyplot as plt
import seaborn as sns
# Setting configuration.
pd.set_option('display.float_format', lambda x: '%.5f' %x)
warnings.filterwarnings('ignore')
sns.set_style('whitegrid')
optuna.logging.set_verbosity(optuna.logging.WARNING)
SEED = 42
创建数据集
print('Loading data...')
path = '../datasets/Home-Credit-Default-Risk/selected_data.csv'
df = pd.read_csv(path, index_col='SK_ID_CURR')
Loading data...
# Split data into training and testing sets
X_train, X_valid, y_train, y_valid = train_test_split(
df.drop(columns="TARGET"),
df["TARGET"],
test_size=0.25,
random_state=SEED
)
print("X_train shape:", X_train.shape)
print('train:', y_train.value_counts(), sep='\n')
print('valid:', y_valid.value_counts(), sep='\n')
X_train shape: (230633, 835)
train:
TARGET
0 211999
1 18634
Name: count, dtype: int64
valid:
TARGET
0 70687
1 6191
Name: count, dtype: int64
无序分类（unordered）特征原始编码对于树集成模型（tree-ensemble like XGBoost）是可用的，但对于线性回归模型（like Lasso or LogisticRegression）则必须使用one-hot重编码。因此，我们先把数据重编码。
# Specific feature names and categorical features
feature_name = X_train.columns.tolist()
categorical_feature = X_train.select_dtypes(object).columns.tolist()
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import make_column_transformer
# Encode categorical features
encoder = make_column_transformer(
(OneHotEncoder(
drop='if_binary',
min_frequency=0.02,
max_categories=20,
sparse_output=False,
handle_unknown='ignore'
), categorical_feature),
remainder='passthrough',
verbose_feature_names_out=False,
verbose=True
)
print('fitting...')
encoder.fit(X_train)
print('encoding...')
train_dummies = encoder.transform(X_train)
valid_dummies = encoder.transform(X_valid)
print('train data shape:', X_train.shape)
fitting...
[ColumnTransformer] . (1 of 2) Processing onehotencoder, total= 4.7s
[ColumnTransformer] ..... (2 of 2) Processing remainder, total= 0.0s
encoding...
train data shape: (230633, 835)
del df, X_train, X_valid
gc.collect()
2948
创建优化器
先定义一个评估函数
# Define a cross validation strategy
# We use the cross_val_score function of Sklearn.
# However this function has not a shuffle attribute, we add then one line of code,
# in order to shuffle the dataset prior to cross-validation
def evaluate(model, X, y, n_folds = 5, verbose=True):
kf = KFold(n_folds, shuffle=True, random_state=SEED).get_n_splits(X)
scores = cross_val_score(
model,
X,
y,
scoring="roc_auc",
cv = kf
)
if verbose:
print(f"valid auc: {scores.mean():.3f} +/- {scores.std():.3f}")
return scores.mean()
然后，我们定义一个优化器，对这些基分类器超参数调优。
class Objective:
estimators = (
LogisticRegression,
SGDClassifier,
GaussianNB,
RandomForestClassifier,
HistGradientBoostingClassifier
)
def __init__(self, estimator, X, y):
# assert isinstance(estimator, estimators), f"estimator must be one of {estimators}"
self.model = estimator
self.X = X
self.y = y
def __call__(self, trial):
# Create hyperparameter space
if isinstance(self.model, LogisticRegression):
search_space = dict(
class_weight = 'balanced',
C = trial.suggest_float('C', 0.01, 100.0, log=True),
l1_ratio = trial.suggest_float('l1_ratio', 0.0, 1.0) # The Elastic-Net mixing parameter, with 0 <= l1_ratio <= 1.
)
elif isinstance(self.model, SGDClassifier):
search_space = dict(
class_weight = 'balanced',
loss = trial.suggest_categorical('loss', ['hinge', 'log_loss', 'modified_huber']),
alpha = trial.suggest_float('alpha', 1e-5, 10.0, log=True),
penalty = 'elasticnet',
l1_ratio = trial.suggest_float('l1_ratio', 0.0, 1.0),
early_stopping = True
)
elif isinstance(self.model, GaussianNB):
search_space = dict(
priors = None
)
elif isinstance(self.model, RandomForestClassifier):
search_space = dict(
class_weight = 'balanced',
n_estimators = trial.suggest_int('n_estimators', 50, 500, step=50),
max_depth = trial.suggest_int('max_depth', 2, 20),
max_features = trial.suggest_float('max_features', 0.2, 0.9),
random_state = SEED
)
elif isinstance(self.model, HistGradientBoostingClassifier):
search_space = dict(
class_weight = 'balanced',
learning_rate = trial.suggest_float('learning_rate', 1e-3, 10.0, log=True),
max_iter = trial.suggest_int('max_iter', 50, 500, step=50),
max_depth = trial.suggest_int('max_depth', 2, 20),
max_features = trial.suggest_float('max_features', 0.2, 0.9),
l2_regularization = trial.suggest_float('l2_regularization', 1e-3, 10.0, log=True),
random_state = SEED,
verbose = 0
)
# Setting hyperparameters
self.model.set_params(**search_space)
# Training with 5-fold CV:
score = evaluate(self.model, self.X, self.y)
return score
超参数优化
并行执行贝叶斯优化
def timer(func):
import time
import functools
def strfdelta(tdelta, fmt):
hours, remainder = divmod(tdelta, 3600)
minutes, seconds = divmod(remainder, 60)
return fmt.format(hours, minutes, seconds)
@functools.wraps(func)
def wrapper(*args, **kwargs):
click = time.time()
result = func(*args, **kwargs)
delta = strfdelta(time.time() - click, "{:.0f} hours {:.0f} minutes {:.0f} seconds")
print(f"{func.__name__} cost time {delta}")
return result
return wrapper
# Creating a pipeline & Hyperparameter tuning
@timer
def tuning(model, X, y):
# create a study object
study = optuna.create_study(direction="maximize")
# Invoke optimization of the objective function.
objective = Objective(model, X, y)
study.optimize(
objective,
n_trials = 50,
timeout = 2400,
gc_after_trial = True,
show_progress_bar = True
)
print(model, 'best score:', study.best_value)
return study
Objective.estimators
(sklearn.linear_model._logistic.LogisticRegression,
sklearn.linear_model._stochastic_gradient.SGDClassifier,
sklearn.naive_bayes.GaussianNB,
sklearn.ensemble._forest.RandomForestClassifier,
sklearn.ensemble._hist_gradient_boosting.gradient_boosting.HistGradientBoostingClassifier)
# opt_results = []
# for model in Objective.estimators:
# study = tuning(model(), train_dummies, y_train)
# opt_results.append(study)
# print(model)
# print(study.best_trial.params)
模型训练
集成模型调优
# define the search space and the objecive function
def stacking_obj(trial):
stacking = StackingClassifier(
# The `estimators` parameter corresponds to the list of the estimators which are stacked.
estimators = [
('Logit', LogisticRegression(
class_weight = 'balanced',
C = trial.suggest_float('Logit__C', 0.01, 100.0, log=True),
l1_ratio = trial.suggest_float('Logit__l1_ratio', 0.0, 1.0) # The Elastic-Net mixing parameter, with 0 <= l1_ratio <= 1.
)),
('SGD', SGDClassifier(
class_weight = 'balanced',
loss = trial.suggest_categorical('SGD__loss', ['hinge', 'log_loss', 'modified_huber']),
alpha = trial.suggest_float('SGD__alpha', 1e-5, 10.0, log=True),
penalty = 'elasticnet',
l1_ratio = trial.suggest_float('SGD__l1_ratio', 0.0, 1.0),
early_stopping = True
)),
('GaussianNB', GaussianNB())
],
# The final_estimator will use the predictions of the estimators as input
final_estimator = LogisticRegression(
class_weight = 'balanced',
C = trial.suggest_float('final__C', 0.01, 100.0, log=True),
# The Elastic-Net mixing parameter, with 0 <= l1_ratio <= 1.
l1_ratio = trial.suggest_float('final__l1_ratio', 0.0, 1.0)
),
verbose = 1
)
score = evaluate(stacking, train_dummies, y_train, n_folds = 3)
return score
# create a study object.
study = optuna.create_study(
study_name = 'stacking-study', # Unique identifier of the study.
direction = 'maximize'
)
# Invoke optimization of the objective function.
study.optimize(
stacking_obj,
n_trials = 100,
timeout = 3600,
gc_after_trial = True,
show_progress_bar = True
)
valid auc: 0.676 +/- 0.017
valid auc: 0.669 +/- 0.021
valid auc: 0.673 +/- 0.016
valid auc: 0.451 +/- 0.121
valid auc: 0.592 +/- 0.045
valid auc: 0.666 +/- 0.017
valid auc: 0.675 +/- 0.014
valid auc: 0.666 +/- 0.021
valid auc: 0.672 +/- 0.016
valid auc: 0.667 +/- 0.021
valid auc: 0.672 +/- 0.012
joblib.dump(study, path + "stacking-study.pkl")
study = joblib.load(path + "stacking-study.pkl")
print("Best trial until now:")
print(" Value: ", study.best_trial.value)
print(" Params: ")
for key, value in study.best_trial.params.items():
print(f" {key}: {value}")
Best trial until now:
Value: 0.6761396385434888
Params:
Logit__C: 0.020329668727865235
Logit__l1_ratio: 0.5165207006926232
SGD__loss: modified_huber
SGD__alpha: 1.6638099778831132
SGD__l1_ratio: 0.7330208370976262
final__C: 14.1468564043383
final__l1_ratio: 0.4977751012657087
stacking = StackingClassifier(
# The `estimators` parameter corresponds to the list of the estimators which are stacked.
estimators = [
('Logit', LogisticRegression(
class_weight = 'balanced',
C = 0.020329668727865235,
l1_ratio = 0.5165207006926232 # The Elastic-Net mixing parameter, with 0 <= l1_ratio <= 1.
)),
('SGD', SGDClassifier(
class_weight = 'balanced',
loss = 'modified_huber',
alpha = 1.6638099778831132,
penalty = 'elasticnet',
l1_ratio = 0.7330208370976262,
early_stopping = True
)),
('GaussianNB', GaussianNB())
],
# The final_estimator will use the predictions of the estimators as input
final_estimator = LogisticRegression(
class_weight = 'balanced',
C = 14.1468564043383,
# The Elastic-Net mixing parameter, with 0 <= l1_ratio <= 1.
l1_ratio = 0.4977751012657087
),
verbose = 1
)
score = evaluate(stacking, train_dummies, y_train)
valid auc: 0.674 +/- 0.009
stacking.fit(train_dummies, y_train)
train_auc = roc_auc_score(y_train, stacking.predict_proba(train_dummies)[:, 1])
valid_auc = roc_auc_score(y_valid, stacking.predict_proba(valid_dummies)[:, 1])
print('train auc:', train_auc)
print('valid auc:', valid_auc)
train auc: 0.6753919322392181
valid auc: 0.6752015627178207
