题目是要预测银行里什么样的客户会流失，流失的概率是多少

我这边先展示一下我写的二分类的算法

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

# 读取训练集和测试集数据
train = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\train.csv")
test = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\test.csv")

# 删除不需要的列
data = train
data.drop(['id','CustomerId','Surname'],axis=1,inplace=True)

# 对分类变量进行独热编码
object_cols = data.select_dtypes(include=['object']).columns
dumm = pd.get_dummies(data, columns=object_cols, prefix_sep='')

# 数据缩放
data = dumm
data['CreditScore'] = (data['CreditScore'] - data['CreditScore'].min()) / (data['CreditScore'].max() - data['CreditScore'].min())
data['EstimatedSalary'] = (data['EstimatedSalary'] - data['EstimatedSalary'].min()) / (data['EstimatedSalary'].max() - data['EstimatedSalary'].min())

# 划分训练集和测试集
X = data.drop('Exited',axis=1)  # 特征集 X
y = data['Exited']    # 标签集 y
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)

# 使用逻辑回归模型进行训练和预测
LR = LogisticRegression()
LR.fit(X_train, y_train)
print('训练集准确率：\n', LR.score(X_train, y_train))
print('验证集准确率：\n', LR.score(X_test, y_test))

# 对测试集进行预测
data = test
data.drop(['id','CustomerId','Surname'],axis=1,inplace=True)
object_cols = data.select_dtypes(include=['object']).columns
dumm = pd.get_dummies(data, columns=object_cols, prefix_sep='')

# 数据缩放
data = dumm
data['CreditScore'] = (data['CreditScore'] - data['CreditScore'].min()) / (data['CreditScore'].max() - data['CreditScore'].min())
data['EstimatedSalary'] = (data['EstimatedSalary'] - data['EstimatedSalary'].min()) / (data['EstimatedSalary'].max() - data['EstimatedSalary'].min())

# 进行预测
y_pred = LR.predict(data)
print(y_pred)

# 将预测结果保存到CSV文件中
df = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\test.csv")
id = df['id']
result = pd.DataFrame({'id':id, 'Exited':y_pred})
result.to_csv('2combined_columns.csv', index=False)

但是我预测出来基本Exited全是0

这里我应该是特征处理做的太草率，或者是数据参数问题

跑了这么多分

然后加以了改进，我用到了管道，交叉验证

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

# 读取训练集和测试集数据
train = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\train.csv")
test = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\test.csv")

# 删除不需要的列
train.drop(['id', 'CustomerId', 'Surname'], axis=1, inplace=True)
test.drop(['id', 'CustomerId', 'Surname'], axis=1, inplace=True)

# 定义特征和目标变量
X = train.drop('Exited', axis=1)
y = train['Exited']

# 划分训练集和验证集
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=42)

# 定义数值和分类特征
num_cols = ['CreditScore', 'Age', 'Balance', 'EstimatedSalary']
cat_cols = ['Geography', 'Gender', 'Tenure', 'NumOfProducts', 'HasCrCard', 'IsActiveMember']

# 创建预处理步骤
numeric_transformer = Pipeline(steps=[
    ('scaler', StandardScaler())
])

categorical_transformer = Pipeline(steps=[
    ('onehot', OneHotEncoder(handle_unknown='ignore'))
])

preprocessor = ColumnTransformer(
    transformers=[
        ('num', numeric_transformer, num_cols),
        ('cat', categorical_transformer, cat_cols)
    ])

# 创建逻辑回归模型的管道
model = Pipeline(steps=[
    ('preprocessor', preprocessor),
    ('classifier', LogisticRegression(max_iter=1000))
])
# 定义要尝试的参数网格
param_grid = {
    'classifier__C': [0.1, 1, 10],  # 逻辑回归的正则化强度
    'classifier__penalty': ['l1', 'l2']  # 正则化类型
}

# 创建 GridSearchCV 对象
grid_search = GridSearchCV(estimator=model, param_grid=param_grid, cv=5, scoring='accuracy', verbose=1)

# 训练模型
grid_search.fit(X_train, y_train)

# 获取最佳参数和最佳模型
best_params = grid_search.best_params_
best_model = grid_search.best_estimator_



# 训练模型
model.fit(X_train, y_train)

# 验证模型
y_val_pred = best_model.predict(X_val)
print('验证集准确率：', accuracy_score(y_val, y_val_pred))
print(confusion_matrix(y_val, y_val_pred))
print(classification_report(y_val, y_val_pred))

# 对测试集进行预测
test_predictions = best_model.predict(test)
test = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\test.csv")

# 将预测结果保存到CSV文件中
submission = pd.DataFrame({
    'id': test['id'],  # 如果需要保留 id 列
    'Exited': test_predictions
})
submission.to_csv('3catboost_submission.csv', index=False)
print("Submission file created: catboost_submission.csv")

1. 数据预处理整合到管道中：通过使用 ColumnTransformer 和 Pipeline，将数据预处理步骤（包括数值特征的标准化和分类特征的独热编码）整合到了模型训练的管道中。这样做的好处是，预处理步骤和模型训练步骤可以一起执行，简化了代码，并且确保了训练集和测试集使用相同的预处理步骤。

2. 使用 GridSearchCV 进行参数调优：这是一个重要的改进，因为模型的性能很大程度上取决于其参数的设置。

3. 避免数据泄露：通过在管道中整合预处理步骤，您确保了测试集的预测是在与训练集相同的预处理步骤之后进行的，这有助于避免数据泄露。

4. 模型参数调整：在 LogisticRegression 中设置了 max_iter=1000，这有助于确保收敛，特别是在处理较大的数据集时。

跑了这么多分

然后还有一大佬写的用GBM梯度提升来做的

导入库

# import libraries

# to handle the data
import pandas as pd
import numpy as np

# to visualize the dataset
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.graph_objects as go

# to preprocess the data
from sklearn.preprocessing import MinMaxScaler, LabelEncoder #用于将特征缩放到给定的最小值和最大值之间。#用于将标签编码为从0开始的连续整数

# machine learning
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_extraction.text import TfidfVectorizer #用于将文本数据转换为TF-IDF特征向量。
from sklearn.decomposition import TruncatedSVD  #用于降维的奇异值分解（SVD）方法。
from sklearn.model_selection import cross_val_score

# model
import lightgbm as lgb
from catboost import CatBoostClassifier, Pool
import xgboost as xgb

# evaluation
from sklearn.metrics import roc_auc_score, accuracy_score

# max columns 
pd.set_option('display.max_columns', None)

# hide warnings
import warnings
warnings.filterwarnings('ignore')

数据

df_train = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\train.csv")
df_test = pd.read_csv("C:\\Users\\91144\\Desktop\\kaggle比赛数据\\Bank Churn 数据集进行二元分类\\playground-series-s4e1\\test.csv")

处理缺失值

#缺失值
train_null = df_train.isnull().sum().sum()
test_null = df_test.isnull().sum().sum()
print(f"Null count in Training Data: {train_null}")
print(f"Null count in Test Data: {test_null}")

处理重复值

#重复值
train_duplicate = df_train.drop("id",axis = 1).duplicated().sum()
test_duplicate = df_test.drop("id",axis = 1).duplicated().sum()
print(f"Duplicate count in Training Data: {train_duplicate}")
print(f"Duplicate count in Test Data: {test_duplicate}")

df_train.info()

df_train.describe().T

查看各个特征对流失率的影响

#目标变量是不平衡的，所以我们对这个不平衡的数据集使用分层交叉验证
fig, ax = plt.subplots(figsize=(9, 4))

# Create the count plot
sns.countplot(y="Exited", data=df_train, ax=ax, palette="deep")

# Customize the plot
ax.set_title("Distribution of Exited", fontsize=18, fontweight='semibold', pad=20)
ax.set_xlabel("Count", fontsize=14, labelpad=10)
ax.set_ylabel("Exited", fontsize=14, labelpad=10)

# Add value labels to the bars
for container in ax.containers:
    ax.bar_label(container, label_type='center', fontsize=12, padding=5, color='white', fontweight='bold')

plt.show()

# Create the figure and axes
fig, ax = plt.subplots(figsize=(9, 4))

# Create the count plot
sns.countplot(y="Gender", data=df_train, ax=ax, palette="deep")

# Customize the plot
ax.set_title("Distribution of Gender", fontsize=18, fontweight='semibold', pad=20)
ax.set_xlabel("Count", fontsize=14, labelpad=10)
ax.set_ylabel("Gender", fontsize=14, labelpad=10)

# Add value labels to the bars
for container in ax.containers:
    ax.bar_label(container, label_type='center', fontsize=12, padding=5, color='white', fontweight='bold')

plt.show()

# Create the figure and axes
fig, ax = plt.subplots(figsize=(9, 4))

# Create the count plot
sns.countplot(y="Tenure", data=df_train, ax=ax, palette="deep")

# Customize the plot
ax.set_title("Distribution of Tenure", fontsize=18, fontweight='semibold', pad=20)
ax.set_xlabel("Count", fontsize=14, labelpad=10)
ax.set_ylabel("Tenure", fontsize=14, labelpad=10)

# Add value labels to the bars
for container in ax.containers:
    ax.bar_label(container, label_type='edge', fontsize=10, padding=2, color='black', fontweight='normal')

# Adjust layout to prevent label cutoff
plt.tight_layout()

plt.show()

# Create the figure and axes
fig, ax = plt.subplots(figsize=(9, 4))

# Create the count plot
sns.countplot(y="NumOfProducts", data=df_train, ax=ax, palette="deep")

# Customize the plot
ax.set_title("Distribution of NumOfProducts", fontsize=18, fontweight='semibold', pad=20)
ax.set_xlabel("Count", fontsize=14, labelpad=10)
ax.set_ylabel("NumOfProducts", fontsize=14, labelpad=10)

# Add value labels to the bars
for container in ax.containers:
    ax.bar_label(container, label_type='edge', fontsize=10, padding=2, color='black', fontweight='normal')

# Adjust layout to prevent label cutoff
plt.tight_layout()

plt.show()

# Create the figure and axes
fig, ax = plt.subplots(figsize=(9, 4))

# Create the count plot
sns.countplot(y="HasCrCard", data=df_train, ax=ax, palette="deep")

# Customize the plot
ax.set_title("Distribution of HasCrCard", fontsize=18, fontweight='semibold', pad=20)
ax.set_xlabel("Count", fontsize=14, labelpad=10)
ax.set_ylabel("HasCrCard", fontsize=14, labelpad=10)

# Add value labels to the bars
for container in ax.containers:
    ax.bar_label(container, label_type='center', fontsize=12, padding=5, color='white', fontweight='bold')

plt.show()

# Create the figure and axes
fig, ax = plt.subplots(figsize=(9, 4))

# Create the count plot
sns.countplot(y="IsActiveMember", data=df_train, ax=ax, palette="deep")

# Customize the plot
ax.set_title("Distribution of IsActiveMember", fontsize=18, fontweight='semibold', pad=20)
ax.set_xlabel("Count", fontsize=14, labelpad=10)
ax.set_ylabel("IsActiveMember", fontsize=14, labelpad=10)

# Add value labels to the bars
for container in ax.containers:
    ax.bar_label(container, label_type='center', fontsize=12, padding=5, color='white', fontweight='bold')

plt.show()

cat_cols = ['Geography', 'Gender', 'Tenure', 'NumOfProducts', 'HasCrCard',
       'IsActiveMember']

target = 'Exited'

fig = plt.figure(figsize=(9, len(cat_cols)*1.8))

# background_color = 'grey'
for i, col in enumerate(cat_cols):
    
    plt.subplot(len(cat_cols)//2 + len(cat_cols) % 2, 2, i+1)
    sns.countplot(x=col, hue=target, data=df_train, palette='deep', color='#26090b', edgecolor='#26090b')
    plt.title(f"{col} countplot by target", fontweight = 'bold')
    plt.ylim(0, df_train[col].value_counts().max() + 10)
    
plt.tight_layout()
plt.show()

num_cols = ['CreditScore', 'Age', 'Balance', 'EstimatedSalary']
colors = ['#4e79a7', '#f28e2b', '#e15759', '#76b7b2']

fig, axes = plt.subplots(2, 2, figsize=(12, 10))
fig.suptitle("Distribution of Numerical Features", fontsize=20, fontweight='bold', y=1.02)

for i, column in enumerate(num_cols):
    ax = axes[i//2, i%2]
    sns.histplot(data=df_train, x=column, kde=True, bins=30, ax=ax, color=colors[i], edgecolor='white', linewidth=0.8)
    
    mean, median = df_train[column].mean(), df_train[column].median()
    ax.axvline(mean, color='red', linestyle='dashed', linewidth=2, label=f'Mean: {mean:.2f}')
    ax.axvline(median, color='blue', linestyle='dashed', linewidth=2, label=f'Median: {median:.2f}')

    ax.set_title(column, fontsize=16, pad=15)
    ax.set_xlabel(column, fontsize=14, labelpad=10)
    ax.set_ylabel('Frequency', fontsize=14, labelpad=10)
    ax.tick_params(axis='both', which='major', labelsize=12)
    ax.grid(True, linestyle='--', alpha=0.7)
    ax.set_axisbelow(True)
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.legend(fontsize=12)

plt.tight_layout()
fig.subplots_adjust(top=0.93, hspace=0.3, wspace=0.25)
plt.show()

palette_cmap = ["#6c9a76","#cc4b57","#764a23","#f25a29","#f7941d"]
df_corr = df_train.copy()

catcol = [col for col in df_corr.columns if df_corr[col].dtype == "object"]
le = LabelEncoder()
for col in catcol:
        df_corr[col] = le.fit_transform(df_corr[col])


plt.subplots(figsize =(10, 10))

sns.heatmap(df_corr.corr(), cmap = palette_cmap, square=True, cbar_kws=dict(shrink =.82), 
            annot=True, vmin=-1, vmax=1, linewidths=3,linecolor='#e0b583',annot_kws=dict(fontsize =8))
plt.title("Pearson Correlation Of Features\n", fontsize=25)
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.show()

我们可以看到，年龄和人口流动之间有很强的正相关性，这意味着，当年龄越大，人口流动的可能性越大，这可以告诉我们，随着年龄的增长，人口流动的可能性越大。

此外，我们可以看到这次(产品数量和活跃成员)与退出概率之间的强烈负相关，这告诉我们，当一个人更活跃的时候，他们退出的概率是低的，也是当一个客户有更多的产品在银行，他们不太可能搅局。

由此我们可以看出，这两个变量在决定客户流失的可能性是非常重要的。

numeirc_cols = ['Age','CreditScore', 'Balance','EstimatedSalary']
#Use Loop Function
for col in numeirc_cols:
    sc = MinMaxScaler()
    df_train[col+"_scaled"] = sc.fit_transform(df_train[[col]])
    df_test[col+"_scaled"] = sc.fit_transform(df_test[[col]])

拼接 

df_train['Sur_Geo_Gend_Sal'] = df_train['CustomerId'].astype(str) + \
                               df_train['Surname'] + \
                               df_train['Geography'] + \
                               df_train['Gender'] + \
                               np.round(df_train['EstimatedSalary']).astype(str)

df_test['Sur_Geo_Gend_Sal'] = df_test['CustomerId'].astype(str) + \
                              df_test['Surname'] + \
                              df_test['Geography'] + \
                              df_test['Gender'] + \
                              np.round(df_test['EstimatedSalary']).astype(str)

 将文本数据转换为TF-IDF特征向量

def get_vectors(df_train,df_test,col_name):

    vectorizer = TfidfVectorizer(max_features=1000)
    vectors_train = vectorizer.fit_transform(df_train[col_name])
    vectors_test = vectorizer.transform(df_test[col_name])
    
    #用svd降维
    svd = TruncatedSVD(3)
    x_sv_train = svd.fit_transform(vectors_train)
    x_sv_test = svd.transform(vectors_test)

    #将数据转换为 pandas 的 DataFrame 结构
    tfidf_df_train = pd.DataFrame(x_sv_train)
    tfidf_df_test = pd.DataFrame(x_sv_test)

    #命名
    cols = [(col_name + "_tfidf_" + str(f)) for f in tfidf_df_train.columns.to_list()]
    tfidf_df_train.columns = cols
    tfidf_df_test.columns = cols

    #合并
    df_train = df_train.reset_index(drop=True)
    df_test = df_test.reset_index(drop=True)
    df_train = pd.concat([df_train, tfidf_df_train], axis="columns")
    df_test = pd.concat([df_test, tfidf_df_test], axis="columns")
    return df_train,df_test

 SVD降维通常用于文本挖掘（如TF-IDF矩阵降维）、图像处理、推荐系统等领域。然而，SVD也有一些局限性，比如计算复杂度较高，对于非常大的数据集可能不够高效。此外，SVD是一种线性降维方法，可能无法捕捉到数据中的所有非线性结构。在这些情况下，可以考虑使用其他降维技术，如主成分分析（PCA）或t-SNE。

df_train,df_test = get_vectors(df_train,df_test,'Surname')
df_train,df_test = get_vectors(df_train,df_test,'Sur_Geo_Gend_Sal')

df_train.head()

 将数据集中的某些列转换为新的特征，并对这些特征进行处理

def feature_data(df):
    
    df['Senior'] = df['Age'].apply(lambda x: 1 if x >= 60 else 0)
    df['Active_by_CreditCard'] = df['HasCrCard'] * df['IsActiveMember']
    df['Products_Per_Tenure'] =  df['Tenure'] / df['NumOfProducts']
    df['AgeCat'] = np.round(df.Age/20).astype('int').astype('category')
    
    cat_cols = ['Geography', 'Gender', 'NumOfProducts','AgeCat']    #onehotEncoding
    df=pd.get_dummies(df,columns=cat_cols)
    return df

#Genrating New Features
df_train = feature_data(df_train)
df_test = feature_data(df_test)

##Selecting Columns FOr use 
feat_cols=df_train.columns.drop(['id', 'CustomerId', 'Surname','Exited','Sur_Geo_Gend_Sal'])
feat_cols=feat_cols.drop(numeirc_cols)

#Printing
print(feat_cols)
df_train.head()

X=df_train[feat_cols]
y=df_train['Exited']

# LightGBM Parameters
lgbParams = {'n_estimators': 1000,
             'max_depth': 25, 
             'learning_rate': 0.025,
             'min_child_weight': 3.43,
             'min_child_samples': 216, 
             'subsample': 0.782,
             'subsample_freq': 4, 
             'colsample_bytree': 0.29, 
             'num_leaves': 21,
             'verbose':-1}

lgb_model = lgb.LGBMClassifier(**lgbParams)
lgb_cv_scores = cross_val_score(lgb_model, X, y, cv=10, scoring='roc_auc')

print("Cross-validation scores:", lgb_cv_scores)
print("Mean AUC:", lgb_cv_scores.mean())

 这段代码是使用Python的LightGBM库进行机器学习模型训练和交叉验证的例子。LightGBM是一个梯度提升框架，用于训练预测模型，它在处理大型数据集时非常高效。

lgb_model.fit(X,y)

test_predictions = lgb_model.predict_proba(df_test[feat_cols])[:, 1]

# Create a submission DataFrame
submission = pd.DataFrame({
    'id': df_test['id'],
    'Exited': test_predictions
})

# # Save the submission file
submission.to_csv('3submission.csv', index=False)

# Initialize CatBoostClassifier
cat_model = CatBoostClassifier(
    eval_metric='AUC',
    learning_rate=0.022,
    iterations=1000,
    verbose=False
)

# Perform cross-validation with StratifiedKFold
catboost_cv_scores = cross_val_score(cat_model, X, y, cv=5, scoring='roc_auc')

print("Cross-validation scores:", catboost_cv_scores)
print("Mean AUC:", catboost_cv_scores.mean())

#Cat_features
cat_features = np.where(X.dtypes != np.float64)[0]

# Train the model on the entire dataset
train_pool = Pool(X, y, cat_features=cat_features)
cat_model.fit(train_pool)

# Make predictions on the test set
test_pool = Pool(df_test[feat_cols], cat_features=cat_features)
test_predictions = cat_model.predict_proba(test_pool)[:, 1]

# Create submission DataFrame
submission = pd.DataFrame({
    'id': df_test['id'],
    'Exited': test_predictions
})

# Save the submission file
submission.to_csv('catboost_submission.csv', index=False)
print("Submission file created: catboost_submission.csv")

xgb_params = {
    'max_depth': 6,
    'learning_rate': 0.01,
    'n_estimators': 1000,
    'min_child_weight': 1,
    'subsample': 0.8,
    'colsample_bytree': 0.8,
    'gamma': 0,
    'objective': 'binary:logistic',
    'eval_metric': 'auc',
    'use_label_encoder': False,
    'nthread': -1,
    'random_state': 42
}

# Initialize XGBoost Classifier
xgb_model = xgb.XGBClassifier(**xgb_params)

# Perform cross-validation with StratifiedKFold
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
xgb_cv_scores = cross_val_score(xgb_model, X, y, cv=cv, scoring='roc_auc')

print("Cross-validation scores:", xgb_cv_scores)
print("Mean AUC:", xgb_cv_scores.mean())

classifiers = ['LightGBM', 'CatBoost', 'XGBoost']
auc_scores = [lgb_cv_scores.mean(), catboost_cv_scores.mean(), xgb_cv_scores.mean()]

# Create data for the plot
colors = ['#4e79a7', '#f28e2b', '#e15759']

# Create the figure with optimized settings
fig = go.Figure(data=[go.Bar(
    x=classifiers,
    y=auc_scores,
    name='AUC Score',
    marker_color=colors
)])

# Update layout with optimized settings
fig.update_layout(
    title='AUC Comparison',
    xaxis_title='Classifier',
    yaxis_title='AUC Score',
    template='plotly_white',
    font=dict(family="Arial", size=12),
    width=600,
    margin=dict(l=50, r=50, t=50, b=50)
)

# Add gridlines
fig.update_yaxes(showgrid=True, gridwidth=1, gridcolor='#E0E0E0')

# Show the plot
fig.show()

创建一个柱状图，用于比较不同分类器的 AUC 分数。 

# Selcting Best and Highest AUC_Score  from Above trained Models 

# Find the index of the maximum AUc_Score
best_accuracy_index = auc_scores.index(max(auc_scores))

# Print the best model for accuracy
print(f'Best Accuracy: {auc_scores[best_accuracy_index]:.4f} with Model: {classifiers[best_accuracy_index]}')

Best Accuracy: 0.8946 with Model: LightGBM

​​​​​​​