Ensemble Methods

Real-World Scenario

A Kaggle competition winner explains their solution: individual models score 0.89 AUC each. Stacking 4 diverse models (gradient boosting, random forest, logistic regression, neural network) with a logistic regression meta-learner scores 0.924 AUC — a 3.4 percentage point improvement. The ensemble works because the models make different types of errors on different parts of the feature space.

Voting Ensemble

from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.ensemble import (
    VotingClassifier, RandomForestClassifier, GradientBoostingClassifier
)
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score
import numpy as np

X, y = make_classification(n_samples=5000, n_features=20, n_informative=12,
                            n_redundant=4, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Individual models (pre-scaled for LR and SVM)
lr  = Pipeline([("scaler", StandardScaler()), ("clf", LogisticRegression(C=1.0, max_iter=1000))])
svm = Pipeline([("scaler", StandardScaler()), ("clf", SVC(probability=True, C=1.0))])
rf  = RandomForestClassifier(n_estimators=200, random_state=42, n_jobs=-1)
gb  = GradientBoostingClassifier(n_estimators=200, learning_rate=0.1, max_depth=4, random_state=42)

# Soft voting: averages predicted probabilities (better than hard voting for AUC)
soft_voting = VotingClassifier(
    estimators=[("lr", lr), ("svm", svm), ("rf", rf), ("gb", gb)],
    voting="soft",
    n_jobs=-1,
)

# Compare individual models vs ensemble
models = {"LR": lr, "SVM": svm, "RF": rf, "GB": gb, "Ensemble": soft_voting}
print(f"{'Model':<12} {'CV AUC':>10}")
print("-" * 24)
for name, model in models.items():
    scores = cross_val_score(model, X_train, y_train, cv=5, scoring="roc_auc", n_jobs=-1)
    print(f"{name:<12} {scores.mean():>10.4f} ± {scores.std():.4f}")

# Test set evaluation
soft_voting.fit(X_train, y_train)
test_auc = roc_auc_score(y_test, soft_voting.predict_proba(X_test)[:, 1])
print(f"\nTest AUC (soft voting): {test_auc:.4f}")

Stacking

from sklearn.datasets import make_classification
from sklearn.model_selection import (
    train_test_split, StratifiedKFold, cross_val_predict, cross_val_score
)
from sklearn.ensemble import (
    RandomForestClassifier, GradientBoostingClassifier, StackingClassifier
)
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score
import numpy as np

X, y = make_classification(n_samples=5000, n_features=20, n_informative=12, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Level 0: diverse base models
base_models = [
    ("rf",  RandomForestClassifier(n_estimators=200, random_state=42, n_jobs=-1)),
    ("gb",  GradientBoostingClassifier(n_estimators=200, learning_rate=0.1, random_state=42)),
    ("lr",  Pipeline([
        ("scaler", StandardScaler()),
        ("clf", LogisticRegression(C=1.0, max_iter=1000)),
    ])),
]

# Level 1: meta-learner trained on out-of-fold predictions
meta_learner = LogisticRegression(C=0.1)

stacking = StackingClassifier(
    estimators=base_models,
    final_estimator=meta_learner,
    cv=5,                      # inner CV to generate OOF predictions for meta-learner
    stack_method="predict_proba",
    passthrough=False,         # True: also pass original features to meta-learner
    n_jobs=-1,
)

cv_scores = cross_val_score(stacking, X_train, y_train, cv=5, scoring="roc_auc", n_jobs=1)
print(f"Stacking CV AUC: {cv_scores.mean():.4f} ± {cv_scores.std():.4f}")

stacking.fit(X_train, y_train)
test_auc = roc_auc_score(y_test, stacking.predict_proba(X_test)[:, 1])
print(f"Stacking Test AUC: {test_auc:.4f}")

Manual Stacking (OOF Approach)

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import StratifiedKFold, train_test_split, cross_val_score
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import roc_auc_score

X, y = make_classification(n_samples=5000, n_features=20, n_informative=12, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

BASE_MODELS = {
    "rf":  RandomForestClassifier(n_estimators=200, random_state=42, n_jobs=-1),
    "gb":  GradientBoostingClassifier(n_estimators=200, learning_rate=0.1, random_state=42),
    "mlp": MLPClassifier(hidden_layer_sizes=(128, 64), max_iter=200, random_state=42),
    "lr":  LogisticRegression(C=1.0, max_iter=1000),
}

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

# Step 1: Generate out-of-fold (OOF) predictions for training the meta-learner
oof_preds_train = np.zeros((len(X_train), len(BASE_MODELS)))
test_preds      = np.zeros((len(X_test),  len(BASE_MODELS)))

for i, (name, model) in enumerate(BASE_MODELS.items()):
    oof_fold = np.zeros(len(X_train))
    fold_test_preds = np.zeros((len(X_test), 5))

    for fold, (tr_idx, val_idx) in enumerate(cv.split(X_train, y_train)):
        X_tr, X_val = X_train[tr_idx], X_train[val_idx]
        y_tr, y_val = y_train[tr_idx], y_train[val_idx]

        model.fit(X_tr, y_tr)
        oof_fold[val_idx]        = model.predict_proba(X_val)[:, 1]
        fold_test_preds[:, fold] = model.predict_proba(X_test)[:, 1]

    oof_preds_train[:, i] = oof_fold
    test_preds[:, i]      = fold_test_preds.mean(axis=1)  # average across folds
    oof_auc = roc_auc_score(y_train, oof_fold)
    print(f"{name:5s}: OOF AUC = {oof_auc:.4f}")

# Step 2: Train meta-learner on OOF predictions
meta = LogisticRegression(C=0.1, max_iter=1000)
meta.fit(oof_preds_train, y_train)

# Step 3: Predict on test set using averaged base model test predictions
meta_test_probs = meta.predict_proba(test_preds)[:, 1]
test_auc = roc_auc_score(y_test, meta_test_probs)
print(f"\nMeta-learner Test AUC: {test_auc:.4f}")
print(f"Meta weights: {dict(zip(BASE_MODELS.keys(), meta.coef_[0].round(3)))}")

Blending (Weighted Average)

import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score
from scipy.optimize import minimize

X, y = make_classification(n_samples=5000, n_features=20, random_state=42)
X_train, X_val, X_test = np.split(X, [3000, 4000])
y_train, y_val, y_test = np.split(y, [3000, 4000])

# Train base models
models = {
    "rf": RandomForestClassifier(200, random_state=42).fit(X_train, y_train),
    "gb": GradientBoostingClassifier(200, random_state=42).fit(X_train, y_train),
    "lr": LogisticRegression(max_iter=1000).fit(X_train, y_train),
}

# Get predictions on validation set to find optimal weights
val_preds = np.column_stack([m.predict_proba(X_val)[:, 1] for m in models.values()])
test_preds = np.column_stack([m.predict_proba(X_test)[:, 1] for m in models.values()])

def neg_auc(weights: np.ndarray) -> float:
    weights = np.abs(weights) / np.abs(weights).sum()  # normalize
    blended = val_preds @ weights
    return -roc_auc_score(y_val, blended)

# Optimize blend weights
result = minimize(neg_auc, x0=np.ones(len(models)) / len(models), method="Nelder-Mead")
optimal_weights = np.abs(result.x) / np.abs(result.x).sum()

print("Optimal blend weights:")
for name, w in zip(models.keys(), optimal_weights):
    print(f"  {name}: {w:.3f}")

# Evaluate on test set
blended_test = test_preds @ optimal_weights
test_auc = roc_auc_score(y_test, blended_test)
print(f"\nBlended Test AUC: {test_auc:.4f}")

# Compare with equal-weight blend
equal_blend_auc = roc_auc_score(y_test, test_preds.mean(axis=1))
print(f"Equal-weight AUC: {equal_blend_auc:.4f}")