Machine Learning Cross-Validation

Real-World Scenario

A data scientist trains a fraud detection model with a 95% accuracy on the test set. But the dataset has 1000 frauds out of 100,000 transactions. Their single test split happened to capture most frauds in training. Stratified 5-fold CV with the right metrics reveals the true picture: 72% recall on fraud. Without proper cross-validation, they would have shipped a broken model.

K-Fold Cross-Validation

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import KFold, StratifiedKFold, cross_validate
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
import numpy as np
import pandas as pd

X, y = load_breast_cancer(return_X_y=True)

pipe = Pipeline([
    ("scaler", StandardScaler()),
    ("model",  RandomForestClassifier(n_estimators=100, random_state=42)),
])

# StratifiedKFold preserves class proportions — always use for classification
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

results = cross_validate(
    pipe, X, y,
    cv=cv,
    scoring=["accuracy", "f1", "roc_auc"],
    return_train_score=True,   # detect overfitting: train >> test = overfit
    n_jobs=-1,
)

for metric in ["accuracy", "f1", "roc_auc"]:
    train = results[f"train_{metric}"]
    test  = results[f"test_{metric}"]
    print(f"{metric:10s}: train={train.mean():.4f}±{train.std():.4f}  "
          f"test={test.mean():.4f}±{test.std():.4f}")

Leave-One-Out and Other Strategies

from sklearn.model_selection import LeaveOneOut, ShuffleSplit, RepeatedStratifiedKFold
from sklearn.datasets import load_iris
from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import cross_val_score
import numpy as np

X, y = load_iris(return_X_y=True)
model = KNeighborsClassifier(n_neighbors=5)

# Leave-One-Out — use every sample as a test set once
# Very high variance but useful for tiny datasets (< 50 samples)
loo = LeaveOneOut()
loo_scores = cross_val_score(model, X, y, cv=loo, scoring="accuracy")
print(f"LOO Accuracy: {loo_scores.mean():.4f}")

# ShuffleSplit — random train/test splits, repeated
ss = ShuffleSplit(n_splits=10, test_size=0.2, random_state=42)
ss_scores = cross_val_score(model, X, y, cv=ss, scoring="accuracy")
print(f"ShuffleSplit Accuracy: {ss_scores.mean():.4f} ± {ss_scores.std():.4f}")

# Repeated StratifiedKFold — more stable estimate via multiple repetitions
rskf = RepeatedStratifiedKFold(n_splits=5, n_repeats=10, random_state=42)
rskf_scores = cross_val_score(model, X, y, cv=rskf, scoring="accuracy", n_jobs=-1)
print(f"Repeated 5-fold Accuracy: {rskf_scores.mean():.4f} ± {rskf_scores.std():.4f}")

Time-Series Cross-Validation

Standard k-fold leaks future data into the past — a fatal error for time series. Use walk-forward (expanding or sliding window) splits instead.

import numpy as np
import pandas as pd
from sklearn.model_selection import TimeSeriesSplit
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import root_mean_squared_error

rng = np.random.default_rng(42)
n = 500

# Simulated time series: sales with trend + seasonality + noise
dates = pd.date_range("2022-01-01", periods=n, freq="D")
trend = np.linspace(100, 200, n)
seasonality = 20 * np.sin(np.arange(n) * 2 * np.pi / 30)
noise = rng.normal(0, 5, n)
sales = trend + seasonality + noise

# Create lag features (previous days as predictors)
df = pd.DataFrame({"sales": sales}, index=dates)
for lag in [1, 7, 14, 30]:
    df[f"lag_{lag}"] = df["sales"].shift(lag)
df = df.dropna()

X = df.drop("sales", axis=1).values
y = df["sales"].values

# TimeSeriesSplit: each fold uses only past data for training
tscv = TimeSeriesSplit(n_splits=5, gap=0)

fold_rmses = []
for fold, (train_idx, test_idx) in enumerate(tscv.split(X)):
    X_train, X_test = X[train_idx], X[test_idx]
    y_train, y_test = y[train_idx], y[test_idx]

    model = GradientBoostingRegressor(n_estimators=100, random_state=42)
    model.fit(X_train, y_train)
    rmse = root_mean_squared_error(y_test, model.predict(X_test))
    fold_rmses.append(rmse)
    print(f"Fold {fold+1}: train_size={len(train_idx):4d}  test_size={len(test_idx):3d}  RMSE={rmse:.2f}")

print(f"\nMean RMSE: {np.mean(fold_rmses):.2f} ± {np.std(fold_rmses):.2f}")

Nested Cross-Validation

Nested CV provides an unbiased performance estimate when both tuning and evaluation happen on the same dataset.

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import (
    StratifiedKFold, RandomizedSearchCV, cross_val_score
)
from scipy.stats import uniform, randint
import numpy as np

X, y = load_breast_cancer(return_X_y=True)

# Inner loop: hyperparameter tuning
inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=1)

# Outer loop: performance estimation
outer_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

param_dist = {
    "n_estimators":  randint(50, 300),
    "max_depth":     randint(2, 6),
    "learning_rate": uniform(0.01, 0.2),
}

# The inner RandomizedSearchCV is the estimator in the outer CV
tuned_model = RandomizedSearchCV(
    GradientBoostingClassifier(random_state=42),
    param_dist, n_iter=20, cv=inner_cv,
    scoring="roc_auc", n_jobs=-1, random_state=42,
)

# Nested CV: outer loop runs 5 times, each time inner loop tunes on that fold's train set
nested_auc = cross_val_score(tuned_model, X, y, cv=outer_cv, scoring="roc_auc", n_jobs=-1)

print(f"Nested CV AUC: {nested_auc.mean():.4f} ± {nested_auc.std():.4f}")
print(f"Per fold:      {nested_auc.round(4)}")

# Compare with non-nested (optimistically biased)
non_nested = RandomizedSearchCV(
    GradientBoostingClassifier(random_state=42),
    param_dist, n_iter=20, cv=inner_cv,
    scoring="roc_auc", n_jobs=-1, random_state=42,
)
non_nested.fit(X, y)
print(f"\nNon-nested best CV AUC: {non_nested.best_score_:.4f}  "
      f"(optimistic bias: {non_nested.best_score_ - nested_auc.mean():.4f})")

Learning Curves — Diagnose with Data

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import learning_curve, StratifiedKFold
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
model = GradientBoostingClassifier(n_estimators=100, random_state=42)
cv    = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

train_sizes, train_scores, val_scores = learning_curve(
    model, X, y,
    train_sizes=np.linspace(0.1, 1.0, 10),
    cv=cv,
    scoring="roc_auc",
    n_jobs=-1,
)

print("Learning curve (AUC):")
print(f"{'Train size':>12} {'Train AUC':>12} {'Val AUC':>12} {'Gap':>8}")
for size, tr, val in zip(train_sizes, train_scores.mean(axis=1), val_scores.mean(axis=1)):
    gap = tr - val
    status = "⚠ overfit" if gap > 0.05 else "✓"
    print(f"{size:12d} {tr:12.4f} {val:12.4f} {gap:8.4f} {status}")

# Interpretation:
# - High gap between train and val → overfitting (add data or regularize)
# - Both low → underfitting (more complex model)
# - Val plateaus → more data won't help (hit model capacity ceiling)