Scikit-Learn Hyperparameter Tuning

Real-World Scenario

A data scientist has trained a GradientBoostingClassifier but isn’t satisfied with the performance. They know from experience that learning_rate, n_estimators, and max_depth all interact. Exhaustive grid search over these three parameters with 5-fold CV would require 3×4×3×5=180 model fits — 45 minutes. RandomizedSearchCV with 50 iterations finds nearly the same optimum in 12 minutes.

Grid Search

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV, train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
import pandas as pd

X, y = load_breast_cancer(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42, stratify=y)

# Wrap in pipeline to include preprocessing
pipe = Pipeline([
    ("scaler", StandardScaler()),
    ("rf",     RandomForestClassifier(random_state=42, n_jobs=-1)),
])

# Small grid — only use for < 50 total combinations
param_grid = {
    "rf__n_estimators": [100, 200, 300],
    "rf__max_depth":    [None, 5, 10],
    "rf__min_samples_leaf": [1, 5],
}
# Total fits: 3 × 3 × 2 × 5-fold = 90

grid_search = GridSearchCV(
    pipe, param_grid,
    cv=5,                    # 5-fold stratified CV
    scoring="roc_auc",       # optimize for AUC
    n_jobs=-1,               # use all cores
    verbose=1,
    return_train_score=True, # detect overfitting in CV
)
grid_search.fit(X_train, y_train)

print(f"Best AUC (CV):   {grid_search.best_score_:.4f}")
print(f"Best params:     {grid_search.best_params_}")
print(f"Test AUC:        {grid_search.score(X_test, y_test):.4f}")

# Inspect all results
results = pd.DataFrame(grid_search.cv_results_)
top5 = results.nlargest(5, "mean_test_score")[
    ["params", "mean_test_score", "std_test_score", "mean_train_score"]
]
print(top5.to_string(index=False))

Randomized Search

from sklearn.datasets import fetch_california_housing
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.model_selection import RandomizedSearchCV, train_test_split
from sklearn.metrics import root_mean_squared_error
from scipy.stats import uniform, randint
import numpy as np

X, y = fetch_california_housing(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Continuous distributions — more efficient than discrete grids
param_dist = {
    "n_estimators":     randint(100, 600),
    "max_depth":        randint(3, 8),
    "min_samples_leaf": randint(1, 20),
    "learning_rate":    uniform(0.01, 0.19),   # uniform [0.01, 0.20]
    "subsample":        uniform(0.6, 0.4),     # uniform [0.6, 1.0]
    "max_features":     uniform(0.5, 0.5),     # uniform [0.5, 1.0]
}

model = GradientBoostingRegressor(random_state=42)
random_search = RandomizedSearchCV(
    model, param_dist,
    n_iter=50,        # try 50 random combinations vs 3×5×19×18×5×5=25650 grid
    cv=5,
    scoring="neg_root_mean_squared_error",
    n_jobs=-1,
    random_state=42,
    verbose=1,
)
random_search.fit(X_train, y_train)

best_model = random_search.best_estimator_
rmse = root_mean_squared_error(y_test, best_model.predict(X_test))

print(f"Best CV RMSE:  {-random_search.best_score_:.4f}")
print(f"Test RMSE:     {rmse:.4f}")
print(f"Best params:\n{random_search.best_params_}")

Bayesian Optimization with Optuna

Optuna learns from each trial to sample more promising regions of the parameter space. It typically finds better results than random search in fewer iterations.

# pip install optuna
import optuna
import numpy as np
from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

optuna.logging.set_verbosity(optuna.logging.WARNING)  # suppress per-trial output

X, y = load_breast_cancer(return_X_y=True)
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

def objective(trial: optuna.Trial) -> float:
    """Optuna objective function — returns the metric to maximize."""
    params = {
        "n_estimators":     trial.suggest_int("n_estimators",  50, 500),
        "max_depth":        trial.suggest_int("max_depth",     2, 8),
        "learning_rate":    trial.suggest_float("learning_rate",  0.01, 0.3, log=True),
        "min_samples_leaf": trial.suggest_int("min_samples_leaf", 1, 20),
        "subsample":        trial.suggest_float("subsample",   0.5, 1.0),
        "max_features":     trial.suggest_float("max_features", 0.3, 1.0),
    }

    model = GradientBoostingClassifier(**params, random_state=42)
    scores = cross_val_score(model, X, y, cv=cv, scoring="roc_auc", n_jobs=-1)
    return scores.mean()

# Run Bayesian optimization — 100 trials
study = optuna.create_study(
    direction="maximize",
    sampler=optuna.samplers.TPESampler(seed=42),  # Tree-structured Parzen Estimator
    pruner=optuna.pruners.MedianPruner(),          # early-stop unpromising trials
)
study.optimize(objective, n_trials=100, n_jobs=-1, show_progress_bar=True)

print(f"\nBest AUC:    {study.best_value:.4f}")
print(f"Best params: {study.best_params}")

# Importance of each parameter
importance = optuna.importance.get_param_importances(study)
print("\nParameter importance:")
for param, imp in sorted(importance.items(), key=lambda x: -x[1]):
    print(f"  {param:25s}: {imp:.3f}")

# Train final model with best params
best_model = GradientBoostingClassifier(**study.best_params, random_state=42)

Nested Cross-Validation — Unbiased Performance Estimate

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import (
    StratifiedKFold, RandomizedSearchCV, cross_val_score
)
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from scipy.stats import uniform, randint
import numpy as np

X, y = load_breast_cancer(return_X_y=True)

# Inner CV: tune hyperparameters
inner_cv = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)

# Outer CV: estimate generalization performance
outer_cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)

param_dist = {
    "n_estimators":  randint(50, 300),
    "max_depth":     randint(2, 6),
    "learning_rate": uniform(0.01, 0.2),
}

estimator = RandomizedSearchCV(
    GradientBoostingClassifier(random_state=42),
    param_dist,
    n_iter=20,
    cv=inner_cv,
    scoring="roc_auc",
    n_jobs=-1,
    random_state=42,
)

# Nested CV — outer loop estimates performance, inner loop tunes params
nested_scores = cross_val_score(
    estimator, X, y,
    cv=outer_cv,
    scoring="roc_auc",
    n_jobs=-1,
)

print(f"Nested CV AUC: {nested_scores.mean():.4f} ± {nested_scores.std():.4f}")
print(f"Individual folds: {nested_scores.round(4)}")

Early Stopping to Speed Up Tuning

from sklearn.datasets import load_breast_cancer
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.model_selection import train_test_split, cross_val_score
import numpy as np

X, y = load_breast_cancer(return_X_y=True)
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)

# HistGradientBoosting supports native early stopping
model = HistGradientBoostingClassifier(
    max_iter=1000,             # maximum trees
    early_stopping=True,       # stop if validation score doesn't improve
    validation_fraction=0.1,   # use 10% of training data for early stopping
    n_iter_no_change=20,       # stop after 20 rounds without improvement
    random_state=42,
)

model.fit(X_train, y_train)
print(f"Stopped at iteration: {model.n_iter_}")
print(f"Validation AUC: {model.score(X_val, y_val):.4f}")