Supervised Learning Algorithms

Real-World Scenario

A bank needs to predict loan defaults. They evaluate 5 algorithms to find the best trade-off between predictive power, interpretability for regulators, and training speed. The comparison reveals that HistGradientBoosting gives the best AUC, but logistic regression is chosen for the core model because regulators require feature-weight explainability.

Linear Models

from sklearn.datasets import make_classification, make_regression
from sklearn.linear_model import (
    LogisticRegression, Ridge, Lasso, ElasticNet, SGDClassifier
)
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import classification_report, roc_auc_score
import numpy as np

# Classification dataset
X, y = make_classification(n_samples=5000, n_features=20, n_informative=10,
                            n_redundant=5, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Linear models require feature scaling
scaler  = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s  = scaler.transform(X_test)

# Logistic Regression — interpretable, fast, strong baseline
lr = LogisticRegression(
    C=1.0,           # inverse of regularization strength — smaller = more regularization
    penalty="l2",    # L2 ridge regularization (use "l1" for feature selection)
    max_iter=1000,
    solver="lbfgs",
    random_state=42,
)
lr.fit(X_train_s, y_train)
lr_auc = roc_auc_score(y_test, lr.predict_proba(X_test_s)[:, 1])
print(f"Logistic Regression AUC: {lr_auc:.4f}")

# Feature importance from coefficients
feature_importance = np.abs(lr.coef_[0])
top_features = np.argsort(feature_importance)[::-1][:5]
print(f"Top 5 features by |coefficient|: {top_features}")

# SGD Classifier — same math, scales to millions of samples
sgd = SGDClassifier(loss="log_loss", penalty="l2", alpha=1e-4,
                    max_iter=100, random_state=42)
sgd.fit(X_train_s, y_train)
sgd_auc = roc_auc_score(y_test, sgd.predict_proba(X_test_s)[:, 1])
print(f"SGD Classifier AUC:      {sgd_auc:.4f}")

Decision Trees and Ensembles

from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import (
    RandomForestClassifier,
    GradientBoostingClassifier,
    HistGradientBoostingClassifier,
    AdaBoostClassifier,
    ExtraTreesClassifier,
)
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import roc_auc_score
import numpy as np

X, y = make_classification(n_samples=5000, n_features=20, n_informative=10,
                            random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

algorithms = {
    "Decision Tree": DecisionTreeClassifier(max_depth=5, random_state=42),
    "Random Forest": RandomForestClassifier(n_estimators=200, max_features="sqrt",
                                             random_state=42, n_jobs=-1),
    "Extra Trees":   ExtraTreesClassifier(n_estimators=200, random_state=42, n_jobs=-1),
    "Gradient Boosting": GradientBoostingClassifier(n_estimators=200, learning_rate=0.1,
                                                     max_depth=4, random_state=42),
    "HistGradientBoosting": HistGradientBoostingClassifier(max_iter=200, learning_rate=0.1,
                                                            max_depth=4, random_state=42),
}

print(f"{'Algorithm':<25} {'AUC':>8} {'CV AUC':>10}")
print("-" * 45)
for name, clf in algorithms.items():
    clf.fit(X_train, y_train)
    test_auc = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
    cv_auc   = cross_val_score(clf, X, y, cv=5, scoring="roc_auc", n_jobs=-1).mean()
    print(f"{name:<25} {test_auc:>8.4f} {cv_auc:>10.4f}")

How Gradient Boosting Works

import numpy as np
from sklearn.tree import DecisionTreeRegressor

class SimpleGradientBoosting:
    """Minimal gradient boosting for binary classification — shows the core algorithm."""

    def __init__(self, n_estimators=50, learning_rate=0.1, max_depth=3):
        self.n_estimators  = n_estimators
        self.learning_rate = learning_rate
        self.max_depth     = max_depth
        self.trees         = []
        self.F0            = 0.0   # initial prediction (log-odds of mean)

    def _sigmoid(self, x):
        return 1 / (1 + np.exp(-x))

    def fit(self, X: np.ndarray, y: np.ndarray) -> "SimpleGradientBoosting":
        # Initial prediction: log-odds of the base rate
        p = y.mean()
        self.F0 = np.log(p / (1 - p))

        F = np.full(len(y), self.F0)   # running prediction (log-odds space)

        for _ in range(self.n_estimators):
            # Gradient of log-loss = residuals (actual - predicted probability)
            p_hat    = self._sigmoid(F)
            residuals = y - p_hat   # pseudo-residuals

            # Fit a regression tree to the residuals
            tree = DecisionTreeRegressor(max_depth=self.max_depth)
            tree.fit(X, residuals)

            # Update running predictions
            F += self.learning_rate * tree.predict(X)
            self.trees.append(tree)

        return self

    def predict_proba(self, X: np.ndarray) -> np.ndarray:
        F = np.full(len(X), self.F0)
        for tree in self.trees:
            F += self.learning_rate * tree.predict(X)
        probs = self._sigmoid(F)
        return np.column_stack([1 - probs, probs])

    def predict(self, X: np.ndarray) -> np.ndarray:
        return (self.predict_proba(X)[:, 1] >= 0.5).astype(int)


# Verify it works
from sklearn.datasets import make_classification
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

X, y = make_classification(n_samples=1000, n_features=10, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

custom_gb = SimpleGradientBoosting(n_estimators=100, learning_rate=0.1, max_depth=3)
custom_gb.fit(X_train, y_train)
auc = roc_auc_score(y_test, custom_gb.predict_proba(X_test)[:, 1])
print(f"Custom GradBoost AUC: {auc:.4f}")

Support Vector Machines

from sklearn.svm import SVC, SVR
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import roc_auc_score
import numpy as np

X, y = make_classification(n_samples=2000, n_features=20, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# SVMs require feature scaling
scaler    = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s  = scaler.transform(X_test)

# RBF kernel SVM — the standard choice for non-linear data
svm = SVC(
    C=1.0,             # regularization — smaller C = wider margin, more misclassification
    kernel="rbf",      # radial basis function kernel
    gamma="scale",     # kernel coefficient — "scale" = 1/(n_features * X.var())
    probability=True,  # enables predict_proba (slower training)
    random_state=42,
)
svm.fit(X_train_s, y_train)
svm_auc = roc_auc_score(y_test, svm.predict_proba(X_test_s)[:, 1])
print(f"SVM RBF AUC: {svm_auc:.4f}")

# Tune C and gamma
param_grid = {"C": [0.1, 1, 10], "gamma": ["scale", "auto", 0.01]}
search = GridSearchCV(
    SVC(kernel="rbf", probability=True),
    param_grid, cv=3, scoring="roc_auc", n_jobs=-1
)
search.fit(X_train_s, y_train)
print(f"Best params: {search.best_params_}")
print(f"Best CV AUC: {search.best_score_:.4f}")

# When NOT to use SVM
# - n_samples > 100,000: too slow (O(n²) to O(n³))
# - You need feature importances: SVMs don't provide them natively
# - Multiclass with many classes: use GBM or RF instead

K-Nearest Neighbors

from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import make_classification
from sklearn.model_selection import cross_val_score
import numpy as np

X, y = make_classification(n_samples=2000, n_features=20, random_state=42)

scaler = StandardScaler()
X_s    = scaler.fit_transform(X)

# Find optimal K using cross-validation
k_values = [1, 3, 5, 7, 11, 21, 51]
for k in k_values:
    knn = KNeighborsClassifier(n_neighbors=k, metric="minkowski", p=2, n_jobs=-1)
    scores = cross_val_score(knn, X_s, y, cv=5, scoring="roc_auc")
    print(f"K={k:3d}: AUC={scores.mean():.4f} ± {scores.std():.4f}")

# KNN weaknesses:
# - Slow inference: O(n) per prediction for brute force
# - Curse of dimensionality: distance loses meaning in high-d space
# - No feature importance
# Use when: small dataset, non-linear decision boundary, interpretability needed