Handling Imbalanced Data

Real-World Scenario

A fraud detection model trained on 1 million transactions where 0.1% are fraud. Without handling imbalance, the model learns to always predict “not fraud” and scores 99.9% accuracy. The real goal: catch 90% of fraud (recall) while keeping false positive rate under 1% (precision). This requires resampling, cost-sensitive training, and threshold tuning.

Diagnosing the Problem

import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (
    classification_report, confusion_matrix,
    roc_auc_score, average_precision_score
)

# Create imbalanced dataset: 98% class 0, 2% class 1
X, y = make_classification(
    n_samples=10_000,
    n_features=20,
    n_informative=10,
    weights=[0.98, 0.02],   # 98/2 split
    random_state=42,
)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
                                                      stratify=y, random_state=42)

print(f"Training positives: {y_train.sum()} / {len(y_train)}  ({y_train.mean():.1%})")
print(f"Test positives:     {y_test.sum()} / {len(y_test)}  ({y_test.mean():.1%})")

# Naive model — ignores imbalance
naive = LogisticRegression(max_iter=1000)
naive.fit(X_train, y_train)

print("\nNaive model (no imbalance handling):")
y_pred = naive.predict(X_test)
print(classification_report(y_test, y_pred, target_names=["normal", "fraud"]))
print(f"AUC-ROC:  {roc_auc_score(y_test, naive.predict_proba(X_test)[:,1]):.4f}")
print(f"Avg Prec: {average_precision_score(y_test, naive.predict_proba(X_test)[:,1]):.4f}")

Class Weights

from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier, HistGradientBoostingClassifier
from sklearn.utils.class_weight import compute_class_weight
from sklearn.metrics import classification_report, roc_auc_score, average_precision_score
import numpy as np

# class_weight="balanced" automatically sets weight inversely proportional to frequency
lr_balanced = LogisticRegression(class_weight="balanced", max_iter=1000)
lr_balanced.fit(X_train, y_train)
y_pred = lr_balanced.predict(X_test)

print("Balanced logistic regression:")
print(classification_report(y_test, y_pred, target_names=["normal", "fraud"]))
print(f"AUC-ROC:  {roc_auc_score(y_test, lr_balanced.predict_proba(X_test)[:,1]):.4f}")
print(f"Avg Prec: {average_precision_score(y_test, lr_balanced.predict_proba(X_test)[:,1]):.4f}")

# Manual weight computation (same result, but you can tune it)
weights = compute_class_weight("balanced", classes=np.array([0, 1]), y=y_train)
class_weight_dict = {0: weights[0], 1: weights[1]}
print(f"\nAuto-computed weights: class 0 = {weights[0]:.2f}, class 1 = {weights[1]:.2f}")

# For HistGradientBoosting, use sample_weight
from sklearn.utils import class_weight as cw_module
sample_weights = np.where(y_train == 1, weights[1], weights[0])

hgb = HistGradientBoostingClassifier(max_iter=200, random_state=42)
hgb.fit(X_train, y_train, sample_weight=sample_weights)
print(f"\nHGB with sample weights — AUC: {roc_auc_score(y_test, hgb.predict_proba(X_test)[:,1]):.4f}")

Resampling with imbalanced-learn

# pip install imbalanced-learn
from imblearn.over_sampling import SMOTE, ADASYN
from imblearn.under_sampling import RandomUnderSampler, TomekLinks
from imblearn.combine import SMOTETomek
from imblearn.pipeline import Pipeline as ImbPipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import roc_auc_score, average_precision_score

# SMOTE: Synthetic Minority Over-sampling Technique
# Creates synthetic minority samples by interpolating between existing ones
smote = SMOTE(sampling_strategy=0.1, random_state=42)  # minority:majority = 1:10
X_res, y_res = smote.fit_resample(X_train, y_train)
print(f"After SMOTE: {y_res.sum()} positives / {len(y_res)} total ({y_res.mean():.1%})")

# Train on resampled data
lr = LogisticRegression(max_iter=1000)
lr.fit(X_res, y_res)
print(f"SMOTE + LR — AUC: {roc_auc_score(y_test, lr.predict_proba(X_test)[:,1]):.4f}")

# Combine oversampling + Tomek Links cleaning (removes borderline samples)
smt = SMOTETomek(sampling_strategy=0.1, random_state=42)
X_comb, y_comb = smt.fit_resample(X_train, y_train)
lr.fit(X_comb, y_comb)
print(f"SMOTETomek + LR — AUC: {roc_auc_score(y_test, lr.predict_proba(X_test)[:,1]):.4f}")

# Best practice: resampling inside pipeline to prevent leakage
imb_pipeline = ImbPipeline([
    ("smote",  SMOTE(sampling_strategy=0.1, random_state=42)),
    ("scaler", StandardScaler()),
    ("model",  LogisticRegression(max_iter=1000)),
])
imb_pipeline.fit(X_train, y_train)
print(f"Imb-Pipeline AUC: {roc_auc_score(y_test, imb_pipeline.predict_proba(X_test)[:,1]):.4f}")

Threshold Tuning

import numpy as np
from sklearn.metrics import (
    precision_recall_curve, f1_score, precision_score, recall_score
)
import matplotlib  # noqa — for documentation; use matplotlib.pyplot to plot

# Get predicted probabilities (don't use default 0.5 threshold)
y_probs = hgb.predict_proba(X_test)[:, 1]

# Find the threshold that maximizes F1 score
precisions, recalls, thresholds = precision_recall_curve(y_test, y_probs)

f1_scores = 2 * precisions[:-1] * recalls[:-1] / (precisions[:-1] + recalls[:-1] + 1e-9)
best_idx   = f1_scores.argmax()
best_threshold = thresholds[best_idx]

print(f"Default threshold (0.5):")
y_pred_default = (y_probs >= 0.5).astype(int)
print(f"  Precision: {precision_score(y_test, y_pred_default):.4f}")
print(f"  Recall:    {recall_score(y_test, y_pred_default):.4f}")
print(f"  F1:        {f1_score(y_test, y_pred_default):.4f}")

print(f"\nOptimal threshold ({best_threshold:.3f}):")
y_pred_optimal = (y_probs >= best_threshold).astype(int)
print(f"  Precision: {precision_score(y_test, y_pred_optimal):.4f}")
print(f"  Recall:    {recall_score(y_test, y_pred_optimal):.4f}")
print(f"  F1:        {f1_score(y_test, y_pred_optimal):.4f}")

# Business-driven threshold: require recall >= 0.90
recall_90_mask = recalls[:-1] >= 0.90
if recall_90_mask.any():
    # Among thresholds with recall >= 0.90, maximize precision
    best_prec_at_90recall = precisions[:-1][recall_90_mask].max()
    idx_90 = np.where(recall_90_mask & (precisions[:-1] == best_prec_at_90recall))[0][0]
    threshold_90 = thresholds[idx_90]
    y_pred_90 = (y_probs >= threshold_90).astype(int)
    print(f"\nThreshold for recall ≥ 90% ({threshold_90:.3f}):")
    print(f"  Precision: {precision_score(y_test, y_pred_90):.4f}")
    print(f"  Recall:    {recall_score(y_test, y_pred_90):.4f}")

Evaluation Metrics for Imbalance

from sklearn.metrics import (
    roc_auc_score, average_precision_score,
    balanced_accuracy_score, matthews_corrcoef,
    confusion_matrix,
)
import numpy as np

y_probs = hgb.predict_proba(X_test)[:, 1]
y_pred  = (y_probs >= best_threshold).astype(int)

tn, fp, fn, tp = confusion_matrix(y_test, y_pred).ravel()

print("Evaluation metrics for imbalanced classification:")
print(f"  AUC-ROC:            {roc_auc_score(y_test, y_probs):.4f}  (threshold-independent)")
print(f"  Avg Precision (AP): {average_precision_score(y_test, y_probs):.4f}  (area under PR curve)")
print(f"  Balanced Accuracy:  {balanced_accuracy_score(y_test, y_pred):.4f}  (mean recall per class)")
print(f"  MCC:                {matthews_corrcoef(y_test, y_pred):.4f}  (robust to imbalance)")
print(f"\nConfusion matrix:")
print(f"  True Positives:  {tp:4d}  (fraud caught)")
print(f"  False Positives: {fp:4d}  (false alarms)")
print(f"  False Negatives: {fn:4d}  (missed fraud)")
print(f"  True Negatives:  {tn:4d}  (correct clear)")
print(f"\n  Fraud catch rate (recall): {tp/(tp+fn):.1%}")
print(f"  False alarm rate:          {fp/(fp+tn):.1%}")

# Which metrics matter depends on the business cost of each error type
COST_FN = 500   # cost of missing one fraud
COST_FP = 10    # cost of false alarm (manual review)
business_cost = fn * COST_FN + fp * COST_FP
print(f"\nBusiness cost: ${business_cost:,.0f}  (FN=${fn*COST_FN:,.0f}, FP=${fp*COST_FP:,.0f})")