XGBoost and LightGBM
Master gradient boosting with XGBoost and LightGBM — the dominant algorithms for tabular ML competitions and production systems.
Real-World Scenario
A fintech company needs a credit scoring model that predicts default probability. The dataset has 500,000 customers, 150 features, many missing values, and mixed types. LightGBM handles missing values natively, trains in 90 seconds, achieves 0.84 AUC, and integrates seamlessly with sklearn pipelines for preprocessing and cross-validation.
XGBoost
# pip install xgboost
import xgboost as xgb
import numpy as np
import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.metrics import roc_auc_score
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
X, y = make_classification(n_samples=10_000, n_features=30, n_informative=15,
n_redundant=5, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
# Native XGBoost API — required for early stopping
dtrain = xgb.DMatrix(X_tr, label=y_tr)
dval = xgb.DMatrix(X_val, label=y_val)
dtest = xgb.DMatrix(X_test)
params = {
"objective": "binary:logistic",
"eval_metric": "auc",
"max_depth": 5,
"learning_rate": 0.05,
"subsample": 0.8, # row subsampling per tree
"colsample_bytree": 0.8, # feature subsampling per tree
"min_child_weight": 5, # min samples in leaf (regularization)
"gamma": 0.1, # min loss reduction to split a node
"lambda": 1.0, # L2 regularization on weights
"alpha": 0.1, # L1 regularization on weights
"seed": 42,
}
evals_result = {}
model = xgb.train(
params,
dtrain,
num_boost_round=2000,
evals=[(dtrain, "train"), (dval, "val")],
early_stopping_rounds=50, # stop if val AUC doesn't improve for 50 rounds
evals_result=evals_result,
verbose_eval=100,
)
best_round = model.best_ntree_limit
test_prob = model.predict(dtest, ntree_limit=best_round)
print(f"Best round: {best_round} Test AUC: {roc_auc_score(y_test, test_prob):.4f}")
# Feature importance
importance = model.get_score(importance_type="gain") # "weight", "gain", "cover"
top5 = sorted(importance.items(), key=lambda x: x[1], reverse=True)[:5]
print("Top 5 features by gain:")
for feat, score in top5:
print(f" {feat}: {score:.2f}")XGBoost with Sklearn API
import xgboost as xgb
from sklearn.model_selection import RandomizedSearchCV, cross_val_score, StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from scipy.stats import uniform, randint
# Sklearn-compatible XGBClassifier
xgb_clf = xgb.XGBClassifier(
n_estimators=500,
learning_rate=0.05,
max_depth=5,
subsample=0.8,
colsample_bytree=0.8,
min_child_weight=5,
gamma=0.1,
reg_lambda=1.0,
reg_alpha=0.1,
use_label_encoder=False,
eval_metric="auc",
early_stopping_rounds=50, # requires eval_set in fit()
n_jobs=-1,
random_state=42,
)
# Fit with validation set for early stopping
xgb_clf.fit(
X_tr, y_tr,
eval_set=[(X_val, y_val)],
verbose=False,
)
print(f"XGB best iteration: {xgb_clf.best_iteration}")
print(f"Test AUC: {roc_auc_score(y_test, xgb_clf.predict_proba(X_test)[:,1]):.4f}")
# Hyperparameter search
param_dist = {
"max_depth": randint(3, 8),
"learning_rate": uniform(0.01, 0.15),
"subsample": uniform(0.6, 0.4),
"colsample_bytree": uniform(0.5, 0.5),
"min_child_weight": randint(1, 10),
"gamma": uniform(0, 0.5),
}
search = RandomizedSearchCV(
xgb.XGBClassifier(n_estimators=200, eval_metric="auc", n_jobs=1, random_state=42),
param_dist, n_iter=20, cv=3, scoring="roc_auc", n_jobs=-1, random_state=42,
)
search.fit(X_train, y_train)
print(f"Best CV AUC: {search.best_score_:.4f}")
print(f"Best params: {search.best_params_}")LightGBM
# pip install lightgbm
import lightgbm as lgb
import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
X, y = make_classification(n_samples=10_000, n_features=30, n_informative=15, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)
# Native LightGBM API
dtrain = lgb.Dataset(X_tr, label=y_tr)
dval = lgb.Dataset(X_val, label=y_val, reference=dtrain)
params = {
"objective": "binary",
"metric": "auc",
"num_leaves": 63, # max leaves per tree (2^6 - 1)
"learning_rate": 0.05,
"feature_fraction": 0.8, # colsample per tree
"bagging_fraction": 0.8, # row subsample
"bagging_freq": 5, # apply bagging every 5 iterations
"min_child_samples": 20, # min data in leaf
"lambda_l1": 0.1,
"lambda_l2": 1.0,
"verbose": -1,
"seed": 42,
}
callbacks = [
lgb.early_stopping(stopping_rounds=50, verbose=False),
lgb.log_evaluation(period=100),
]
model = lgb.train(
params,
dtrain,
num_boost_round=2000,
valid_sets=[dtrain, dval],
valid_names=["train", "val"],
callbacks=callbacks,
)
test_prob = model.predict(X_test)
print(f"Best iteration: {model.best_iteration}")
print(f"Test AUC: {roc_auc_score(y_test, test_prob):.4f}")
# LightGBM sklearn API (drop-in replacement)
lgb_clf = lgb.LGBMClassifier(
n_estimators=500,
learning_rate=0.05,
num_leaves=63,
feature_fraction=0.8,
bagging_fraction=0.8,
bagging_freq=5,
verbose=-1,
random_state=42,
)
lgb_clf.fit(
X_tr, y_tr,
eval_set=[(X_val, y_val)],
callbacks=[lgb.early_stopping(50, verbose=False)],
)
print(f"LGB Test AUC: {roc_auc_score(y_test, lgb_clf.predict_proba(X_test)[:,1]):.4f}")Handling Categorical Features
import lightgbm as lgb
import pandas as pd
import numpy as np
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_auc_score
rng = np.random.default_rng(42)
n = 5000
X = pd.DataFrame({
"income": rng.lognormal(10, 1, n),
"age": rng.integers(18, 70, n),
"region": rng.choice(["north", "south", "east", "west"], n),
"job_type": rng.choice(["employed", "self_employed", "retired", "student"], n),
"score": rng.normal(600, 100, n),
})
y = (rng.random(n) > 0.7).astype(int)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# LightGBM handles categoricals natively — no encoding needed!
cat_features = ["region", "job_type"]
for col in cat_features:
X_train[col] = X_train[col].astype("category")
X_test[col] = X_test[col].astype("category")
model = lgb.LGBMClassifier(
n_estimators=200,
verbose=-1,
random_state=42,
)
model.fit(X_train, y_train)
print(f"With native categoricals AUC: {roc_auc_score(y_test, model.predict_proba(X_test)[:,1]):.4f}")
# Feature importance
fi = pd.Series(model.feature_importances_, index=X_train.columns)
print("\nFeature importance (split-based):")
print(fi.sort_values(ascending=False).to_string())Frequently Asked Questions
When should I use XGBoost vs LightGBM vs HistGradientBoosting?
All three are excellent gradient boosting implementations. LightGBM is typically 3-10x faster than XGBoost and uses less memory (leaf-wise tree growth vs level-wise). HistGradientBoosting is sklearn's native option with no extra install — good for quick experiments. XGBoost has the largest ecosystem, best GPU support, and accepts sparse matrices natively. For new projects: try HistGradientBoosting first, then LightGBM if you need more speed or flexibility.
What is early stopping and why does it prevent overfitting?
Early stopping monitors validation loss during training and stops when it stops improving. Without it, training for too many rounds overfits the training data. With it, you can set n_estimators very high (e.g., 10,000) and let early stopping find the optimal number — typically 100-500 rounds. Always use early stopping with gradient boosting.