#!/usr/bin/env python3
"""Rank classification algorithms for a dataset and its constraints, with starting settings and cautions.
Traits come from a CSV (--csv data.csv --target y) or flags; constraints always come from flags:
data rows, features, share of categorical / high-cardinality columns, missing share,
minority-class share, number of classes, free-text columns
needs --interpretable (coefficients / shape functions a regulator can read),
--monotone (monotonic constraints), --calibrated (probabilities are used),
--latency-ms N (per-row budget at serving time), --tiny-model (edge / SQL deploy)
Candidates: logistic regression (L2 / L1), linear SVM, Gaussian/multinomial naive Bayes,
k-nearest neighbours, RBF SVM, decision tree, random forest, extra trees, HistGradientBoosting,
LightGBM, XGBoost, CatBoost, explainable boosting machine (EBM / GAM), MLP.
Each gets a score from fit-to-data rules; the top 3-4 are the bake-off shortlist.
Usage:
python select_classifier.py --csv data.csv --target churned --drop customer_id
python select_classifier.py --rows 2000000 --features 300 --cat-share 0.4 --minority 0.002 --latency-ms 5
python select_classifier.py --csv data.csv --target y --interpretable --monotone --json
Exit codes: 0 ranking printed, 2 usage or read error.
"""
import argparse
import csv
import json
import math
import sys
from collections import Counter
MISSING = {"", "na", "n/a", "nan", "null", "none", "?"}
def traits_from_csv(path, target, drop):
with open(path, newline="", encoding="utf-8-sig") as fh:
reader = csv.DictReader(fh)
header = reader.fieldnames or []
rows = list(reader)
if target not in header:
raise ValueError(f"target {target!r} not found")
feats = [c for c in header if c != target and c not in drop]
cat = hc = text = 0
miss = 0
for c in feats:
vals = [(r.get(c) or "").strip() for r in rows]
present = [v for v in vals if v.lower() not in MISSING]
miss += len(vals) - len(present)
nums = 0
for v in present:
try:
float(v)
nums += 1
except ValueError:
pass
if present and nums / len(present) < 0.95:
k = len(set(present))
mean_len = sum(len(v) for v in present) / len(present)
if mean_len > 30 and k > 0.5 * len(present):
text += 1
elif k > 50:
hc += 1
cat += 1
else:
cat += 1
y = Counter((r.get(target) or "").strip() for r in rows if (r.get(target) or "").strip())
n = sum(y.values())
return {"rows": len(rows), "features": len(feats), "cat_share": cat / max(1, len(feats)),
"hc_share": hc / max(1, len(feats)), "missing_share": miss / max(1, len(rows) * len(feats)),
"minority": min(y.values()) / n if n else 0.5, "classes": len(y), "text_cols": text}
def score_models(t, need):
n, p = t["rows"], t["features"]
small, big = n < 2_000, n > 1_000_000
wide = p > 0.5 * n or p > 5_000
cats, hc, miss = t["cat_share"], t["hc_share"], t["missing_share"]
rare = t["minority"] < 0.05
out = []
def add(name, score, why, start, caution):
out.append({"model": name, "score": round(score, 1), "why": why, "start": start, "caution": caution})
s = 6 + (2 if small else 0) + (2 if wide else 0) + (3 if need["interpretable"] else 0) + (1 if need["calibrated"] else 0) \
+ (2 if need["tiny_model"] else 0) + (1 if need["latency_ms"] and need["latency_ms"] < 1 else 0) - (1 if big and not wide else 0)
add("Logistic regression (L2)", s, "strong, fast, calibrated-ish baseline; coefficients are explainable; handles wide sparse data",
"LogisticRegression(C=1.0, max_iter=5000) on scaled numerics + one-hot; tune C on a log grid",
"needs scaling, explicit interactions/splines for non-linear effects; class_weight distorts probabilities")
add("Logistic regression (L1 / elastic net)", s - 0.5 + (1.5 if wide else 0),
"sparse, self-selecting model for many features", "LogisticRegression(penalty='elasticnet', solver='saga', l1_ratio=0.5, C=0.1)",
"unstable choice among correlated features; slower solver")
gb = 8 + (1 if cats > 0.2 else 0) + (1 if miss > 0.05 else 0) - (3 if small else 0) - (2 if wide else 0) \
- (2 if need["interpretable"] else 0) + (0.5 if need["monotone"] else 0)
add("HistGradientBoosting (scikit-learn)", gb, "best default for tabular data in pure scikit-learn; native NaN and categoricals",
"HistGradientBoostingClassifier(learning_rate=0.05, max_leaf_nodes=31, early_stopping=True)",
"categoricals < 256 levels; monotonic_cst for constraints; tune min_samples_leaf for small data")
add("LightGBM", gb + (1 if big else 0) + (0.5 if cats > 0.2 else 0),
"fastest GBM at scale; native categoricals; strong accuracy", "LGBMClassifier(n_estimators=5000, learning_rate=0.03, num_leaves=31, "
"min_child_samples=50, subsample=0.8, subsample_freq=1, colsample_bytree=0.8) + early stopping",
"overfits small data (raise min_child_samples); see xgboost-lightgbm-tuner")
add("XGBoost", gb + (0.5 if big else 0), "robust GBM, excellent with tuned regularisation; GPU support",
"XGBClassifier(tree_method='hist', n_estimators=5000, learning_rate=0.03, max_depth=6, min_child_weight=5, "
"subsample=0.8, colsample_bytree=0.8, early_stopping_rounds=200)", "enable_categorical needs pandas category dtype")
add("CatBoost", gb + (2 if cats > 0.3 else 0) + (1.5 if hc > 0.05 else 0) + (0.5 if small else 0),
"strongest out of the box with many / high-cardinality categoricals; ordered boosting resists overfitting",
"CatBoostClassifier(iterations=5000, learning_rate=0.05, depth=6, od_type='Iter', od_wait=200, cat_features=[...])",
"slower training than LightGBM; see catboost-model-builder")
rf = 7 - (1 if big else 0) + (1 if small else 0) - (2 if need["calibrated"] else 0) - (1 if need["latency_ms"] and need["latency_ms"] < 2 else 0)
add("Random forest", rf, "hard to break, little tuning, good for small/medium data and as a sanity check",
"RandomForestClassifier(n_estimators=500, max_features='sqrt', min_samples_leaf=5, n_jobs=-1)",
"probabilities are poorly calibrated; large models; usually below a tuned GBM")
add("Extra trees", rf - 0.5, "faster, more randomised forest; good with noisy features",
"ExtraTreesClassifier(n_estimators=500, min_samples_leaf=3, n_jobs=-1)", "same calibration caveats as RF")
if need["interpretable"] or need["monotone"]:
add("Explainable boosting machine (EBM / GAM)", 9 + (1 if need["monotone"] else 0) - (2 if big else 0),
"glass-box: one shape function per feature (+ few pairwise interactions); near-GBM accuracy on many tabular tasks",
"interpret.glassbox.ExplainableBoostingClassifier(interactions=10)",
"slower to train; shape functions need review for monotonicity (or constrain)")
add("Points scorecard (binned logistic regression)", 7 + (2 if need["tiny_model"] else 0),
"regulated credit-style decisions: integer points per bin, monotone by construction",
"KBinsDiscretizer or WoE binning -> LogisticRegression; scale to points", "coarser; binning choices matter")
add("Decision tree (shallow)", 4 + (3 if need["tiny_model"] else 0) + (1 if need["interpretable"] else 0),
"a readable rule set; deployable as SQL", "DecisionTreeClassifier(max_depth=4, min_samples_leaf=50)",
"usually much weaker than ensembles; unstable")
if t["text_cols"]:
add("Linear model on TF-IDF (logistic / linear SVM)", 10.5, "text columns: sparse linear models are the strong, fast baseline",
"TfidfVectorizer(ngram_range=(1,2), min_df=3) -> LogisticRegression(C=4) or LinearSVC", "sentence embeddings + GBM for semantic tasks")
add("Multinomial naive Bayes", 5 + (2 if small else 0), "very fast text baseline", "MultinomialNB(alpha=0.5) on counts / TF-IDF",
"probabilities are overconfident")
add("k-nearest neighbours", 3 + (2 if small and not wide else 0) - (3 if big else 0) - (2 if need["latency_ms"] and need["latency_ms"] < 10 else 0),
"local similarity, no training", "KNeighborsClassifier(n_neighbors=25, weights='distance') after scaling",
"slow at prediction time, degrades with many features, needs scaling")
add("SVM (RBF kernel)", 5 + (2 if small and not wide else 0) - (5 if n > 50_000 else 0),
"strong on small, dense, well-scaled data", "SVC(C=1, gamma='scale', probability=False) on scaled features",
"O(n^2) training; probability=True is slow and needs calibration")
add("MLP (neural net)", 4 + (1 if big else 0) - (2 if small else 0) - (1 if need["interpretable"] else 0),
"useful mainly with very large data or embeddings", "MLPClassifier(hidden_layer_sizes=(128, 64), early_stopping=True)",
"rarely beats GBMs on tabular data; needs scaling and tuning")
if rare:
for m in out:
if "Logistic" in m["model"] or "Boost" in m["model"] or m["model"] in ("LightGBM", "XGBoost", "CatBoost"):
m["caution"] += "; rare positives: evaluate with PR-AUC, keep the natural test prevalence (imbalanced-class-handler)"
if need["latency_ms"] is not None and need["latency_ms"] < 1:
for m in out:
if m["model"] in ("LightGBM", "XGBoost", "CatBoost", "Random forest", "Extra trees", "HistGradientBoosting (scikit-learn)"):
m["score"] -= 1.5
m["caution"] += "; sub-millisecond budget: cap trees/depth or compile (treelite, ONNX)"
return sorted(out, key=lambda m: -m["score"])
def main():
ap = argparse.ArgumentParser(description="Rank classifiers for a dataset and constraints.", epilog=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--csv")
ap.add_argument("--target")
ap.add_argument("--drop", nargs="*", default=[])
ap.add_argument("--rows", type=int)
ap.add_argument("--features", type=int)
ap.add_argument("--cat-share", type=float, default=0.0)
ap.add_argument("--hc-share", type=float, default=0.0)
ap.add_argument("--missing-share", type=float, default=0.0)
ap.add_argument("--minority", type=float, default=0.5)
ap.add_argument("--text-cols", type=int, default=0)
ap.add_argument("--interpretable", action="store_true")
ap.add_argument("--monotone", action="store_true")
ap.add_argument("--calibrated", action="store_true")
ap.add_argument("--latency-ms", type=float)
ap.add_argument("--tiny-model", action="store_true")
ap.add_argument("--top", type=int, default=8)
ap.add_argument("--json", action="store_true")
a = ap.parse_args()
try:
if a.csv:
if not a.target:
raise ValueError("--csv needs --target")
t = traits_from_csv(a.csv, a.target, set(a.drop))
elif a.rows and a.features:
t = {"rows": a.rows, "features": a.features, "cat_share": a.cat_share, "hc_share": a.hc_share,
"missing_share": a.missing_share, "minority": a.minority, "classes": 2, "text_cols": a.text_cols}
else:
raise ValueError("give --csv/--target or --rows and --features")
except (OSError, ValueError) as e:
print(f"error: {e}", file=sys.stderr)
return 2
need = {"interpretable": a.interpretable, "monotone": a.monotone, "calibrated": a.calibrated,
"latency_ms": a.latency_ms, "tiny_model": a.tiny_model}
ranked = score_models(t, need)[:a.top]
if a.json:
print(json.dumps({"traits": t, "needs": need, "ranking": ranked}, indent=2))
return 0
print(f"Data: {t['rows']:,} rows, {t['features']} features, categorical {t['cat_share']:.0%} "
f"(high-cardinality {t['hc_share']:.0%}), missing {t['missing_share']:.1%}, minority {t['minority']:.1%}, "
f"text columns {t['text_cols']}")
print("Needs: " + (", ".join(k for k, v in need.items() if v) or "none"))
for i, m in enumerate(ranked, 1):
print(f"\n{i}. {m['model']} (score {m['score']})\n why: {m['why']}\n start: {m['start']}\n"
f" caution: {m['caution']}")
print("\nShortlist the top 3-4 plus a logistic-regression baseline; compare them on identical CV folds "
"(python scripts/bakeoff_scaffold.py) before choosing.")
return 0
if __name__ == "__main__":
sys.exit(main())
anomaly-detection-model-builder
Build and evaluate anomaly detection for tabular, sensor and transaction data with few or no labels: define the anomaly type (point, contextual, collective), choose detectors (robust z-score/MAD and IQR rules, Mahalanobis with robust covariance, k-NN distance, LOF, Isolation Forest, one-class SVM, PCA reconstruction / autoencoders, forecasting residuals for time series), combine them by rank, explain each alert by its deviating features, set thresholds from an alert budget or the score distribution, and evaluate with precision@k, PR-AUC and recall on known incidents, or detector agreement when no labels exist. Includes pure-Python detectors (robust z, Mahalanobis, kNN, Isolation Forest, ensemble) and an evaluation script.
catboost-model-builder
Build CatBoost models that use its strengths correctly: raw categorical columns as cat_features (ordered target statistics, no manual encoding), text_features, Pool construction with NaN-safe categories, ordered boosting and has_time for temporal data, symmetric (oblivious) trees, overfitting detector with eval_set and use_best_model, losses for classification, regression (Quantile, Poisson, Tweedie) and ranking (YetiRank), class weights, monotone constraints, feature importance (PredictionValuesChange, LossFunctionChange) and SHAP values, GPU training and model export. Plans cat/text/numeric columns from a CSV and generates a runnable training script, and checks parameter sets for the conflicts CatBoost rejects (iterations vs n_estimators, Bayesian bootstrap with subsample, multiple class-weight options, depth > 16, use_best_model without eval_set).
categorical-encoding-selector
Choose and implement the right encoding for every categorical column given the model family and the data: one-hot (with infrequent-level grouping and handle_unknown), ordinal with an explicit order, cross-fitted target encoding (scikit-learn TargetEncoder, CatBoost ordered statistics, category_encoders inside CV), frequency/count encoding, feature hashing, native categorical support in LightGBM, HistGradientBoosting, XGBoost and CatBoost, prefix levels for codes (zip3, NAICS-2), and embeddings. Measures cardinality, rare-level share, unseen-level rate on a time-ordered holdout and target signal per column, emits the ColumnTransformer, and quantifies target-encoding leakage (in-sample vs out-of-fold vs holdout AUC).
clustering-algorithm-selector
Choose the right clustering algorithm and check that clustering makes sense at all: cluster tendency (Hopkins statistic with a correlated-features caveat, nearest-neighbour isolation), then k-means / MiniBatchKMeans, Gaussian mixtures with BIC, agglomerative (Ward, average, complete, single) with dendrograms, DBSCAN, HDBSCAN, OPTICS, spectral clustering, BIRCH, mean shift, k-prototypes and k-modes for mixed or categorical data, and Gower distance, matched to data size, expected shapes, noise, varying density, known k, soft memberships, hierarchy and streaming. Covers preprocessing (scaling, encoding, dimensionality, distance metrics), validation (internal indices, bootstrap stability, external labels, business profiling) and assigning new points.
collaborative-filtering-implementer
Implement collaborative filtering recommenders that beat popularity: item-based kNN with cosine, Jaccard or asymmetric similarity, shrinkage, neighbourhood size, positive-only or mean-centred rating signals and 'because you liked X' explanations; implicit ALS with confidence weighting, BPR pairwise ranking and the closed-form EASE model in plain NumPy, tuned on a temporal validation split by recall@k and NDCG@k and refit on all data; plus user-kNN, explicit matrix factorisation, the implicit/LightFM libraries, hyperparameter ranges, cold-start fallbacks, scaling with sparse matrices and approximate nearest neighbours. Produces top-k lists in the format of the recommendation evaluator.
counterfactual-explanation-generator
Generate counterfactual explanations and algorithmic recourse: the smallest realistic change to a row's features that changes the model's decision. Solves exact minimal-cost recourse for linear and logistic models (MAD-scaled L1 cost, immutable features, increase/decrease directions, bounds, integer and binary features, diverse alternatives, features every counterfactual must change, and a no-recourse finding when immutable features decide), generates DiCE counterfactuals for any scikit-learn model with constraints turned into permitted ranges and every result re-predicted, and validates counterfactuals from any method for constraint violations, implausible values, off-manifold rows, broken derived-feature relations and failure to flip the prediction.