#!/usr/bin/env python3
"""Rank clustering algorithms for a dataset and a goal, with starting parameters and validation advice.
Traits from a CSV (--csv, numeric/categorical counts, rows) or flags; goals/needs from flags:
--k-known N number of clusters is fixed by the business (segments for N teams)
--shapes convex|arbitrary|unknown expected cluster geometry
--noise there are outliers / background points that should not be forced into clusters
--varying-density clusters of very different densities
--soft need membership probabilities (overlapping segments)
--hierarchy need a dendrogram / nested segments
--mixed numeric + categorical features (or detected from the CSV)
--streaming data arrives continuously / does not fit in memory
Candidates: k-means / MiniBatchKMeans, Gaussian mixture, agglomerative (Ward / average / complete),
HDBSCAN, DBSCAN, OPTICS, spectral clustering, BIRCH, mean shift, k-prototypes (mixed),
k-modes (categorical), affinity propagation.
Usage:
python select_clustering.py --csv customers.csv --drop customer_id
python select_clustering.py --rows 2000000 --dims 12 --k-known 5 --streaming
python select_clustering.py --rows 5000 --dims 3 --shapes arbitrary --noise --varying-density --json
Exit codes: 0 ranking printed, 2 usage or read error.
"""
import argparse
import csv
import json
import math
import sys
MISSING = {"", "na", "n/a", "nan", "null", "none", "?"}
def traits_from_csv(path, drop):
with open(path, newline="", encoding="utf-8-sig") as fh:
rd = csv.DictReader(fh)
header = rd.fieldnames or []
rows = list(rd)
num = cat = 0
for c in header:
if c in drop:
continue
vals = [(r[c] or "").strip() for r in rows if (r[c] or "").strip().lower() not in MISSING]
if not vals:
continue
try:
[float(v) for v in vals]
num += 1
except ValueError:
if len(set(vals)) < 0.5 * len(vals):
cat += 1
return len(rows), num, cat
def rank(t):
n, d, cat = t["rows"], t["dims"], t["categorical"]
big = n > 200_000
huge = n > 2_000_000
mixed = t["mixed"] or (cat > 0 and d > 0)
only_cat = cat > 0 and d == 0
arb = t["shapes"] == "arbitrary"
out = []
def add(name, score, why, start, caution):
out.append({"algorithm": name, "score": round(score, 1), "why": why, "start": start, "caution": caution})
kk = t["k_known"]
add("k-means (KMeans / MiniBatchKMeans)",
7 + (2 if kk else 0) + (1.5 if big else 0) + (1 if t["streaming"] else 0) - (3 if arb else 0)
- (2 if t["noise"] else 0) - (1.5 if t["varying_density"] else 0) - (3 if mixed or only_cat else 0) - (2 if t["soft"] else 0),
"fast, scalable, simple centroids that are easy to explain as segments",
f"KMeans(n_clusters={kk or 'k from kmeans-cluster-count-finder'}, n_init='auto', random_state=0) on scaled features; "
"MiniBatchKMeans(batch_size=4096) for large data",
"assumes convex, similar-sized, similar-variance clusters; every point is assigned (outliers distort centroids)")
add("Gaussian mixture (GaussianMixture)",
6 + (2.5 if t["soft"] else 0) + (1 if kk else 0) - (2 if huge else 0) - (2 if arb else 0) - (3 if mixed or only_cat else 0)
- (1 if d > 50 else 0),
"soft assignments, elliptical clusters of different shapes/sizes, BIC to choose k",
"GaussianMixture(n_components=k, covariance_type='full', n_init=5, random_state=0); choose k by BIC",
"sensitive to initialisation and scaling; 'full' covariance needs many points per cluster in high dimensions")
add("HDBSCAN",
6 + (2.5 if arb else 0) + (2.5 if t["noise"] else 0) + (2 if t["varying_density"] else 0) - (2 if kk else 0)
- (1 if huge else 0) - (1.5 if d > 50 else 0) - (3 if only_cat else 0),
"finds arbitrary shapes and varying densities, labels noise, needs no k",
"sklearn.cluster.HDBSCAN(min_cluster_size=max(15, n//200), min_samples=None) on scaled (or UMAP-reduced) data",
"noise share can be large; cluster count follows the data, not the business; distance-based (scale first)")
add("DBSCAN",
4.5 + (2 if arb else 0) + (2 if t["noise"] else 0) - (2 if t["varying_density"] else 0) - (2 if kk else 0)
- (1 if d > 20 else 0) - (3 if only_cat else 0),
"arbitrary shapes and noise with two interpretable parameters",
"DBSCAN(eps from the k-distance knee, min_samples=2*dims) - see dbscan-parameter-tuner",
"one global density: fails with varying densities; eps is in scaled units")
add("Agglomerative (Ward)",
5.5 + (3 if t["hierarchy"] else 0) + (1 if kk else 0) - (4 if n > 30_000 else 0) - (1.5 if arb else 0) - (2 if mixed else 0)
- (4 if only_cat else 0),
"dendrogram for nested segments; deterministic",
"AgglomerativeClustering(n_clusters=k, linkage='ward'); scipy.cluster.hierarchy for the dendrogram",
"O(n^2) memory; Ward behaves like k-means (convex); use average/complete linkage for other metrics")
add("Spectral clustering",
4 + (2.5 if arb else 0) + (1 if kk else 0) - (5 if n > 20_000 else 0),
"non-convex clusters via a similarity graph, k known",
"SpectralClustering(n_clusters=k, affinity='nearest_neighbors', n_neighbors=15, assign_labels='cluster_qr')",
"does not scale beyond ~10-20k points; sensitive to the affinity choice")
add("BIRCH", 3.5 + (2 if big else 0) + (2 if t["streaming"] else 0) - (2 if arb else 0) - (3 if mixed or only_cat else 0),
"single pass, memory-bounded, incremental (partial_fit)", "Birch(threshold=0.5, n_clusters=k)",
"convex clusters; threshold depends on scaling")
add("OPTICS", 3.5 + (2 if t["varying_density"] else 0) + (1 if arb else 0) - (2 if big else 0),
"density ordering that handles varying density; reachability plot", "OPTICS(min_samples=2*dims, xi=0.05)",
"slower than HDBSCAN, which usually supersedes it")
add("Mean shift", 2.5 + (1 if arb else 0) - (4 if n > 10_000 else 0), "mode seeking; no k",
"MeanShift(bandwidth=estimate_bandwidth(X, quantile=0.2))", "slow; bandwidth-sensitive")
if mixed:
add("k-prototypes (kmodes package)", 8 + (1 if kk else 0) - (1 if huge else 0),
"numeric + categorical in one objective (Euclidean + matching dissimilarity)",
"KPrototypes(n_clusters=k, init='Cao', gamma=None).fit(X, categorical=[indices])",
"gamma weights categorical vs numeric parts; scale numerics first")
add("Gower distance + agglomerative / HDBSCAN (precomputed)", 6.5 - (4 if n > 20_000 else 0),
"any mix of types through a proper dissimilarity", "gower.gower_matrix(df) -> HDBSCAN(metric='precomputed')",
"O(n^2) memory")
if only_cat:
add("k-modes (kmodes package)", 9, "categorical-only data: modes instead of means",
"KModes(n_clusters=k, init='Huang', n_init=5)", "matching dissimilarity ignores level similarity")
if t["soft"]:
for o in out:
if o["algorithm"].startswith("HDBSCAN"):
o["caution"] += "; soft memberships via the hdbscan package's membership vectors"
return sorted(out, key=lambda o: -o["score"])
def main():
ap = argparse.ArgumentParser(description="Rank clustering algorithms for data and goals.", epilog=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--csv")
ap.add_argument("--drop", nargs="*", default=[])
ap.add_argument("--rows", type=int)
ap.add_argument("--dims", type=int, help="numeric feature count")
ap.add_argument("--categorical", type=int, default=0)
ap.add_argument("--k-known", type=int)
ap.add_argument("--shapes", choices=["convex", "arbitrary", "unknown"], default="unknown")
ap.add_argument("--noise", action="store_true")
ap.add_argument("--varying-density", action="store_true")
ap.add_argument("--soft", action="store_true")
ap.add_argument("--hierarchy", action="store_true")
ap.add_argument("--mixed", action="store_true")
ap.add_argument("--streaming", action="store_true")
ap.add_argument("--top", type=int, default=5)
ap.add_argument("--json", action="store_true")
a = ap.parse_args()
try:
if a.csv:
rows, dims, cat = traits_from_csv(a.csv, set(a.drop))
elif a.rows and a.dims is not None:
rows, dims, cat = a.rows, a.dims, a.categorical
else:
raise ValueError("give --csv or --rows and --dims")
except (OSError, ValueError) as e:
print(f"error: {e}", file=sys.stderr)
return 2
t = {"rows": rows, "dims": dims, "categorical": cat, "k_known": a.k_known, "shapes": a.shapes, "noise": a.noise,
"varying_density": a.varying_density, "soft": a.soft, "hierarchy": a.hierarchy, "mixed": a.mixed,
"streaming": a.streaming}
ranked = rank(t)[:a.top]
if a.json:
print(json.dumps({"traits": t, "ranking": ranked}, indent=2))
return 0
flags = [k for k in ("k_known", "noise", "varying_density", "soft", "hierarchy", "mixed", "streaming") if t[k]]
print(f"Data: {rows:,} rows, {dims} numeric, {cat} categorical; shapes {a.shapes}; needs: {', '.join(flags) or 'none'}")
for i, r in enumerate(ranked, 1):
print(f"\n{i}. {r['algorithm']} (score {r['score']})\n why: {r['why']}\n start: {r['start']}\n"
f" caution: {r['caution']}")
print("\nBefore trusting any result: check cluster tendency (cluster_tendency.py), scale features, compare 2-3 "
"algorithms, test stability (bootstrap ARI) and ask whether each cluster leads to a different action.")
return 0
if __name__ == "__main__":
sys.exit(main())
anomaly-detection-model-builder
Build and evaluate anomaly detection for tabular, sensor and transaction data with few or no labels: define the anomaly type (point, contextual, collective), choose detectors (robust z-score/MAD and IQR rules, Mahalanobis with robust covariance, k-NN distance, LOF, Isolation Forest, one-class SVM, PCA reconstruction / autoencoders, forecasting residuals for time series), combine them by rank, explain each alert by its deviating features, set thresholds from an alert budget or the score distribution, and evaluate with precision@k, PR-AUC and recall on known incidents, or detector agreement when no labels exist. Includes pure-Python detectors (robust z, Mahalanobis, kNN, Isolation Forest, ensemble) and an evaluation script.
catboost-model-builder
Build CatBoost models that use its strengths correctly: raw categorical columns as cat_features (ordered target statistics, no manual encoding), text_features, Pool construction with NaN-safe categories, ordered boosting and has_time for temporal data, symmetric (oblivious) trees, overfitting detector with eval_set and use_best_model, losses for classification, regression (Quantile, Poisson, Tweedie) and ranking (YetiRank), class weights, monotone constraints, feature importance (PredictionValuesChange, LossFunctionChange) and SHAP values, GPU training and model export. Plans cat/text/numeric columns from a CSV and generates a runnable training script, and checks parameter sets for the conflicts CatBoost rejects (iterations vs n_estimators, Bayesian bootstrap with subsample, multiple class-weight options, depth > 16, use_best_model without eval_set).
categorical-encoding-selector
Choose and implement the right encoding for every categorical column given the model family and the data: one-hot (with infrequent-level grouping and handle_unknown), ordinal with an explicit order, cross-fitted target encoding (scikit-learn TargetEncoder, CatBoost ordered statistics, category_encoders inside CV), frequency/count encoding, feature hashing, native categorical support in LightGBM, HistGradientBoosting, XGBoost and CatBoost, prefix levels for codes (zip3, NAICS-2), and embeddings. Measures cardinality, rare-level share, unseen-level rate on a time-ordered holdout and target signal per column, emits the ColumnTransformer, and quantifies target-encoding leakage (in-sample vs out-of-fold vs holdout AUC).
classification-model-selector
Choose the right classification algorithm for tabular, text or mixed data and prove the choice with a fair bake-off: logistic regression (L2, L1, elastic net), linear SVM, naive Bayes, k-NN, RBF SVM, decision trees, random forest, extra trees, HistGradientBoosting, LightGBM, XGBoost, CatBoost, explainable boosting machines (EBM/GAM), points scorecards and MLPs, matched to data size, width, categorical and high-cardinality share, missingness, class imbalance, text, and constraints such as interpretability, monotonic constraints, calibrated probabilities, latency budgets and tiny deployable models. Ranks candidates with starting hyperparameters and cautions, then generates a bake-off script running every candidate on identical CV folds with per-fold metrics, fit time and prediction latency.
collaborative-filtering-implementer
Implement collaborative filtering recommenders that beat popularity: item-based kNN with cosine, Jaccard or asymmetric similarity, shrinkage, neighbourhood size, positive-only or mean-centred rating signals and 'because you liked X' explanations; implicit ALS with confidence weighting, BPR pairwise ranking and the closed-form EASE model in plain NumPy, tuned on a temporal validation split by recall@k and NDCG@k and refit on all data; plus user-kNN, explicit matrix factorisation, the implicit/LightFM libraries, hyperparameter ranges, cold-start fallbacks, scaling with sparse matrices and approximate nearest neighbours. Produces top-k lists in the format of the recommendation evaluator.
counterfactual-explanation-generator
Generate counterfactual explanations and algorithmic recourse: the smallest realistic change to a row's features that changes the model's decision. Solves exact minimal-cost recourse for linear and logistic models (MAD-scaled L1 cost, immutable features, increase/decrease directions, bounds, integer and binary features, diverse alternatives, features every counterfactual must change, and a no-recourse finding when immutable features decide), generates DiCE counterfactuals for any scikit-learn model with constraints turned into permitted ranges and every result re-predicted, and validates counterfactuals from any method for constraint violations, implausible values, off-manifold rows, broken derived-feature relations and failure to flip the prediction.