Model Selection & Pipeline

1/**2 * Example 37: Model Selection & Pipeline3 *4 * New in v1.0.0: GridSearchCV, RandomizedSearchCV, Pipeline,5 * ColumnTransformer, and cross-validation utilities.6 */78import { makeClassification } from "deepbox/datasets";9import { accuracy } from "deepbox/metrics";10import {11  cross_validate,12  GridSearchCV,13  KNeighborsClassifier,14  LogisticRegression,15  Pipeline,16  RandomForestClassifier,17  RandomizedSearchCV,18  SVC,19} from "deepbox/ml";20import { StandardScaler, trainTestSplit } from "deepbox/preprocess";2122console.log("=".repeat(60));23console.log("Example 37: Model Selection & Pipeline");24console.log("=".repeat(60));2526// ============================================================================27// Generate dataset28// ============================================================================2930const [X, y] = makeClassification({31  nSamples: 200,32  nFeatures: 8,33  nInformative: 5,34  nClasses: 2,35  randomState: 42,36});3738const [XTrain, XTest, yTrain, yTest] = trainTestSplit(X, y, {39  testSize: 0.25,40  randomState: 42,41});4243console.log(`Dataset: ${X.shape[0]} samples, ${X.shape[1]} features`);44console.log(`Train: ${XTrain.shape[0]}, Test: ${XTest.shape[0]}`);4546// ============================================================================47// Part 1: Pipeline — Chain Preprocessing + Model48// ============================================================================49console.log("\n🔗 Part 1: Pipeline");50console.log("-".repeat(60));5152// A Pipeline chains transformers and a final estimator into a single object53// Steps: [name, estimator] tuples — intermediate steps must be transformers54const pipe = new Pipeline([55  ["scaler", new StandardScaler()],56  ["classifier", new LogisticRegression({ maxIter: 200 })],57]);5859// fit() applies fitTransform to intermediate steps, then fit to final step60pipe.fit(XTrain, yTrain);6162// predict() applies transform to intermediate steps, then predict on final step63const pipePred = pipe.predict(XTest);64const pipeAcc = accuracy(yTest, pipePred);6566console.log("Pipeline (StandardScaler → LogisticRegression):");67console.log(`  Accuracy: ${(Number(pipeAcc) * 100).toFixed(2)}%`);6869// Try a different pipeline with StandardScaler + KNN70const pipe2 = new Pipeline([71  ["scaler", new StandardScaler()],72  ["classifier", new KNeighborsClassifier({ nNeighbors: 5 })],73]);74pipe2.fit(XTrain, yTrain);7576const pipe2Pred = pipe2.predict(XTest);77const pipe2Acc = accuracy(yTest, pipe2Pred);7879console.log("Pipeline (StandardScaler → KNN):");80console.log(`  Accuracy: ${(Number(pipe2Acc) * 100).toFixed(2)}%`);8182// ============================================================================83// Part 2: Cross-Validation84// ============================================================================85console.log("\n📊 Part 2: Cross-Validation");86console.log("-".repeat(60));8788// cross_validate evaluates a model using k-fold cross-validation89const lr = new LogisticRegression({ maxIter: 200 });90const cvResult = cross_validate(lr, XTrain, yTrain, { cv: 5 });9192console.log("LogisticRegression 5-fold Cross-Validation:");93console.log(94  `  Test scores: [${cvResult.testScores["score"]?.map((s) => s.toFixed(4)).join(", ")}]`95);96const meanScore =97  (cvResult.testScores["score"]?.reduce((a, b) => a + b, 0) ?? 0) /98  (cvResult.testScores["score"]?.length ?? 1);99console.log(`  Mean score: ${meanScore.toFixed(4)}`);100101// Cross-validate different models for comparison102const models = [103  {104    name: "LogisticRegression",105    model: new LogisticRegression({ maxIter: 200 }),106  },107  { name: "KNN (k=5)", model: new KNeighborsClassifier({ nNeighbors: 5 }) },108  {109    name: "RandomForest (20)",110    model: new RandomForestClassifier({ nEstimators: 20, randomState: 42 }),111  },112  { name: "SVC (RBF)", model: new SVC({ kernel: "rbf", gamma: "scale" }) },113];114115console.log("\nModel Comparison (5-fold CV):");116for (const { name, model } of models) {117  const cv = cross_validate(model, XTrain, yTrain, { cv: 5 });118  const scores = cv.testScores["score"] ?? [];119  const mean = scores.reduce((a, b) => a + b, 0) / scores.length;120  const std = Math.sqrt(scores.reduce((a, b) => a + (b - mean) ** 2, 0) / scores.length);121  console.log(`  ${name.padEnd(22)} — Mean: ${mean.toFixed(4)} ± ${std.toFixed(4)}`);122}123124// ============================================================================125// Part 3: GridSearchCV — Exhaustive Hyperparameter Search126// ============================================================================127console.log("\n🔍 Part 3: GridSearchCV");128console.log("-".repeat(60));129130// GridSearchCV searches over all combinations of hyperparameters131const knnGrid = new GridSearchCV(132  new KNeighborsClassifier(),133  {134    nNeighbors: [3, 5, 7, 9, 11],135  },136  { cv: 5 }137);138139knnGrid.fit(XTrain, yTrain);140141console.log("GridSearchCV for KNeighborsClassifier:");142console.log(`  Best params: nNeighbors=${knnGrid.bestParams["nNeighbors"]}`);143console.log(`  Best CV score: ${knnGrid.bestScore.toFixed(4)}`);144145// Evaluate best model on test set146if (knnGrid.bestEstimator) {147  const best = knnGrid.bestEstimator as KNeighborsClassifier;148  const bestPred = best.predict(XTest);149  const bestAcc = accuracy(yTest, bestPred);150  console.log(`  Test accuracy (best model): ${(Number(bestAcc) * 100).toFixed(2)}%`);151}152153// Show all CV results154console.log("\n  All GridSearch Results:");155for (const result of knnGrid.cvResults) {156  const params = JSON.stringify(result.params);157  const scores = result.scores;158  const mean = result.meanScore;159  const std = Math.sqrt(scores.reduce((a, b) => a + (b - mean) ** 2, 0) / scores.length);160  console.log(`    ${params.padEnd(20)} → mean: ${mean.toFixed(4)}, std: ${std.toFixed(4)}`);161}162163// ============================================================================164// Part 4: RandomizedSearchCV — Random Hyperparameter Sampling165// ============================================================================166console.log("\n🎲 Part 4: RandomizedSearchCV");167console.log("-".repeat(60));168169// RandomizedSearchCV samples random combinations — faster than exhaustive search170const rfRandomSearch = new RandomizedSearchCV(171  new RandomForestClassifier({ randomState: 42 }),172  {173    nEstimators: [10, 20, 50, 100],174    maxDepth: [3, 5, 10, 15],175  },176  { cv: 3, nIter: 8 }177);178179rfRandomSearch.fit(XTrain, yTrain);180181console.log("RandomizedSearchCV for RandomForestClassifier (8 iterations):");182console.log(`  Best params: ${JSON.stringify(rfRandomSearch.bestParams)}`);183console.log(`  Best CV score: ${rfRandomSearch.bestScore.toFixed(4)}`);184185if (rfRandomSearch.bestEstimator) {186  const best = rfRandomSearch.bestEstimator as RandomForestClassifier;187  const bestPred = best.predict(XTest);188  const bestAcc = accuracy(yTest, bestPred);189  console.log(`  Test accuracy (best model): ${(Number(bestAcc) * 100).toFixed(2)}%`);190}191192// ============================================================================193// Part 5: GridSearchCV with Pipeline194// ============================================================================195console.log("\n🔗🔍 Part 5: Combining Pipeline with GridSearchCV");196console.log("-".repeat(60));197198// You can use GridSearchCV on individual models within a pipeline workflow199// First scale, then search over different KNN params200const scalerForSearch = new StandardScaler();201const XTrainScaled = scalerForSearch.fitTransform(XTrain);202const XTestScaled = scalerForSearch.transform(XTest);203204const knnGridScaled = new GridSearchCV(205  new KNeighborsClassifier(),206  {207    nNeighbors: [3, 5, 7, 9],208  },209  { cv: 5 }210);211212knnGridScaled.fit(XTrainScaled, yTrain);213214console.log("GridSearchCV on scaled data:");215console.log(`  Best nNeighbors: ${knnGridScaled.bestParams["nNeighbors"]}`);216console.log(`  Best CV score: ${knnGridScaled.bestScore.toFixed(4)}`);217218if (knnGridScaled.bestEstimator) {219  const best = knnGridScaled.bestEstimator as KNeighborsClassifier;220  const bestPred = best.predict(XTestScaled);221  const bestAcc = accuracy(yTest, bestPred);222  console.log(`  Test accuracy: ${(Number(bestAcc) * 100).toFixed(2)}%`);223}224225// ============================================================================226// Summary227// ============================================================================228console.log("\n💡 Key Takeaways");229console.log("-".repeat(60));230console.log("• Pipeline: chain transformers + estimator into a single fit/predict call");231console.log("• cross_validate: evaluate models with k-fold CV for robust estimates");232console.log("• GridSearchCV: exhaustive search over all hyperparameter combinations");233console.log("• RandomizedSearchCV: random sampling — faster for large param spaces");234console.log("• Always use CV scores (not single train/test) for model selection");235console.log("• Scale features before using distance-based models (KNN, SVM)");236237console.log("\n✅ Model Selection & Pipeline Example Complete!");238console.log("=".repeat(60));239