Ensemble & Advanced ML Models

1/**2 * Example 34: Ensemble & Advanced ML Models3 *4 * New in v1.0.0: AdaBoost, Bagging, Voting, Stacking, ExtraTrees,5 * Gaussian Processes, Discriminant Analysis, and Semi-supervised Learning.6 */78import { makeClassification, makeRegression } from "deepbox/datasets";9import { accuracy, f1Score, r2Score } from "deepbox/metrics";10import {11  AdaBoostClassifier,12  BaggingClassifier,13  DecisionTreeClassifier,14  ExtraTreesClassifier,15  GaussianProcessRegressor,16  KNeighborsClassifier,17  LinearDiscriminantAnalysis,18  LogisticRegression,19  RandomForestClassifier,20  StackingClassifier,21  VotingClassifier,22} from "deepbox/ml";23import { trainTestSplit } from "deepbox/preprocess";2425console.log("=".repeat(60));26console.log("Example 34: Ensemble & Advanced ML Models");27console.log("=".repeat(60));2829// ============================================================================30// Generate datasets for classification and regression31// ============================================================================3233const [XClass, yClass] = makeClassification({34  nSamples: 200,35  nFeatures: 10,36  nInformative: 6,37  nClasses: 2,38  randomState: 42,39});4041const [XTrain, XTest, yTrain, yTest] = trainTestSplit(XClass, yClass, {42  testSize: 0.25,43  randomState: 42,44});4546const [XReg, yReg] = makeRegression({47  nSamples: 100,48  nFeatures: 5,49  noise: 10,50  randomState: 42,51});5253const [XTrainReg, XTestReg, yTrainReg, yTestReg] = trainTestSplit(XReg, yReg, {54  testSize: 0.25,55  randomState: 42,56});5758// ============================================================================59// Part 1: AdaBoost Classifier60// ============================================================================61console.log("\n🚀 Part 1: AdaBoost Classifier");62console.log("-".repeat(60));6364// AdaBoost builds an ensemble of weak learners, focusing on misclassified samples65const adaboost = new AdaBoostClassifier({66  nEstimators: 50,67  learningRate: 1.0,68});69adaboost.fit(XTrain, yTrain);7071const adaPred = adaboost.predict(XTest);72const adaAcc = accuracy(yTest, adaPred);73const adaF1 = f1Score(yTest, adaPred);7475console.log("AdaBoost Classifier (50 estimators):");76console.log(`  Accuracy: ${(Number(adaAcc) * 100).toFixed(2)}%`);77console.log(`  F1 Score: ${Number(adaF1).toFixed(4)}`);7879// ============================================================================80// Part 2: Bagging Classifier81// ============================================================================82console.log("\n🎒 Part 2: Bagging Classifier");83console.log("-".repeat(60));8485// Bagging trains multiple models on random subsets of the data (bootstrap)86const bagging = new BaggingClassifier({87  nEstimators: 20,88  maxSamples: 0.8,89  maxFeatures: 0.8,90  randomState: 42,91});92bagging.fit(XTrain, yTrain);9394const bagPred = bagging.predict(XTest);95const bagAcc = accuracy(yTest, bagPred);96const bagF1 = f1Score(yTest, bagPred);9798console.log("Bagging Classifier (20 estimators, 80% samples/features):");99console.log(`  Accuracy: ${(Number(bagAcc) * 100).toFixed(2)}%`);100console.log(`  F1 Score: ${Number(bagF1).toFixed(4)}`);101102// ============================================================================103// Part 3: Voting Classifier104// ============================================================================105console.log("\n🗳️  Part 3: Voting Classifier");106console.log("-".repeat(60));107108// Voting combines multiple diverse classifiers for better predictions109const voting = new VotingClassifier({110  estimators: [111    new LogisticRegression({ maxIter: 200 }),112    new RandomForestClassifier({ nEstimators: 20, randomState: 42 }),113    new KNeighborsClassifier({ nNeighbors: 5 }),114  ],115  voting: "hard",116});117voting.fit(XTrain, yTrain);118119const votePred = voting.predict(XTest);120const voteAcc = accuracy(yTest, votePred);121const voteF1 = f1Score(yTest, votePred);122123console.log("Voting Classifier (LogReg + RandomForest + KNN, hard voting):");124console.log(`  Accuracy: ${(Number(voteAcc) * 100).toFixed(2)}%`);125console.log(`  F1 Score: ${Number(voteF1).toFixed(4)}`);126127// ============================================================================128// Part 4: Stacking Classifier129// ============================================================================130console.log("\n📚 Part 4: Stacking Classifier");131console.log("-".repeat(60));132133// Stacking uses a meta-learner to combine base estimator predictions134const stacking = new StackingClassifier({135  estimators: [136    new DecisionTreeClassifier({ maxDepth: 5 }),137    new KNeighborsClassifier({ nNeighbors: 5 }),138  ],139  finalEstimator: new LogisticRegression({ maxIter: 200 }),140});141stacking.fit(XTrain, yTrain);142143const stackPred = stacking.predict(XTest);144const stackAcc = accuracy(yTest, stackPred);145const stackF1 = f1Score(yTest, stackPred);146147console.log("Stacking Classifier (DecisionTree + KNN → LogReg meta-learner):");148console.log(`  Accuracy: ${(Number(stackAcc) * 100).toFixed(2)}%`);149console.log(`  F1 Score: ${Number(stackF1).toFixed(4)}`);150151// ============================================================================152// Part 5: ExtraTrees Classifier153// ============================================================================154console.log("\n🌳 Part 5: ExtraTrees Classifier");155console.log("-".repeat(60));156157// ExtraTrees: like Random Forest but with random split thresholds (more randomized)158const extraTrees = new ExtraTreesClassifier({159  nEstimators: 50,160  maxDepth: 10,161  randomState: 42,162});163extraTrees.fit(XTrain, yTrain);164165const etPred = extraTrees.predict(XTest);166const etAcc = accuracy(yTest, etPred);167const etF1 = f1Score(yTest, etPred);168169console.log("ExtraTrees Classifier (50 estimators, maxDepth=10):");170console.log(`  Accuracy: ${(Number(etAcc) * 100).toFixed(2)}%`);171console.log(`  F1 Score: ${Number(etF1).toFixed(4)}`);172173// Feature importances174const importances = extraTrees.featureImportances;175console.log("  Feature importances:", importances.toString());176177// ============================================================================178// Part 6: Gaussian Process Regressor179// ============================================================================180console.log("\n📐 Part 6: Gaussian Process Regressor");181console.log("-".repeat(60));182183// Gaussian Processes provide probabilistic predictions with uncertainty estimates184const gpr = new GaussianProcessRegressor({185  alpha: 1e-2,186});187gpr.fit(XTrainReg, yTrainReg);188189const gprPred = gpr.predict(XTestReg);190const gprR2 = r2Score(yTestReg, gprPred);191192console.log("Gaussian Process Regressor:");193console.log(`  R² Score: ${Number(gprR2).toFixed(4)}`);194console.log(`  Predictions shape: ${gprPred.shape}`);195196// ============================================================================197// Part 7: Linear Discriminant Analysis198// ============================================================================199console.log("\n📏 Part 7: Linear Discriminant Analysis");200console.log("-".repeat(60));201202// LDA finds linear combinations of features that best separate classes203const lda = new LinearDiscriminantAnalysis();204lda.fit(XTrain, yTrain);205206const ldaPred = lda.predict(XTest);207const ldaAcc = accuracy(yTest, ldaPred);208209console.log("Linear Discriminant Analysis:");210console.log(`  Accuracy: ${(Number(ldaAcc) * 100).toFixed(2)}%`);211212// LDA can also transform data for dimensionality reduction213const ldaTransformed = lda.transform(XTest);214console.log(`  Transformed shape: ${XTest.shape} → ${ldaTransformed.shape}`);215216// ============================================================================217// Part 8: Model Comparison Summary218// ============================================================================219console.log("\n📊 Part 8: Model Comparison");220console.log("-".repeat(60));221222const results = [223  { name: "AdaBoost", acc: Number(adaAcc), f1: Number(adaF1) },224  { name: "Bagging", acc: Number(bagAcc), f1: Number(bagF1) },225  { name: "Voting", acc: Number(voteAcc), f1: Number(voteF1) },226  { name: "Stacking", acc: Number(stackAcc), f1: Number(stackF1) },227  { name: "ExtraTrees", acc: Number(etAcc), f1: Number(etF1) },228  { name: "LDA", acc: Number(ldaAcc), f1: 0 },229];230231console.log("\nClassification Model Comparison:");232console.log("┌─────────────┬──────────┬──────────┐");233console.log("│ Model       │ Accuracy │ F1 Score │");234console.log("├─────────────┼──────────┼──────────┤");235for (const r of results) {236  const name = r.name.padEnd(11);237  const acc = `${(r.acc * 100).toFixed(2).padStart(6)}%`;238  const f1 = r.f1 > 0 ? r.f1.toFixed(4).padStart(8) : "     N/A";239  console.log(`│ ${name} │ ${acc}  │ ${f1} │`);240}241console.log("└─────────────┴──────────┴──────────┘");242243// ============================================================================244// Summary245// ============================================================================246console.log("\n💡 Key Takeaways");247console.log("-".repeat(60));248console.log("• AdaBoost: boosting weak learners sequentially, focuses on hard examples");249console.log("• Bagging: reduces variance by training on random data subsets");250console.log("• Voting: combines diverse models via majority vote or probability averaging");251console.log("• Stacking: uses a meta-learner to combine base model predictions");252console.log("• ExtraTrees: extremely randomized trees for faster training");253console.log("• Gaussian Processes: probabilistic regression with uncertainty estimates");254console.log("• LDA: simultaneous classification and dimensionality reduction");255console.log("• All models follow the unified fit/predict/score API");256257console.log("\n✅ Ensemble & Advanced ML Example Complete!");258console.log("=".repeat(60));259