Advanced Neural Networks

1/**2 * Example 39: Advanced Neural Networks3 *4 * New in v1.0.0: Trainer with EarlyStopping, weight initialization functions,5 * Embedding layers, normalization layers, containers (ModuleList, ModuleDict),6 * and advanced activation functions (GELU, PReLU).7 */89import { type AnyTensor, type Tensor, tensor } from "deepbox/ndarray";10import {11  Dropout,12  EarlyStopping,13  Embedding,14  GELU,15  GroupNorm,16  kaiming_normal_,17  LayerNorm,18  Linear,19  ModelCheckpoint,20  ModuleDict,21  ModuleList,22  mseLoss,23  PReLU,24  ReLU,25  Sequential,26  Trainer,27  xavier_uniform_,28  zeros_,29} from "deepbox/nn";30import { Adam } from "deepbox/optim";3132console.log("=".repeat(60));33console.log("Example 39: Advanced Neural Networks");34console.log("=".repeat(60));3536// ============================================================================37// Part 1: Weight Initialization38// ============================================================================39console.log("\n🎲 Part 1: Weight Initialization");40console.log("-".repeat(60));4142// Weight initialization is crucial for training stability43const layer = new Linear(64, 32);4445// Xavier/Glorot uniform — best for sigmoid/tanh activations46xavier_uniform_(layer.getWeight());47if (layer.getBias()) zeros_(layer.getBias()!);48console.log("xavier_uniform_ applied to Linear(64, 32):");49console.log(`  Weight shape: ${layer.getWeight().shape}, Bias shape: ${layer.getBias()?.shape}`);5051// Kaiming/He normal — best for ReLU activations52const reluLayer = new Linear(128, 64);53kaiming_normal_(reluLayer.getWeight(), 0, "fan_in", "relu");54if (reluLayer.getBias()) zeros_(reluLayer.getBias()!);55console.log("kaiming_normal_ applied to Linear(128, 64):");56console.log(`  Weight shape: ${reluLayer.getWeight().shape}`);5758// ============================================================================59// Part 2: Advanced Activation Functions60// ============================================================================61console.log("\n⚡ Part 2: Advanced Activations (GELU, PReLU)");62console.log("-".repeat(60));6364// GELU — Gaussian Error Linear Unit, used in Transformers65const gelu = new GELU();66const geluInput = tensor([-2, -1, 0, 1, 2]);67const geluOutput = gelu.forward(geluInput);68console.log("GELU activation:");69console.log(`  Input:  ${geluInput.toString()}`);70console.log(`  Output: ${geluOutput.toString()}`);7172// PReLU — Parametric ReLU with learnable negative slope73const prelu = new PReLU();74const preluInput = tensor([-2, -1, 0, 1, 2], { dtype: "float64" });75const preluOutput = prelu.forward(preluInput);76console.log("\nPReLU activation (learnable slope):");77console.log(`  Input:  ${preluInput.toString()}`);78console.log(`  Output: ${preluOutput.toString()}`);7980// ============================================================================81// Part 3: Normalization Layers82// ============================================================================83console.log("\n📏 Part 3: Normalization Layers");84console.log("-".repeat(60));8586// LayerNorm — normalizes across features (used in Transformers)87const ln = new LayerNorm([4]);88const lnInput = tensor([89  [1, 2, 3, 4],90  [5, 6, 7, 8],91]);92const lnOutput = ln.forward(lnInput);93console.log("LayerNorm([4]):");94console.log(`  Input:  ${lnInput.toString()}`);95console.log(`  Output: ${lnOutput.toString()}`);9697// GroupNorm — normalizes within groups of channels98const gn = new GroupNorm(2, 4); // 2 groups, 4 channels99const gnInput = tensor([100  [101    [1, 2],102    [3, 4],103    [5, 6],104    [7, 8],105  ],106]);107const gnOutput = gn.forward(gnInput);108console.log(`\nGroupNorm(2 groups, 4 channels):`);109console.log(`  Input shape: ${gnInput.shape}`);110console.log(`  Output shape: ${gnOutput.shape}`);111112// ============================================================================113// Part 4: Embedding Layer114// ============================================================================115console.log("\n📖 Part 4: Embedding Layer");116console.log("-".repeat(60));117118// Embedding maps integer indices to dense vectors (used for words, tokens, etc.)119const vocabSize = 10;120const embeddingDim = 4;121const emb = new Embedding(vocabSize, embeddingDim);122123// Look up embeddings for token indices124const tokenIds = tensor([0, 3, 7, 1]);125const embeddings = emb.forward(tokenIds);126console.log(`Embedding(vocab=${vocabSize}, dim=${embeddingDim}):`);127console.log(`  Token IDs: ${tokenIds.toString()}`);128console.log(`  Embeddings shape: ${embeddings.shape}`);129console.log(`  Each token → ${embeddingDim}D vector`);130131// ============================================================================132// Part 5: Module Containers (ModuleList, ModuleDict)133// ============================================================================134console.log("\n📦 Part 5: Module Containers");135console.log("-".repeat(60));136137// ModuleList — ordered list of modules (proper parameter tracking)138const layers = new ModuleList([139  new Linear(32, 16),140  new ReLU(),141  new Linear(16, 8),142  new ReLU(),143  new Linear(8, 1),144]);145146console.log("ModuleList with 5 layers:");147let totalParams = 0;148for (const [name, param] of layers.namedParameters()) {149  totalParams += param.size;150  console.log(`  ${name}: ${param.shape}`);151}152console.log(`  Total parameters: ${totalParams}`);153154// ModuleDict — dictionary of named modules155const branches = new ModuleDict({156  encoder: new Sequential(new Linear(10, 8), new ReLU()),157  decoder: new Sequential(new Linear(8, 10), new ReLU()),158});159160console.log("\nModuleDict with encoder/decoder branches:");161for (const [name, param] of branches.namedParameters()) {162  console.log(`  ${name}: ${param.shape}`);163}164165// ============================================================================166// Part 6: Sequential with Dropout167// ============================================================================168console.log("\n🏗️  Part 6: Sequential Model with Dropout");169console.log("-".repeat(60));170171// Build a multi-layer model with dropout regularization172const model = new Sequential(173  new Linear(4, 16),174  new ReLU(),175  new Dropout(0.2),176  new Linear(16, 8),177  new ReLU(),178  new Dropout(0.1),179  new Linear(8, 1)180);181182console.log("Sequential model architecture:");183let paramCount = 0;184for (const [name, param] of model.namedParameters()) {185  paramCount += param.size;186  console.log(`  ${name}: ${param.shape}`);187}188console.log(`  Total parameters: ${paramCount}`);189190// Forward pass191const xDemo = tensor([192  [1, 2, 3, 4],193  [5, 6, 7, 8],194]);195model.eval(); // disable dropout for inference196const yDemo = model.forward(xDemo);197console.log(`\n  Input shape:  ${xDemo.shape}`);198console.log(`  Output shape: ${yDemo.shape}`);199200// ============================================================================201// Part 7: Trainer with EarlyStopping202// ============================================================================203console.log("\n🏋️  Part 7: Trainer with EarlyStopping");204console.log("-".repeat(60));205206// Create a simple regression model207const trainerModel = new Sequential(new Linear(4, 16), new ReLU(), new Linear(16, 1));208209const optimizer = new Adam(trainerModel.parameters(), { lr: 0.01 });210const lossFn = (pred: AnyTensor, target: Tensor) => mseLoss(pred, target);211212// The Trainer manages the training loop with callbacks213const trainer = new Trainer(trainerModel, optimizer, lossFn, {214  epochs: 50,215  earlyStopping: { patience: 5, minDelta: 0.001 },216  verbose: false,217});218219// Generate simple training data: y = sum(x) + noise220const trainBatches: [ReturnType<typeof tensor>, ReturnType<typeof tensor>][] = [];221for (let i = 0; i < 10; i++) {222  const xBatch = tensor([223    [1 + i * 0.1, 2 + i * 0.2, 3 - i * 0.1, 4 + i * 0.05],224    [2 + i * 0.1, 1 + i * 0.1, 4 - i * 0.2, 3 + i * 0.1],225  ]);226  const yBatch = tensor([[10 + i * 0.25], [10 + i * 0.0]]);227  trainBatches.push([xBatch, yBatch]);228}229230const result = trainer.fit(trainBatches);231232console.log("Trainer results:");233console.log(`  Total epochs run: ${result.history.length}`);234console.log(235  `  Final train loss: ${result.history[result.history.length - 1]?.trainLoss.toFixed(6)}`236);237if (result.stoppedEarly) {238  console.log(`  Early stopping triggered at epoch ${result.bestEpoch}`);239} else {240  console.log("  Completed all epochs");241}242243// ============================================================================244// Part 8: EarlyStopping & ModelCheckpoint (standalone)245// ============================================================================246console.log("\n💾 Part 8: EarlyStopping & ModelCheckpoint");247console.log("-".repeat(60));248249// EarlyStopping monitors a metric and stops when it stops improving250const earlyStop = new EarlyStopping({251  patience: 3,252  minDelta: 0.01,253  mode: "min",254});255256const losses = [1.0, 0.8, 0.6, 0.55, 0.54, 0.54, 0.54, 0.53];257console.log("EarlyStopping (patience=3, minDelta=0.01, mode=min):");258for (let i = 0; i < losses.length; i++) {259  const shouldStop = earlyStop.step(losses[i]!);260  console.log(`  Epoch ${i + 1}: loss=${losses[i]!.toFixed(2)}, stop=${shouldStop}`);261  if (shouldStop) break;262}263264// ModelCheckpoint saves and restores the best model state265const checkpointModel = new Sequential(new Linear(4, 2));266const checkpoint = new ModelCheckpoint({ mode: "min" });267268console.log("\nModelCheckpoint (saves best model state):");269const checkLosses = [1.0, 0.8, 0.9, 0.7, 0.75];270for (let i = 0; i < checkLosses.length; i++) {271  const improved = checkpoint.step(checkpointModel, checkLosses[i]!);272  console.log(`  Epoch ${i + 1}: loss=${checkLosses[i]!.toFixed(2)}, saved=${improved}`);273}274275// Restore best model weights276checkpoint.restore(checkpointModel);277console.log("  Best model restored!");278279// ============================================================================280// Summary281// ============================================================================282console.log("\n💡 Key Takeaways");283console.log("-".repeat(60));284console.log("• xavier_uniform_/kaiming_normal_: proper initialization for stable training");285console.log("• GELU: smooth activation used in modern Transformers");286console.log("• PReLU: learnable negative slope for adaptive activation");287console.log("• LayerNorm/GroupNorm: normalization layers for different architectures");288console.log("• Embedding: maps discrete indices to dense learned vectors");289console.log("• ModuleList/ModuleDict: containers with proper parameter tracking");290console.log("• Trainer: managed training loop with epochs, callbacks, logging");291console.log("• EarlyStopping: prevents overfitting by monitoring validation metrics");292console.log("• ModelCheckpoint: saves and restores the best model weights");293294console.log("\n✅ Advanced Neural Networks Example Complete!");295console.log("=".repeat(60));296