Normalization & Dropout Layers

1/**2 * Example 30: Normalization & Dropout Layers3 *4 * Demonstrates BatchNorm1d, LayerNorm, and Dropout for training stability5 * and regularization. These layers are essential for deep network training.6 */78import { GradTensor, tensor } from "deepbox/ndarray";9import { BatchNorm1d, Dropout, LayerNorm } from "deepbox/nn";1011console.log("=== Normalization & Dropout Layers ===\n");1213// ---------------------------------------------------------------------------14// Part 1: BatchNorm1d — Normalize over the batch dimension15// ---------------------------------------------------------------------------16console.log("--- Part 1: BatchNorm1d ---");1718// BatchNorm1d(numFeatures) — normalizes each feature across the batch19const bn = new BatchNorm1d(3);20console.log("BatchNorm1d(numFeatures=3)");21console.log("  Formula: y = (x - E[x]) / sqrt(Var[x] + eps) * gamma + beta\n");2223// Input shape: (batch, features)24const bnInput = tensor([25  [10, 20, 30],26  [11, 22, 28],27  [9, 18, 32],28  [12, 21, 29],29]);30console.log(`Input shape: [${bnInput.shape.join(", ")}]`);31console.log(`Input:\n${bnInput.toString()}`);3233// Training mode: uses batch statistics34bn.train();35const bnOut = bn.forward(bnInput);36const bnTensor = bnOut instanceof GradTensor ? bnOut.tensor : bnOut;37console.log(`\nOutput (training mode): shape [${bnTensor.shape.join(", ")}]`);38console.log("  Uses batch mean/variance, updates running statistics\n");3940// Eval mode: uses running statistics41bn.eval();42const bnEvalOut = bn.forward(bnInput);43const bnEvalTensor = bnEvalOut instanceof GradTensor ? bnEvalOut.tensor : bnEvalOut;44console.log(`Output (eval mode): shape [${bnEvalTensor.shape.join(", ")}]`);45console.log("  Uses accumulated running mean/variance\n");4647// ---------------------------------------------------------------------------48// Part 2: LayerNorm — Normalize over the feature dimension49// ---------------------------------------------------------------------------50console.log("--- Part 2: LayerNorm ---");5152// LayerNorm normalizes over the last dimension(s)53const ln = new LayerNorm(3);54console.log("LayerNorm(normalizedShape=3)");55console.log("  Normalizes each sample independently across features\n");5657const lnOut = ln.forward(bnInput);58const lnTensor = lnOut instanceof GradTensor ? lnOut.tensor : lnOut;59console.log(`Input shape:  [${bnInput.shape.join(", ")}]`);60console.log(`Output shape: [${lnTensor.shape.join(", ")}]`);61console.log("  LayerNorm is batch-size independent (used in Transformers)\n");6263// ---------------------------------------------------------------------------64// Part 3: Dropout — Regularization by random zeroing65// ---------------------------------------------------------------------------66console.log("--- Part 3: Dropout ---");6768const dropout = new Dropout(0.5);69console.log("Dropout(p=0.5) — drops 50% of elements during training");7071const dropInput = tensor([[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]);7273// Training mode: randomly zeros elements74dropout.train();75console.log("\nTraining mode:");76const dropOut1 = dropout.forward(dropInput);77const drop1 = dropOut1 instanceof GradTensor ? dropOut1.tensor : dropOut1;78console.log(`  Output: ${drop1.toString()}`);79console.log("  Surviving elements are scaled by 1/(1-p) = 2.0");8081// Eval mode: passes input unchanged82dropout.eval();83const dropOut2 = dropout.forward(dropInput);84const drop2 = dropOut2 instanceof GradTensor ? dropOut2.tensor : dropOut2;85console.log("\nEval mode:");86console.log(`  Output: ${drop2.toString()}`);87console.log("  Input passed through unchanged\n");8889// ---------------------------------------------------------------------------90// Part 4: Parameter counts91// ---------------------------------------------------------------------------92console.log("--- Part 4: Parameter Counts ---");93const bnParams = Array.from(bn.parameters()).length;94const lnParams = Array.from(ln.parameters()).length;95const dropParams = Array.from(dropout.parameters()).length;96console.log(`BatchNorm1d(3) params: ${bnParams} (gamma + beta)`);97console.log(`LayerNorm(3)   params: ${lnParams} (weight + bias)`);98console.log(`Dropout(0.5)   params: ${dropParams} (no learnable params)`);99100console.log("\n=== Normalization & Dropout Complete ===");101