Learning Rate Schedulers

1/**2 * Example 16: Learning Rate Schedulers3 *4 * Control the learning rate during training for better convergence.5 * Deepbox provides 8 learning rate schedulers.6 */78import { Linear, ReLU, Sequential } from "deepbox/nn";9import {10  Adam,11  CosineAnnealingLR,12  ExponentialLR,13  LinearLR,14  MultiStepLR,15  OneCycleLR,16  ReduceLROnPlateau,17  StepLR,18  WarmupLR,19} from "deepbox/optim";2021console.log("=== Learning Rate Schedulers ===\n");2223// Create a small model and optimizer for demonstration24const createOptimizer = () => {25  const model = new Sequential(new Linear(4, 8), new ReLU(), new Linear(8, 1));26  return new Adam(model.parameters(), { lr: 0.1 });27};2829// ---------------------------------------------------------------------------30// Part 1: StepLR — decay every N steps31// ---------------------------------------------------------------------------32console.log("--- Part 1: StepLR ---");3334const opt1 = createOptimizer();35const stepLR = new StepLR(opt1, { stepSize: 3, gamma: 0.5 });3637for (let epoch = 0; epoch < 10; epoch++) {38  const lr = stepLR.getLastLr()[0] ?? 0;39  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);40  stepLR.step();41}4243// ---------------------------------------------------------------------------44// Part 2: MultiStepLR — decay at specific milestones45// ---------------------------------------------------------------------------46console.log("\n--- Part 2: MultiStepLR ---");4748const opt2 = createOptimizer();49const multiStepLR = new MultiStepLR(opt2, {50  milestones: [3, 6, 8],51  gamma: 0.5,52});5354for (let epoch = 0; epoch < 10; epoch++) {55  const lr = multiStepLR.getLastLr()[0] ?? 0;56  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);57  multiStepLR.step();58}5960// ---------------------------------------------------------------------------61// Part 3: ExponentialLR — exponential decay each epoch62// ---------------------------------------------------------------------------63console.log("\n--- Part 3: ExponentialLR ---");6465const opt3 = createOptimizer();66const expLR = new ExponentialLR(opt3, { gamma: 0.9 });6768for (let epoch = 0; epoch < 10; epoch++) {69  const lr = expLR.getLastLr()[0] ?? 0;70  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);71  expLR.step();72}7374// ---------------------------------------------------------------------------75// Part 4: CosineAnnealingLR — cosine annealing76// ---------------------------------------------------------------------------77console.log("\n--- Part 4: CosineAnnealingLR ---");7879const opt4 = createOptimizer();80const cosineLR = new CosineAnnealingLR(opt4, { T_max: 10, etaMin: 0.001 });8182for (let epoch = 0; epoch < 10; epoch++) {83  const lr = cosineLR.getLastLr()[0] ?? 0;84  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);85  cosineLR.step();86}8788// ---------------------------------------------------------------------------89// Part 5: LinearLR — linear warmup / decay90// ---------------------------------------------------------------------------91console.log("\n--- Part 5: LinearLR ---");9293const opt5 = createOptimizer();94const linearLR = new LinearLR(opt5, {95  startFactor: 0.1,96  endFactor: 1.0,97  totalIters: 5,98});99100for (let epoch = 0; epoch < 8; epoch++) {101  const lr = linearLR.getLastLr()[0] ?? 0;102  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);103  linearLR.step();104}105106// ---------------------------------------------------------------------------107// Part 6: ReduceLROnPlateau — reduce when metric stops improving108// ---------------------------------------------------------------------------109console.log("\n--- Part 6: ReduceLROnPlateau ---");110111const opt6 = createOptimizer();112const plateauLR = new ReduceLROnPlateau(opt6, { factor: 0.5, patience: 2 });113114// Simulate a training loop where loss plateaus115const fakeLosses = [1.0, 0.8, 0.6, 0.59, 0.58, 0.58, 0.58, 0.3, 0.29, 0.29];116for (let epoch = 0; epoch < fakeLosses.length; epoch++) {117  const loss = fakeLosses[epoch];118  plateauLR.step(loss);119  console.log(120    `  Epoch ${epoch}: loss = ${loss.toFixed(2)}, lr = ${plateauLR.getLastLr()[0]?.toFixed(6)}`121  );122}123124// ---------------------------------------------------------------------------125// Part 7: WarmupLR — linear warmup then constant126// ---------------------------------------------------------------------------127console.log("\n--- Part 7: WarmupLR ---");128129const opt7 = createOptimizer();130const warmupLR = new WarmupLR(opt7, null, { warmupEpochs: 5 });131132for (let epoch = 0; epoch < 8; epoch++) {133  const lr = warmupLR.getLastLr()[0] ?? 0;134  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);135  warmupLR.step();136}137138// ---------------------------------------------------------------------------139// Part 8: OneCycleLR — super-convergence schedule140// ---------------------------------------------------------------------------141console.log("\n--- Part 8: OneCycleLR ---");142143const opt8 = createOptimizer();144const oneCycleLR = new OneCycleLR(opt8, { maxLr: 0.1, totalSteps: 10 });145146for (let epoch = 0; epoch < 10; epoch++) {147  const lr = oneCycleLR.getLastLr()[0] ?? 0;148  console.log(`  Epoch ${epoch}: lr = ${lr.toFixed(6)}`);149  oneCycleLR.step();150}151152console.log("\n=== Learning Rate Schedulers Complete ===");153