Experimentation Platform

1/**2 * Experimentation Platform3 *4 * A production-style experimentation workflow for Deepbox v1.0.0 that combines5 * DataFrame reporting, confidence intervals, bootstrap uplift analysis,6 * multiple-comparison correction, KDE diagnostics, and sample-size planning.7 */89import { mkdir, writeFile } from "node:fs/promises";10import { DataFrame } from "deepbox/dataframe";11import { tensor } from "deepbox/ndarray";12import { axhline, figure, groupedBar, kdeplot, legend, saveFig } from "deepbox/plot";13import { Generator } from "deepbox/random";14import {15  benjaminiHochberg,16  bootstrap,17  cohenD,18  meanConfidenceInterval,19  meanConfidenceIntervalZ,20  meanDiffConfidenceInterval,21  proportionConfidenceInterval,22  tTestPower,23  ttest_ind,24} from "deepbox/stats";2526const OUTPUT_DIR = "docs/projects/09-experimentation-platform/output";27const TOTAL_SESSIONS = 3600;28const RANDOM_SEED = 20260327;2930type Variant = "control" | "streamlined-checkout" | "smart-bundle";31type Device = "mobile" | "desktop";32type Segment = "self-serve" | "mid-market" | "enterprise";33type Region = "gcc" | "europe" | "north-america";3435type SessionRecord = {36  readonly variant: Variant;37  readonly device: Device;38  readonly segment: Segment;39  readonly region: Region;40  readonly converted: number;41  readonly retained7d: number;42  readonly revenuePerSession: number;43  readonly orderValue: number;44  readonly latencyMs: number;45};4647const rng = new Generator(RANDOM_SEED);4849function choose<T>(values: readonly T[]): T {50  return values[rng.randint(0, values.length)] ?? values[0]!;51}5253function clampProbability(value: number): number {54  return Math.max(0.001, Math.min(0.98, value));55}5657function generateSessions(count: number): SessionRecord[] {58  const variants: readonly Variant[] = ["control", "streamlined-checkout", "smart-bundle"];59  const devices: readonly Device[] = ["mobile", "desktop"];60  const segments: readonly Segment[] = ["self-serve", "mid-market", "enterprise"];61  const regions: readonly Region[] = ["gcc", "europe", "north-america"];6263  const sessions: SessionRecord[] = [];6465  for (let i = 0; i < count; i++) {66    const variant = variants[i % variants.length] ?? "control";67    const device = choose(devices);68    const segment = choose(segments);69    const region = choose(regions);7071    const baseConversion =72      segment === "enterprise" ? 0.12 : segment === "mid-market" ? 0.085 : 0.06;73    const devicePenalty = device === "mobile" ? -0.012 : 0;74    const regionAdjustment = region === "gcc" ? 0.004 : region === "north-america" ? 0.002 : -0.001;75    const variantAdjustment =76      variant === "streamlined-checkout"77        ? device === "mobile"78          ? 0.0279          : 0.01380        : variant === "smart-bundle"81          ? 0.00882          : 0;8384    const conversionProbability = clampProbability(85      baseConversion + devicePenalty + regionAdjustment + variantAdjustment86    );87    const converted = rng.random() < conversionProbability ? 1 : 0;8889    const latencyBase = device === "mobile" ? 1360 : 1020;90    const segmentLatency = segment === "enterprise" ? 40 : segment === "self-serve" ? -25 : 0;91    const variantLatencyShift =92      variant === "streamlined-checkout" ? -120 : variant === "smart-bundle" ? 45 : 0;93    const latencyMs = Math.max(94      680,95      rng.normal(latencyBase + segmentLatency + variantLatencyShift, 55)96    );9798    const orderValueBase = segment === "enterprise" ? 340 : segment === "mid-market" ? 220 : 95;99    const bundleLift =100      variant === "smart-bundle" ? 42 : variant === "streamlined-checkout" ? 16 : 0;101    const orderValue = converted102      ? Math.max(45, rng.normal(orderValueBase + bundleLift, orderValueBase * 0.18))103      : 0;104105    const retentionBase = segment === "enterprise" ? 0.74 : segment === "mid-market" ? 0.62 : 0.48;106    const retentionShift =107      variant === "streamlined-checkout" ? 0.02 : variant === "smart-bundle" ? 0.035 : 0;108    const retained7d =109      converted && rng.random() < clampProbability(retentionBase + retentionShift) ? 1 : 0;110111    sessions.push({112      variant,113      device,114      segment,115      region,116      converted,117      retained7d,118      revenuePerSession: Number(orderValue.toFixed(2)),119      orderValue: Number(orderValue.toFixed(2)),120      latencyMs: Number(latencyMs.toFixed(2)),121    });122  }123124  return sessions;125}126127function sessionsForVariant(sessions: readonly SessionRecord[], variant: Variant): SessionRecord[] {128  return sessions.filter((session) => session.variant === variant);129}130131function numericValues(132  sessions: readonly SessionRecord[],133  selector: (session: SessionRecord) => number134): number[] {135  return sessions.map(selector);136}137138function mean(values: readonly number[]): number {139  return values.reduce((sum, value) => sum + value, 0) / values.length;140}141142function sum(values: readonly number[]): number {143  return values.reduce((accumulator, value) => accumulator + value, 0);144}145146function buildVariantSummary(147  variant: Variant,148  sessions: readonly SessionRecord[]149): {150  readonly variant: Variant;151  readonly sessions: number;152  readonly conversionRate: number;153  readonly conversionCi: ReturnType<typeof proportionConfidenceInterval>;154  readonly revenueMean: number;155  readonly revenueCi: ReturnType<typeof meanConfidenceInterval>;156  readonly retentionRate: number;157  readonly retentionCi: ReturnType<typeof proportionConfidenceInterval>;158  readonly latencyCi: ReturnType<typeof meanConfidenceIntervalZ>;159} {160  const conversions = numericValues(sessions, (session) => session.converted);161  const retention = numericValues(sessions, (session) => session.retained7d);162  const revenue = numericValues(sessions, (session) => session.revenuePerSession);163  const latency = numericValues(sessions, (session) => session.latencyMs);164165  return {166    variant,167    sessions: sessions.length,168    conversionRate: mean(conversions),169    conversionCi: proportionConfidenceInterval(sum(conversions), sessions.length, 0.95),170    revenueMean: mean(revenue),171    revenueCi: meanConfidenceInterval(revenue, 0.95),172    retentionRate: mean(retention),173    retentionCi: proportionConfidenceInterval(sum(retention), sessions.length, 0.95),174    latencyCi: meanConfidenceIntervalZ(latency, 55, 0.95),175  };176}177178type PairwiseInference = {179  readonly variant: Exclude<Variant, "control">;180  readonly revenuePvalue: number;181  readonly revenueCorrected: number;182  readonly revenueDiffCi: ReturnType<typeof meanDiffConfidenceInterval>;183  readonly latencyPvalue: number;184  readonly latencyCorrected: number;185};186187console.log("═".repeat(72));188console.log("  EXPERIMENTATION PLATFORM");189console.log("  Deepbox v1.0.0 production example");190console.log("═".repeat(72));191192await mkdir(OUTPUT_DIR, { recursive: true });193194const sessions = generateSessions(TOTAL_SESSIONS);195const experimentFrame = new DataFrame({196  variant: sessions.map((session) => session.variant),197  device: sessions.map((session) => session.device),198  segment: sessions.map((session) => session.segment),199  region: sessions.map((session) => session.region),200  converted: sessions.map((session) => session.converted),201  retained7d: sessions.map((session) => session.retained7d),202  revenuePerSession: sessions.map((session) => session.revenuePerSession),203  orderValue: sessions.map((session) => session.orderValue),204  latencyMs: sessions.map((session) => session.latencyMs),205});206207// ============================================================================208// Step 1: Operational summary209// ============================================================================210console.log("\n📊 STEP 1: Experiment Operations Summary");211console.log("─".repeat(72));212console.log(`Sessions generated: ${sessions.length}`);213console.log("Variant-level numeric means:");214console.log(experimentFrame.groupBy("variant").mean().toString());215console.log("\nVariant × device latency means:");216console.log(experimentFrame.groupBy(["variant", "device"]).mean().toString());217218// ============================================================================219// Step 2: Variant scorecards220// ============================================================================221console.log("\n🧾 STEP 2: Variant Scorecards");222console.log("─".repeat(72));223224const controlSessions = sessionsForVariant(sessions, "control");225const streamlinedSessions = sessionsForVariant(sessions, "streamlined-checkout");226const bundleSessions = sessionsForVariant(sessions, "smart-bundle");227228const variantSummaries = [229  buildVariantSummary("control", controlSessions),230  buildVariantSummary("streamlined-checkout", streamlinedSessions),231  buildVariantSummary("smart-bundle", bundleSessions),232];233234for (const summary of variantSummaries) {235  console.log(236    `${summary.variant.padEnd(21)} conv=${(summary.conversionRate * 100).toFixed(2)}% revenue/session=${summary.revenueMean.toFixed(2)} retention=${(summary.retentionRate * 100).toFixed(2)}% latency=${summary.latencyCi.mean.toFixed(1)}ms`237  );238}239240// ============================================================================241// Step 3: Pairwise inference and correction242// ============================================================================243console.log("\n🧪 STEP 3: Pairwise Inference");244console.log("─".repeat(72));245246const pairwiseCandidates = [247  {248    variant: "streamlined-checkout" as const,249    revenue: numericValues(streamlinedSessions, (session) => session.revenuePerSession),250    latency: numericValues(streamlinedSessions, (session) => session.latencyMs),251  },252  {253    variant: "smart-bundle" as const,254    revenue: numericValues(bundleSessions, (session) => session.revenuePerSession),255    latency: numericValues(bundleSessions, (session) => session.latencyMs),256  },257];258const controlRevenue = numericValues(controlSessions, (session) => session.revenuePerSession);259const controlLatency = numericValues(controlSessions, (session) => session.latencyMs);260261const rawRevenuePvalues = pairwiseCandidates.map(262  (candidate) => ttest_ind(tensor(controlRevenue), tensor(candidate.revenue)).pvalue263);264const rawLatencyPvalues = pairwiseCandidates.map(265  (candidate) => ttest_ind(tensor(controlLatency), tensor(candidate.latency)).pvalue266);267268const revenueCorrection = benjaminiHochberg(rawRevenuePvalues, 0.05);269const latencyCorrection = benjaminiHochberg(rawLatencyPvalues, 0.05);270271const pairwiseInference: PairwiseInference[] = pairwiseCandidates.map((candidate, index) => ({272  variant: candidate.variant,273  revenuePvalue: rawRevenuePvalues[index] ?? 1,274  revenueCorrected: revenueCorrection.corrected[index] ?? 1,275  revenueDiffCi: meanDiffConfidenceInterval(candidate.revenue, controlRevenue, 0.95),276  latencyPvalue: rawLatencyPvalues[index] ?? 1,277  latencyCorrected: latencyCorrection.corrected[index] ?? 1,278}));279280for (const result of pairwiseInference) {281  console.log(282    `${result.variant.padEnd(21)} revenue p=${result.revenuePvalue.toFixed(6)} -> ${result.revenueCorrected.toFixed(6)} | latency p=${result.latencyPvalue.toFixed(6)} -> ${result.latencyCorrected.toFixed(6)}`283  );284}285286const winner =287  variantSummaries.slice().sort((left, right) => right.revenueMean - left.revenueMean)[0]288    ?.variant ?? "control";289console.log(`Selected winner by revenue/session: ${winner}`);290291// ============================================================================292// Step 4: Bootstrap uplift and power planning293// ============================================================================294console.log("\n🎯 STEP 4: Decision Support");295console.log("─".repeat(72));296297const winnerSessions = sessionsForVariant(sessions, winner);298const winnerRevenue = numericValues(winnerSessions, (session) => session.revenuePerSession);299300const cellKeys = Array.from(301  new Set(sessions.map((session) => `${session.segment}:${session.device}`))302);303const cellUplifts = cellKeys.map((key) => {304  const [segment, device] = key.split(":") as [Segment, Device];305  const controlCell = sessions.filter(306    (session) =>307      session.variant === "control" && session.segment === segment && session.device === device308  );309  const winnerCell = sessions.filter(310    (session) =>311      session.variant === winner && session.segment === segment && session.device === device312  );313  return (314    mean(numericValues(winnerCell, (session) => session.revenuePerSession)) -315    mean(numericValues(controlCell, (session) => session.revenuePerSession))316  );317});318319const upliftBootstrap = bootstrap(cellUplifts, (sample) => mean(sample), {320  nResamples: 5000,321  seed: RANDOM_SEED,322  confidenceLevel: 0.95,323});324const observedEffectSize = Math.abs(cohenD(controlRevenue, winnerRevenue));325const currentPower = tTestPower({326  effectSize: observedEffectSize,327  nObs: controlRevenue.length,328  alpha: 0.05,329});330const requiredSample = tTestPower({331  effectSize: observedEffectSize,332  alpha: 0.05,333  power: 0.9,334});335336console.log(337  `Bootstrap revenue uplift vs control: ${upliftBootstrap.estimate.toFixed(2)} | 95% CI [${upliftBootstrap.ci[0].toFixed(2)}, ${upliftBootstrap.ci[1].toFixed(2)}]`338);339console.log(`Observed |Cohen's d|: ${observedEffectSize.toFixed(3)}`);340console.log(`Current power:         ${currentPower.power.toFixed(3)}`);341console.log(`Needed per arm for 90% power: ${requiredSample.nObs}`);342343// ============================================================================344// Step 5: Persist scorecards and plots345// ============================================================================346console.log("\n💾 STEP 5: Reports and Artifacts");347console.log("─".repeat(72));348349const summaryPayload = variantSummaries.map((summary) => ({350  variant: summary.variant,351  sessions: summary.sessions,352  conversionRate: summary.conversionRate,353  conversionCi: summary.conversionCi,354  revenueMean: summary.revenueMean,355  revenueCi: summary.revenueCi,356  retentionRate: summary.retentionRate,357  retentionCi: summary.retentionCi,358  latencyCi: summary.latencyCi,359}));360await writeFile(361  `${OUTPUT_DIR}/variant-scorecard.json`,362  JSON.stringify(summaryPayload, null, 2),363  "utf-8"364);365366const upliftBootstrapForReport = {367  estimate: upliftBootstrap.estimate,368  ci: upliftBootstrap.ci,369  nResamples: upliftBootstrap.samples.length,370};371372await writeFile(373  `${OUTPUT_DIR}/decision-report.json`,374  JSON.stringify(375    {376      winner,377      pairwiseInference,378      upliftBootstrap: upliftBootstrapForReport,379      currentPower,380      requiredSample,381    },382    null,383    2384  ),385  "utf-8"386);387388const rateFigure = figure({ width: 860, height: 520 });389groupedBar(390  tensor([1, 2, 3]),391  [392    tensor(variantSummaries.map((summary) => summary.conversionRate)),393    tensor(variantSummaries.map((summary) => summary.retentionRate)),394  ],395  {396    colors: ["#2563eb", "#059669"],397    labels: ["conversion rate", "7d retention rate"],398  }399);400legend();401await saveFig(`${OUTPUT_DIR}/variant-rates.svg`, { figure: rateFigure });402403const densityFigure = figure({ width: 860, height: 520 });404kdeplot(405  tensor(406    numericValues(controlSessions, (session) => session.orderValue).filter((value) => value > 0)407  ),408  { color: "#1d4ed8", label: "control order values", bw_method: "silverman" }409);410kdeplot(411  tensor(412    numericValues(winnerSessions, (session) => session.orderValue).filter((value) => value > 0)413  ),414  { color: "#f97316", label: `${winner} order values`, bw_method: "silverman" }415);416legend();417await saveFig(`${OUTPUT_DIR}/winner-order-value-density.svg`, {418  figure: densityFigure,419});420421const significanceFigure = figure({ width: 860, height: 520 });422groupedBar(423  tensor([1, 2]),424  [tensor(rawRevenuePvalues), tensor(Array.from(revenueCorrection.corrected))],425  {426    colors: ["#7c3aed", "#f43f5e"],427    labels: ["raw revenue p-values", "BH corrected"],428  }429);430axhline(0.05, { color: "#991b1b", linewidth: 2, label: "alpha = 0.05" });431legend();432await saveFig(`${OUTPUT_DIR}/revenue-significance.svg`, {433  figure: significanceFigure,434});435436console.log(`Saved variant scorecard:      ${OUTPUT_DIR}/variant-scorecard.json`);437console.log(`Saved decision report:        ${OUTPUT_DIR}/decision-report.json`);438console.log(`Saved grouped rate chart:     ${OUTPUT_DIR}/variant-rates.svg`);439console.log(`Saved order-value density:    ${OUTPUT_DIR}/winner-order-value-density.svg`);440console.log(`Saved significance chart:     ${OUTPUT_DIR}/revenue-significance.svg`);441442console.log("\n✅ Experimentation Platform Complete!");443