Data Analysis & Visualization

1/**2 * Example 03: Data Analysis & Visualization3 *4 * Comprehensive data analysis workflow using DataFrames, statistics, and plotting.5 * Learn to explore, analyze, and visualize employee data.6 */78import { mkdirSync, writeFileSync } from "node:fs";9import { DataFrame } from "deepbox/dataframe";10import { tensor } from "deepbox/ndarray";11import { Figure } from "deepbox/plot";12import { corrcoef, mean, std } from "deepbox/stats";1314const expectNumber = (value: unknown): number => {15  if (typeof value !== "number") {16    throw new Error("Expected number");17  }18  return value;19};2021const expectNumberArray = (value: unknown): number[] => {22  if (!Array.isArray(value) || value.some((v) => typeof v !== "number")) {23    throw new Error("Expected number[]");24  }25  return value;26};2728console.log("=".repeat(60));29console.log("Example 03: Data Analysis & Visualization");30console.log("=".repeat(60));3132mkdirSync("docs/examples/03-data-analysis/output", { recursive: true });3334// Create a DataFrame with employee information35const employeeData = new DataFrame({36  name: [37    "Alice",38    "Bob",39    "Charlie",40    "David",41    "Eve",42    "Frank",43    "Grace",44    "Henry",45    "Ivy",46    "Jack",47    "Kate",48    "Leo",49    "Mia",50    "Noah",51    "Olivia",52    "Paul",53    "Quinn",54    "Rachel",55    "Sam",56    "Tina",57  ],58  department: [59    "Engineering",60    "Sales",61    "Engineering",62    "HR",63    "Engineering",64    "Sales",65    "Marketing",66    "Engineering",67    "HR",68    "Sales",69    "Engineering",70    "Marketing",71    "Sales",72    "Engineering",73    "HR",74    "Sales",75    "Engineering",76    "Marketing",77    "Engineering",78    "Sales",79  ],80  salary: [81    95000, 65000, 105000, 55000, 98000, 72000, 68000, 110000, 58000, 70000, 102000, 71000, 67000,82    115000, 60000, 69000, 108000, 73000, 112000, 66000,83  ],84  experience: [5, 3, 8, 2, 6, 4, 3, 10, 2, 5, 7, 4, 3, 12, 3, 4, 9, 5, 11, 3],85  age: [28, 25, 32, 24, 30, 27, 26, 35, 24, 29, 31, 28, 26, 38, 27, 28, 34, 30, 36, 26],86});8788// Display dataset overview89console.log("\n📊 Dataset Overview");90console.log("-".repeat(60));91console.log(`Total Employees: ${employeeData.shape[0]}`);92console.log(`Columns: ${employeeData.columns.join(", ")}`);9394// Show first few rows95console.log("\n📋 First 5 Rows:");96console.log(employeeData.head(5).toString());9798// Calculate descriptive statistics99console.log("\n📈 Statistical Summary");100console.log("-".repeat(60));101102// Extract columns as arrays for analysis103const salaries = expectNumberArray(employeeData.get("salary").toArray());104const experiences = expectNumberArray(employeeData.get("experience").toArray());105const ages = expectNumberArray(employeeData.get("age").toArray());106107// Convert arrays to tensors for statistical operations108const salaryTensor = tensor(salaries);109const expTensor = tensor(experiences);110111// Calculate salary statistics112const salaryMean = Number(mean(salaryTensor).data[0]);113const salarySd = Number(std(salaryTensor).data[0]);114115console.log(`Salary Statistics:`);116console.log(`  Mean: $${salaryMean.toFixed(2)}`);117console.log(`  Std Dev: $${salarySd.toFixed(2)}`);118console.log(`  Min: $${Math.min(...salaries)}`);119console.log(`  Max: $${Math.max(...salaries)}`);120121// Calculate experience statistics122const expMean = Number(mean(expTensor).data[0]);123const expSd = Number(std(expTensor).data[0]);124125console.log(`\nExperience Statistics:`);126console.log(`  Mean: ${expMean.toFixed(1)} years`);127console.log(`  Std Dev: ${expSd.toFixed(1)} years`);128129// Group by department and calculate averages130console.log("\n🏢 Department Analysis");131console.log("-".repeat(60));132133// GroupBy operation to aggregate by department134const deptGroups = employeeData.groupBy("department");135const deptStats = deptGroups.agg({136  salary: "mean",137  experience: "mean",138});139140console.log("Average Salary by Department:");141console.log(deptStats.toString());142143// Filter data based on conditions144console.log("\n🔍 Filtering Examples");145console.log("-".repeat(60));146147// Find employees earning over $100k148const highEarners = employeeData.filter((row) => expectNumber(row.salary) > 100000);149console.log(`High Earners (>$100k): ${highEarners.shape[0]} employees`);150console.log(highEarners.select(["name", "department", "salary"]).toString());151152// Filter by department153const engineeringDept = employeeData.filter((row) => row.department === "Engineering");154console.log(`\nEngineering Department: ${engineeringDept.shape[0]} employees`);155156// Analyze correlations between variables157console.log("\n📊 Correlation Analysis");158console.log("-".repeat(60));159160// corrcoef expects rows=observations and columns=variables.161const dataMatrix = tensor(162  salaries.map((salary, index) => [salary, experiences[index] ?? 0, ages[index] ?? 0])163);164const correlationMatrix = corrcoef(dataMatrix);165166console.log("Correlation Matrix (Salary, Experience, Age):");167console.log(correlationMatrix.toString());168169// Generate visualizations170console.log("\n🎨 Creating Visualizations");171console.log("-".repeat(60));172173// 1. Scatter plot showing relationship between experience and salary174console.log("1. Scatter Plot: Salary vs Experience");175const fig1 = new Figure();176const ax1 = fig1.addAxes();177ax1.scatter(expTensor, salaryTensor, { color: "#1f77b4", size: 8 });178ax1.setTitle("Salary vs Experience");179ax1.setXLabel("Years of Experience");180ax1.setYLabel("Salary ($)");181const svg1 = fig1.renderSVG();182writeFileSync("docs/examples/03-data-analysis/output/salary-vs-experience.svg", svg1.svg);183console.log("   ✓ Saved: output/salary-vs-experience.svg");184185// 2. Histogram showing salary distribution186console.log("2. Histogram: Salary Distribution");187const fig2 = new Figure();188const ax2 = fig2.addAxes();189ax2.hist(salaryTensor, 8, { color: "#2ca02c" });190ax2.setTitle("Salary Distribution");191ax2.setXLabel("Salary ($)");192ax2.setYLabel("Frequency");193const svg2 = fig2.renderSVG();194writeFileSync("docs/examples/03-data-analysis/output/salary-distribution.svg", svg2.svg);195console.log("   ✓ Saved: output/salary-distribution.svg");196197// 3. Bar chart comparing departments198console.log("3. Bar Chart: Average Salary by Department");199// Calculate average salary for each department200const depts = ["Engineering", "Sales", "Marketing", "HR"];201const avgSalaries = depts.map((dept) => {202  const deptData = employeeData.filter((row) => row.department === dept);203  const deptSalaries = expectNumberArray(deptData.get("salary").toArray());204  return Number(mean(tensor(deptSalaries)).data[0]);205});206207const fig3 = new Figure();208const ax3 = fig3.addAxes();209ax3.bar(tensor([0, 1, 2, 3]), tensor(avgSalaries), {210  color: "#ff7f0e",211  edgecolor: "#000000",212});213ax3.setTitle("Average Salary by Department");214ax3.setXLabel("Department");215ax3.setYLabel("Average Salary ($)");216const svg3 = fig3.renderSVG();217writeFileSync("docs/examples/03-data-analysis/output/dept-salaries.svg", svg3.svg);218console.log("   ✓ Saved: output/dept-salaries.svg");219220// 4. Heatmap visualizing correlations221console.log("4. Heatmap: Correlation Matrix");222const fig4 = new Figure();223const ax4 = fig4.addAxes();224ax4.heatmap(correlationMatrix, { vmin: -1, vmax: 1 });225ax4.setTitle("Correlation Matrix");226const svg4 = fig4.renderSVG();227writeFileSync("docs/examples/03-data-analysis/output/correlation-heatmap.svg", svg4.svg);228console.log("   ✓ Saved: output/correlation-heatmap.svg");229230// Summary of findings231console.log("\n💡 Key Insights");232console.log("-".repeat(60));233console.log("• Engineering has the highest average salary");234console.log("• Strong positive correlation between experience and salary");235console.log("• Age shows moderate correlation with both salary and experience");236console.log("• Salary distribution shows clustering around $70k and $105k");237238console.log("\n✅ Analysis Complete!");239console.log("=".repeat(60));240