panopticon init

src/index.ts

@@ -0,0 +1,584 @@
+#!/usr/bin/env node
+
+/**
+ * Panopticon — Automated Project Documentation Registry
+ *
+ * Usage:
+ *   node dist/index.js                          # Incremental update (all projects)
+ *   node dist/index.js --full-analysis <name>   # Full analysis for one project
+ *   node dist/index.js --project <name>         # Incremental update for one project
+ */
+
+import { resolve } from "path";
+import { loadConfig } from "./config.js";
+import { getCurrentSha, pull, getFileTree, getDiffSince } from "./git.js";
+import { gatherStructuralContext, hashFileAtPath } from "./structural.js";
+import { loadState, saveState } from "./state.js";
+import { readAllSkillFiles, readSkillFile, writeSkillFiles, countLinesChanged, cleanSkillDir } from "./writer.js";
+import { runIncrementalOrchestrator, runFullOrchestrator } from "./orchestrator.js";
+import { runIncrementalWorker, runFullAnalysisWorker, runWorkersConcurrently } from "./worker.js";
+import { runSynthesizer } from "./synthesizer.js";
+import { pushMetrics, buildProjectMetrics } from "./metrics.js";
+import { writeProjectReport, writeNightlyReport, detectAnomalies } from "./reporter.js";
+import type {
+  Config,
+  ProjectConfig,
+  ProjectRunReport,
+  NightlyReport,
+  OrchestratorResult,
+  WorkerResult,
+  PhaseTimings,
+  SessionMetrics,
+} from "./types.js";
+import { join } from "path";
+
+// ── CLI Parsing ──
+
+interface CliArgs {
+  fullAnalysis: string | null;
+  project: string | null;
+  configPath: string | null;
+  dryRun: boolean;
+}
+
+function parseArgs(): CliArgs {
+  const args = process.argv.slice(2);
+  const result: CliArgs = {
+    fullAnalysis: null,
+    project: null,
+    configPath: null,
+    dryRun: false,
+  };
+
+  for (let i = 0; i < args.length; i++) {
+    switch (args[i]) {
+      case "--full-analysis":
+        result.fullAnalysis = args[++i] ?? null;
+        break;
+      case "--project":
+        result.project = args[++i] ?? null;
+        break;
+      case "--config":
+        result.configPath = args[++i] ?? null;
+        break;
+      case "--dry-run":
+        result.dryRun = true;
+        break;
+    }
+  }
+
+  return result;
+}
+
+// ── Main Pipeline ──
+
+async function main() {
+  const cliArgs = parseArgs();
+  const config = loadConfig(cliArgs.configPath ?? undefined);
+  const date = new Date().toISOString().split("T")[0];
+
+  console.log(`Panopticon run: ${date}`);
+  console.log(`Projects configured: ${config.projects.map((p) => p.name).join(", ")}`);
+
+  if (cliArgs.fullAnalysis) {
+    // Full analysis mode
+    const project = config.projects.find((p) => p.name === cliArgs.fullAnalysis);
+    if (!project) {
+      console.error(`Project not found: ${cliArgs.fullAnalysis}`);
+      console.error(`Available: ${config.projects.map((p) => p.name).join(", ")}`);
+      process.exit(1);
+    }
+
+    console.log(`\nRunning full analysis for: ${project.name}`);
+    const report = await runFullAnalysis(project, config, cliArgs.dryRun);
+    writeProjectReport(resolve(config.runsDir), date, report);
+
+    if (config.metrics.enabled) {
+      const metrics = buildProjectMetrics(report, config.metrics.jobLabel);
+      await pushMetrics(config.metrics, metrics);
+    }
+
+    console.log(`\nFull analysis complete. Status: ${report.status}`);
+    return;
+  }
+
+  // Incremental mode (nightly)
+  const projectsToRun = cliArgs.project
+    ? config.projects.filter((p) => p.name === cliArgs.project)
+    : config.projects;
+
+  if (projectsToRun.length === 0) {
+    console.error(`No matching projects found`);
+    process.exit(1);
+  }
+
+  const nightlyStart = Date.now();
+  const projectReports: ProjectRunReport[] = [];
+
+  for (const project of projectsToRun) {
+    console.log(`\nProcessing: ${project.name}`);
+    const report = await runIncremental(project, config, cliArgs.dryRun);
+    projectReports.push(report);
+    console.log(`  Status: ${report.status} (${(report.duration / 1000).toFixed(1)}s)`);
+  }
+
+  // Nightly report
+  const anomalies = detectAnomalies(projectReports);
+  const nightlyReport: NightlyReport = {
+    date,
+    projectReports,
+    totalDuration: Date.now() - nightlyStart,
+    totalCost: projectReports.reduce((sum, r) => sum + r.estimatedCost, 0),
+    anomalies,
+  };
+
+  writeNightlyReport(resolve(config.runsDir), date, nightlyReport);
+
+  // Push metrics for all projects
+  if (config.metrics.enabled) {
+    for (const report of projectReports) {
+      const metrics = buildProjectMetrics(report, config.metrics.jobLabel);
+      await pushMetrics(config.metrics, metrics);
+    }
+  }
+
+  console.log(`\nNightly run complete.`);
+  console.log(`  Duration: ${(nightlyReport.totalDuration / 1000).toFixed(1)}s`);
+  console.log(`  Cost: ~$${nightlyReport.totalCost.toFixed(3)}`);
+  if (anomalies.length > 0) {
+    console.log(`  Anomalies: ${anomalies.length}`);
+    for (const a of anomalies) console.log(`    - ${a}`);
+  }
+}
+
+// ── Full Analysis Pipeline ──
+
+async function runFullAnalysis(
+  project: ProjectConfig,
+  config: Config,
+  dryRun: boolean
+): Promise<ProjectRunReport> {
+  const runStart = Date.now();
+  const timings: PhaseTimings = { git: 0, structural: 0, orchestrator: 0, workers: 0, synthesizer: 0 };
+  const errors: string[] = [];
+  const stateDir = resolve(config.stateDir);
+
+  // Phase: Git
+  let gitStart = Date.now();
+  let currentSha: string;
+  try {
+    currentSha = getCurrentSha(project.path);
+  } catch (err: any) {
+    errors.push(`Git error: ${err.message}`);
+    return makeFailedReport(project.name, 0, errors, timings, Date.now() - runStart);
+  }
+  timings.git = Date.now() - gitStart;
+
+  // Phase: Structural extraction
+  let structStart = Date.now();
+  const structural = gatherStructuralContext(project.path, project);
+  timings.structural = Date.now() - structStart;
+
+  console.log(`  Files: ${structural.fileTree.length}, AST entries: ${structural.astSummaries.length}`);
+
+  // Phase: Orchestrator
+  let orchStart = Date.now();
+  const { result: orchResult, metrics: orchMetrics } = await runFullOrchestrator(
+    project.path,
+    config,
+    structural
+  );
+  timings.orchestrator = Date.now() - orchStart;
+  errors.push(...orchMetrics.errors);
+
+  console.log(`  Orchestrator: ${orchResult.updates.length} work units planned`);
+
+  // Phase: Workers
+  let workerStart = Date.now();
+  const workerTasks = orchResult.updates.map((workUnit) => {
+    return () =>
+      runFullAnalysisWorker(
+        project.path,
+        config,
+        workUnit.target as "structure.md" | "guide.md" | "changelog.md",
+        structural,
+        workUnit
+      );
+  });
+
+  const workerResults = await runWorkersConcurrently(workerTasks, config.limits.maxWorkerConcurrency);
+  timings.workers = Date.now() - workerStart;
+
+  for (const w of workerResults) {
+    console.log(`  Worker ${w.target}: ${w.status}${w.error ? ` (${w.error})` : ""}`);
+    if (w.metrics.errors.length > 0) errors.push(...w.metrics.errors);
+  }
+
+  // Phase: Synthesizer
+  let synthStart = Date.now();
+  const currentSkillMd = readSkillFile(project.path, "SKILL.md");
+  const { result: synthResult, metrics: synthMetrics } = await runSynthesizer(
+    project.path,
+    config,
+    project.name,
+    workerResults,
+    currentSkillMd
+  );
+  timings.synthesizer = Date.now() - synthStart;
+  errors.push(...synthMetrics.errors);
+
+  // Write outputs
+  if (!dryRun) {
+    const existingDocs = readAllSkillFiles(project.path);
+
+    // Wipe all generated .md files (except SKILL.md) before writing fresh ones.
+    // This removes stale documents from previous runs (e.g. after file renames).
+    const removed = cleanSkillDir(project.path);
+    if (removed.length > 0) {
+      console.log(`  Cleaned stale docs: ${removed.join(", ")}`);
+    }
+
+    const filesToWrite: Record<string, string> = {};
+
+    filesToWrite["SKILL.md"] = synthResult.skillMd;
+    for (const w of workerResults) {
+      if (w.status === "success") {
+        filesToWrite[w.target] = w.content;
+      }
+    }
+
+    // Apply synthesizer fixes
+    for (const fix of synthResult.fixes) {
+      if (filesToWrite[fix.file] && fix.before && fix.after) {
+        filesToWrite[fix.file] = filesToWrite[fix.file].replace(fix.before, fix.after);
+      }
+    }
+
+    writeSkillFiles(project.path, filesToWrite);
+
+    // Update state
+    const fileHashes: Record<string, string> = {};
+    for (const file of structural.fileTree) {
+      try {
+        fileHashes[file] = hashFileAtPath(join(project.path, file));
+      } catch { /* skip */ }
+    }
+
+    saveState(stateDir, project.name, {
+      lastSha: currentSha,
+      lastRunTimestamp: new Date().toISOString(),
+      lastRunStatus: errors.length === 0 ? "success" : "partial",
+      fileHashes,
+      docVersions: Object.fromEntries(
+        Object.entries(filesToWrite).map(([k, v]) => [k, String(v.length)])
+      ),
+    });
+
+    // Doc line changes
+    const docLinesChanged: Record<string, number> = {};
+    for (const [file, content] of Object.entries(filesToWrite)) {
+      docLinesChanged[file] = countLinesChanged(existingDocs[file] ?? null, content);
+    }
+
+    const totalTokensIn =
+      orchMetrics.tokensIn + synthMetrics.tokensIn + workerResults.reduce((s, w) => s + w.metrics.tokensIn, 0);
+    const totalTokensOut =
+      orchMetrics.tokensOut + synthMetrics.tokensOut + workerResults.reduce((s, w) => s + w.metrics.tokensOut, 0);
+
+    return {
+      project: project.name,
+      date: new Date().toISOString(),
+      status: errors.length === 0 ? "success" : "partial",
+      duration: Date.now() - runStart,
+      commitCount: 0,
+      filesChanged: structural.fileTree.length,
+      insertions: 0,
+      deletions: 0,
+      orchestratorDecision: orchResult,
+      workerResults,
+      synthesizerStatus: synthMetrics.errors.length === 0 ? "success" : "failure",
+      docLinesChanged,
+      phaseTimings: timings,
+      totalTokensIn,
+      totalTokensOut,
+      estimatedCost: estimateCost(totalTokensIn, totalTokensOut),
+      errors,
+    };
+  }
+
+  // Dry run
+  return makeFailedReport(project.name, 0, ["dry-run"], timings, Date.now() - runStart);
+}
+
+// ── Incremental Pipeline ──
+
+async function runIncremental(
+  project: ProjectConfig,
+  config: Config,
+  dryRun: boolean
+): Promise<ProjectRunReport> {
+  const runStart = Date.now();
+  const timings: PhaseTimings = { git: 0, structural: 0, orchestrator: 0, workers: 0, synthesizer: 0 };
+  const errors: string[] = [];
+  const stateDir = resolve(config.stateDir);
+
+  // Load state
+  const state = loadState(stateDir, project.name);
+
+  // Phase: Git
+  let gitStart = Date.now();
+  let currentSha: string;
+  try {
+    // Try to pull (non-fatal if fails for local repos)
+    try { pull(project.path, project.branch); } catch { /* local repo, no remote */ }
+    currentSha = getCurrentSha(project.path);
+  } catch (err: any) {
+    errors.push(`Git error: ${err.message}`);
+    timings.git = Date.now() - gitStart;
+    return makeFailedReport(project.name, 0, errors, timings, Date.now() - runStart);
+  }
+  timings.git = Date.now() - gitStart;
+
+  // Check if there are changes
+  if (state.lastSha && state.lastSha === currentSha) {
+    console.log(`  No changes since last run (${currentSha.slice(0, 8)})`);
+    return makeSkippedReport(project.name, timings, Date.now() - runStart);
+  }
+
+  // If no previous state, recommend full analysis
+  if (!state.lastSha) {
+    console.log(`  No previous state — running full analysis instead`);
+    return runFullAnalysis(project, config, dryRun);
+  }
+
+  // Get diff
+  let diff;
+  try {
+    diff = getDiffSince(project.path, state.lastSha, project);
+  } catch (err: any) {
+    errors.push(`Diff error: ${err.message}`);
+    // Fall back to full analysis
+    console.log(`  Diff failed, falling back to full analysis`);
+    return runFullAnalysis(project, config, dryRun);
+  }
+
+  if (diff.filesChanged.length === 0) {
+    console.log(`  No file changes detected`);
+    return makeSkippedReport(project.name, timings, Date.now() - runStart);
+  }
+
+  console.log(`  Changes: ${diff.commitCount} commits, ${diff.filesChanged.length} files (+${diff.insertions}/-${diff.deletions})`);
+
+  // Phase: Structural (lightweight for incremental)
+  let structStart = Date.now();
+  const fileTree = getFileTree(project.path, project);
+  timings.structural = Date.now() - structStart;
+
+  // Phase: Orchestrator
+  let orchStart = Date.now();
+  const existingDocs = readAllSkillFiles(project.path);
+  const { result: orchResult, metrics: orchMetrics } = await runIncrementalOrchestrator(
+    project.path,
+    config,
+    diff,
+    existingDocs,
+    fileTree
+  );
+  timings.orchestrator = Date.now() - orchStart;
+  errors.push(...orchMetrics.errors);
+
+  if (orchResult.skipReason) {
+    console.log(`  Orchestrator: skip (${orchResult.skipReason})`);
+    // Still update state
+    if (!dryRun) {
+      saveState(stateDir, project.name, {
+        ...state,
+        lastSha: currentSha,
+        lastRunTimestamp: new Date().toISOString(),
+        lastRunStatus: "success",
+      });
+    }
+    return makeSkippedReport(project.name, timings, Date.now() - runStart);
+  }
+
+  console.log(`  Orchestrator: ${orchResult.updates.map((u) => u.target).join(", ")}`);
+
+  // Phase: Workers
+  let workerStart = Date.now();
+  const workerTasks = orchResult.updates.map((workUnit) => {
+    return () =>
+      runIncrementalWorker(
+        project.path,
+        config,
+        workUnit,
+        existingDocs[workUnit.target] ?? null,
+        diff.diffContent
+      );
+  });
+
+  const workerResults = await runWorkersConcurrently(workerTasks, config.limits.maxWorkerConcurrency);
+  timings.workers = Date.now() - workerStart;
+
+  for (const w of workerResults) {
+    console.log(`  Worker ${w.target}: ${w.status}${w.error ? ` (${w.error})` : ""}`);
+    if (w.metrics.errors.length > 0) errors.push(...w.metrics.errors);
+  }
+
+  // Phase: Synthesizer
+  let synthStart = Date.now();
+  const currentSkillMd = readSkillFile(project.path, "SKILL.md");
+  const successfulWorkers = workerResults.filter((w) => w.status === "success");
+
+  let synthResult;
+  let synthMetrics: SessionMetrics = { tokensIn: 0, tokensOut: 0, toolCalls: 0, errors: [], durationMs: 0 };
+
+  if (successfulWorkers.length > 0) {
+    const synthResponse = await runSynthesizer(
+      project.path,
+      config,
+      project.name,
+      workerResults,
+      currentSkillMd
+    );
+    synthResult = synthResponse.result;
+    synthMetrics = synthResponse.metrics;
+    errors.push(...synthMetrics.errors);
+  }
+  timings.synthesizer = Date.now() - synthStart;
+
+  // Write outputs
+  if (!dryRun && successfulWorkers.length > 0) {
+    const filesToWrite: Record<string, string> = {};
+
+    if (synthResult) {
+      filesToWrite["SKILL.md"] = synthResult.skillMd;
+      // Apply fixes
+      for (const w of workerResults) {
+        if (w.status === "success") {
+          filesToWrite[w.target] = w.content;
+        }
+      }
+      for (const fix of synthResult.fixes) {
+        if (filesToWrite[fix.file] && fix.before && fix.after) {
+          filesToWrite[fix.file] = filesToWrite[fix.file].replace(fix.before, fix.after);
+        }
+      }
+    } else {
+      // No synthesizer — write worker outputs directly
+      for (const w of workerResults) {
+        if (w.status === "success") {
+          filesToWrite[w.target] = w.content;
+        }
+      }
+    }
+
+    writeSkillFiles(project.path, filesToWrite);
+
+    // Update state
+    saveState(stateDir, project.name, {
+      ...state,
+      lastSha: currentSha,
+      lastRunTimestamp: new Date().toISOString(),
+      lastRunStatus: errors.length === 0 ? "success" : "partial",
+    });
+
+    // Doc line changes
+    const docLinesChanged: Record<string, number> = {};
+    for (const [file, content] of Object.entries(filesToWrite)) {
+      docLinesChanged[file] = countLinesChanged(existingDocs[file] ?? null, content);
+    }
+
+    const totalTokensIn =
+      orchMetrics.tokensIn + synthMetrics.tokensIn + workerResults.reduce((s, w) => s + w.metrics.tokensIn, 0);
+    const totalTokensOut =
+      orchMetrics.tokensOut + synthMetrics.tokensOut + workerResults.reduce((s, w) => s + w.metrics.tokensOut, 0);
+
+    return {
+      project: project.name,
+      date: new Date().toISOString(),
+      status: workerResults.some((w) => w.status === "failure") ? "partial" : "success",
+      duration: Date.now() - runStart,
+      commitCount: diff.commitCount,
+      filesChanged: diff.filesChanged.length,
+      insertions: diff.insertions,
+      deletions: diff.deletions,
+      orchestratorDecision: orchResult,
+      workerResults,
+      synthesizerStatus: synthMetrics.errors.length === 0 ? "success" : "failure",
+      docLinesChanged,
+      phaseTimings: timings,
+      totalTokensIn,
+      totalTokensOut,
+      estimatedCost: estimateCost(totalTokensIn, totalTokensOut),
+      errors,
+    };
+  }
+
+  return makeFailedReport(project.name, diff.commitCount, errors, timings, Date.now() - runStart);
+}
+
+// ── Helpers ──
+
+function estimateCost(tokensIn: number, tokensOut: number): number {
+  // Rough estimate based on Anthropic pricing
+  // Sonnet: $3/Minput, $15/Moutput; Haiku: $0.80/Minput, $4/Moutput
+  // Average it out roughly
+  return (tokensIn * 2.0 + tokensOut * 10.0) / 1_000_000;
+}
+
+function makeSkippedReport(project: string, timings: PhaseTimings, duration: number): ProjectRunReport {
+  return {
+    project,
+    date: new Date().toISOString(),
+    status: "skipped",
+    duration,
+    commitCount: 0,
+    filesChanged: 0,
+    insertions: 0,
+    deletions: 0,
+    orchestratorDecision: null,
+    workerResults: [],
+    synthesizerStatus: "skipped",
+    docLinesChanged: {},
+    phaseTimings: timings,
+    totalTokensIn: 0,
+    totalTokensOut: 0,
+    estimatedCost: 0,
+    errors: [],
+  };
+}
+
+function makeFailedReport(
+  project: string,
+  commitCount: number,
+  errors: string[],
+  timings: PhaseTimings,
+  duration: number
+): ProjectRunReport {
+  return {
+    project,
+    date: new Date().toISOString(),
+    status: "failure",
+    duration,
+    commitCount,
+    filesChanged: 0,
+    insertions: 0,
+    deletions: 0,
+    orchestratorDecision: null,
+    workerResults: [],
+    synthesizerStatus: "skipped",
+    docLinesChanged: {},
+    phaseTimings: timings,
+    totalTokensIn: 0,
+    totalTokensOut: 0,
+    estimatedCost: 0,
+    errors,
+  };
+}
+
+// ── Entry Point ──
+
+main().catch((err) => {
+  console.error("Fatal error:", err);
+  process.exit(1);
+});