phase-8g: add sqllite.

2026-06-05 23:34:25 -04:00
parent 447a86b46e
commit 83116baa3c
20 changed files with 2514 additions and 239 deletions
@@ -0,0 +1,464 @@
+# Integration Example: Hardened Database Layer
+
+This document shows **step-by-step** how to integrate the new QueryBuilder + DatabaseConnection + QueryAudit into your existing codebase.
+
+## Step 1: Update `server/app.ts`
+
+Change from passing raw `Db` to passing `DatabaseConnection`:
+
+### Before
+
+```typescript
+import { createDb, type Db } from './db/index.js';
+import { MarketCallRepository } from './repositories/MarketCallRepository.js';
+import { PortfolioRepository } from './repositories/PortfolioRepository.js';
+
+export async function buildApp(): Promise<FastifyInstance> {
+  const app = fastify();
+  const rawDb: Db = createDb();
+
+  // Pass raw Db to repositories
+  const callsRepo = new MarketCallRepository(rawDb);
+  const portfolioRepo = new PortfolioRepository(rawDb);
+
+  // Register routes...
+  return app;
+}
+```
+
+### After
+
+```typescript
+import BetterSqlite3 from 'better-sqlite3';
+import { createDb, DatabaseConnection, QueryAudit } from './db/index.js';
+import { MarketCallRepository } from './repositories/MarketCallRepository.js';
+import { PortfolioRepository } from './repositories/PortfolioRepository.js';
+
+export async function buildApp(): Promise<FastifyInstance> {
+  const app = fastify();
+
+  // Create the raw database and initialize schema
+  const rawDb = createDb();
+
+  // Wrap with audit and caching
+  const audit = new QueryAudit((entry) => {
+    // Optional: send to logging service
+    if (process.env.LOG_SLOW_QUERIES && entry.durationMs > 100) {
+      console.warn(`[SLOW] ${entry.sql} (${entry.durationMs.toFixed(1)}ms)`);
+    }
+  });
+
+  const db = new DatabaseConnection(rawDb, {
+    audit,
+    logSlowQueries: 100, // warn on >100ms queries
+  });
+
+  // Pass DatabaseConnection to repositories (not raw Db)
+  const callsRepo = new MarketCallRepository(db);
+  const portfolioRepo = new PortfolioRepository(db);
+
+  // Register routes...
+  return app;
+}
+```
+
+## Step 2: Update `MarketCallRepository`
+
+Refactor to use QueryBuilder and DatabaseConnection:
+
+### Complete Refactored Repository
+
+```typescript
+import { randomUUID } from 'crypto';
+import { DatabaseConnection, QueryBuilder } from '../db/index.js';
+import type { MarketCall, CreateCallInput } from '../types/index.js';
+
+interface CallRow {
+  id:         string;
+  title:      string;
+  quarter:    string;
+  date:       string;
+  thesis:     string;
+  tickers:    string;  // JSON
+  snapshot:   string;  // JSON
+  created_at: string;
+}
+
+export class MarketCallRepository {
+  constructor(private readonly db: DatabaseConnection) {}
+
+  /**
+   * List all market calls, newest first.
+   */
+  list(): (MarketCall & { createdAt: string })[] {
+    const qb = new QueryBuilder('market_calls')
+      .select(['id', 'title', 'quarter', 'date', 'thesis', 'tickers', 'snapshot', 'created_at'])
+      .orderBy('created_at', 'DESC');
+
+    const rows = this.db.all<CallRow>(qb);
+    return rows.map(MarketCallRepository.toCall);
+  }
+
+  /**
+   * Get a single market call by ID.
+   */
+  get(id: string): (MarketCall & { createdAt: string }) | null {
+    const qb = new QueryBuilder('market_calls')
+      .select(['id', 'title', 'quarter', 'date', 'thesis', 'tickers', 'snapshot', 'created_at'])
+      .where('id = ?', [id]);
+
+    const row = this.db.get<CallRow>(qb);
+    return row ? MarketCallRepository.toCall(row) : null;
+  }
+
+  /**
+   * Create a new market call with snapshot of current prices.
+   */
+  create({
+    title, quarter, date, thesis, tickers, snapshot,
+  }: CreateCallInput): MarketCall & { createdAt: string } {
+    const call = {
+      id:        randomUUID(),
+      title,
+      quarter,
+      date:      date ?? new Date().toISOString().slice(0, 10),
+      thesis,
+      tickers:   tickers ?? [],
+      snapshot:  snapshot ?? {},
+      createdAt: new Date().toISOString(),
+    };
+
+    const qb = new QueryBuilder('market_calls')
+      .insert(
+        ['id', 'title', 'quarter', 'date', 'thesis', 'tickers', 'snapshot', 'created_at'],
+        [
+          call.id,
+          call.title,
+          call.quarter,
+          call.date,
+          call.thesis,
+          JSON.stringify(call.tickers),
+          JSON.stringify(call.snapshot),
+          call.createdAt,
+        ],
+      );
+
+    this.db.run(qb);
+    return call as MarketCall & { createdAt: string };
+  }
+
+  /**
+   * Delete a market call by ID.
+   * Returns true if the call existed and was deleted, false otherwise.
+   */
+  delete(id: string): boolean {
+    const qb = new QueryBuilder('market_calls')
+      .delete()
+      .where('id = ?', [id]);
+
+    const changes = this.db.run(qb);
+    return changes > 0;
+  }
+
+  /**
+   * Private helper to convert database row to domain object.
+   */
+  private static toCall(row: CallRow): MarketCall & { createdAt: string } {
+    return {
+      id:        row.id,
+      title:     row.title,
+      quarter:   row.quarter,
+      date:      row.date,
+      thesis:    row.thesis,
+      tickers:   JSON.parse(row.tickers),
+      snapshot:  JSON.parse(row.snapshot),
+      createdAt: row.created_at,
+    } as MarketCall & { createdAt: string };
+  }
+}
+```
+
+**Changes:**
+
+- Constructor now accepts `DatabaseConnection` instead of raw `Db`
+- All `.prepare()` calls replaced with `QueryBuilder` + `db.all()` / `db.get()` / `db.run()`
+- Explicit column selection in `SELECT` statements
+- Audit trail automatically generated for every query
+
+## Step 3: Update `PortfolioRepository`
+
+Similar refactoring:
+
+```typescript
+import { DatabaseConnection, QueryBuilder } from '../db/index.js';
+import type { PortfolioData, PortfolioHolding } from '../types/index.js';
+
+interface HoldingRow {
+  ticker:     string;
+  shares:     number;
+  cost_basis: number;
+  type:       string;
+  source:     string;
+}
+
+export class PortfolioRepository {
+  constructor(private readonly db: DatabaseConnection) {}
+
+  /**
+   * Check if portfolio has any holdings.
+   */
+  exists(): boolean {
+    const qb = new QueryBuilder('holdings')
+      .select(['ticker'])
+      .limit(1);
+
+    const row = this.db.get<{ ticker: string }>(qb);
+    return row !== null;
+  }
+
+  /**
+   * Read all holdings.
+   */
+  read(): PortfolioData {
+    const qb = new QueryBuilder('holdings')
+      .select(['ticker', 'shares', 'cost_basis', 'type', 'source'])
+      .orderBy('ticker', 'ASC');
+
+    const rows = this.db.all<HoldingRow>(qb);
+    return { holdings: rows.map(PortfolioRepository.toHolding) };
+  }
+
+  /**
+   * Insert or update a holding.
+   */
+  upsert(entry: PortfolioHolding): PortfolioHolding {
+    const ticker = entry.ticker.toUpperCase().trim();
+
+    // Use raw db.prepare() for UPSERT syntax (not yet wrapped in QueryBuilder)
+    // This is acceptable because the values are all parameterized
+    this.db.raw().prepare(
+      `INSERT INTO holdings (ticker, shares, cost_basis, type, source)
+       VALUES (?, ?, ?, ?, ?)
+       ON CONFLICT(ticker) DO UPDATE SET
+         shares     = excluded.shares,
+         cost_basis = excluded.cost_basis,
+         type       = excluded.type,
+         source     = excluded.source`,
+    ).run(ticker, entry.shares, entry.costBasis ?? 0, entry.type ?? 'stock', entry.source ?? 'Manual');
+
+    return { ...entry, ticker };
+  }
+
+  /**
+   * Delete a holding by ticker.
+   */
+  remove(ticker: string): boolean {
+    const qb = new QueryBuilder('holdings')
+      .delete()
+      .where('ticker = ?', [ticker.toUpperCase()]);
+
+    const changes = this.db.run(qb);
+    return changes > 0;
+  }
+
+  /**
+   * Private helper to convert database row to domain object.
+   */
+  private static toHolding(row: HoldingRow): PortfolioHolding {
+    return {
+      ticker:    row.ticker,
+      shares:    row.shares,
+      costBasis: row.cost_basis,
+      type:      row.type as PortfolioHolding['type'],
+      source:    row.source,
+    };
+  }
+}
+```
+
+**Note:** The `upsert()` method still uses `db.raw().prepare()` because QueryBuilder doesn't yet support `ON CONFLICT`. This is acceptable because the SQL is still parameterized. A future enhancement could add `onConflict()` to QueryBuilder if needed.
+
+## Step 4: Add a Simple Test
+
+Create `tests/MarketCallRepository.test.ts`:
+
+```typescript
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import BetterSqlite3 from 'better-sqlite3';
+import { DatabaseConnection, QueryAudit } from '../server/db/index.js';
+import { MarketCallRepository } from '../server/repositories/MarketCallRepository.js';
+
+// Mini DDL for testing
+const DDL = `
+  CREATE TABLE market_calls (
+    id         TEXT PRIMARY KEY,
+    title      TEXT NOT NULL,
+    quarter    TEXT NOT NULL,
+    date       TEXT NOT NULL,
+    thesis     TEXT NOT NULL,
+    tickers    TEXT NOT NULL,
+    snapshot   TEXT NOT NULL,
+    created_at TEXT NOT NULL
+  );
+`;
+
+test('MarketCallRepository', async (t) => {
+  // Set up in-memory database
+  const rawDb = new BetterSqlite3(':memory:');
+  rawDb.exec(DDL);
+
+  const audit = new QueryAudit();
+  const db = new DatabaseConnection(rawDb, { audit });
+  const repo = new MarketCallRepository(db);
+
+  await t.test('should create and retrieve a call', () => {
+    const created = repo.create({
+      title: 'Q4 Earnings Blitz',
+      quarter: 'Q4',
+      thesis: 'Mega cap tech breakout',
+      tickers: ['AAPL', 'MSFT', 'GOOGL'],
+    });
+
+    assert.ok(created.id);
+    assert.equal(created.title, 'Q4 Earnings Blitz');
+
+    const retrieved = repo.get(created.id);
+    assert.deepEqual(retrieved, created);
+  });
+
+  await t.test('should list calls in order', () => {
+    const call1 = repo.create({
+      title: 'Call 1',
+      quarter: 'Q1',
+      thesis: 'Test 1',
+      tickers: ['AAPL'],
+    });
+
+    const call2 = repo.create({
+      title: 'Call 2',
+      quarter: 'Q2',
+      thesis: 'Test 2',
+      tickers: ['MSFT'],
+    });
+
+    const list = repo.list();
+    assert.equal(list.length, 2);
+    // Most recent first
+    assert.equal(list[0].id, call2.id);
+    assert.equal(list[1].id, call1.id);
+  });
+
+  await t.test('should delete a call', () => {
+    const call = repo.create({
+      title: 'Deletable',
+      quarter: 'Q1',
+      thesis: 'This will be deleted',
+      tickers: ['TEST'],
+    });
+
+    assert.ok(repo.delete(call.id));
+    assert.equal(repo.get(call.id), null);
+    assert.ok(!repo.delete(call.id)); // Already deleted
+  });
+
+  await t.test('should track queries in audit', () => {
+    repo.create({
+      title: 'Audited',
+      quarter: 'Q1',
+      thesis: 'Tracked',
+      tickers: ['AAPL'],
+    });
+
+    const auditLog = audit.all();
+    assert.ok(auditLog.length > 0);
+
+    // Find the INSERT
+    const inserts = audit.byAction('WRITE');
+    assert.ok(inserts.some(e => e.sql.includes('INSERT INTO market_calls')));
+  });
+});
+```
+
+Run it:
+
+```bash
+npm test -- tests/MarketCallRepository.test.ts
+```
+
+## Step 5: Add to Existing Tests
+
+If you already have integration tests, add an audit check:
+
+```typescript
+test('screening creates an audit trail', async (t) => {
+  const result = await app.inject({
+    method: 'POST',
+    url: '/api/screen',
+    payload: { tickers: ['AAPL', 'MSFT'] },
+  });
+
+  assert.equal(result.statusCode, 200);
+
+  // Verify the database was accessed
+  const audit = db.getAudit();
+  const reads = audit.byAction('READ');
+  assert.ok(reads.length > 0, 'SELECT queries should have been executed');
+});
+```
+
+## Step 6: Enable Audit Output (Optional)
+
+If you want to log all queries to a file or external service:
+
+```typescript
+import fs from 'fs/promises';
+
+const audit = new QueryAudit(async (entry) => {
+  // Only log WRITE operations to a log file
+  if (entry.action !== 'READ') {
+    const logLine = JSON.stringify({
+      timestamp: entry.timestamp,
+      action: entry.action,
+      sql: entry.sql,
+      params: entry.params,
+      rowsAffected: entry.rowsAffected,
+      error: entry.error,
+    });
+
+    await fs.appendFile('./audit.log', logLine + '\n');
+  }
+});
+
+const db = new DatabaseConnection(rawDb, { audit, logSlowQueries: 50 });
+```
+
+Then tail the log:
+
+```bash
+tail -f audit.log | jq .
+```
+
+---
+
+## Summary
+
+| File | Change |
+|------|--------|
+| `server/app.ts` | Create `DatabaseConnection` and pass to repositories |
+| `server/repositories/MarketCallRepository.ts` | Use `QueryBuilder` + `DatabaseConnection` |
+| `server/repositories/PortfolioRepository.ts` | Use `QueryBuilder` + `DatabaseConnection` |
+| `tests/MarketCallRepository.test.ts` | Add tests with audit verification |
+
+All changes maintain **backward compatibility** with the existing API — only the internals change. Your controllers don't need to be modified.
+
+## Next: Audit Trail in Production
+
+Once deployed, you can:
+
+1. **Review recent queries**: `db.getAudit().recent(100)`
+2. **Find slow queries**: `db.getAudit().all().filter(e => e.durationMs > 500)`
+3. **Track mutations**: `db.getAudit().byAction('WRITE')`
+4. **Generate compliance reports**: `db.printAudit()`
+
+This gives you visibility into exactly what's happening in your database — invaluable for debugging, security audits, and performance optimization.