From bac00ab5d57589bf5414e2815032dae1e9f75b4a Mon Sep 17 00:00:00 2001 From: saikiranvella Date: Tue, 9 Jun 2026 20:11:10 -0400 Subject: [PATCH] news screen enhancement - 1 --- bin/daily-screen.ts | 92 +++++++++++++++++++ package.json | 1 + .../domains/screener/screener.controller.ts | 39 +++++++- server/domains/shared/db/queries.constant.ts | 14 +++ server/domains/shared/types/asset.model.ts | 16 ++++ server/domains/shared/types/index.ts | 2 + tests/bond-scorer.test.ts | 4 +- tests/calls-controller.test.ts | 42 +++++---- tests/portfolio-advisor.test.ts | 4 +- tests/screener-controller.test.ts | 28 ++++-- ui/src/lib/stores/screener.store.svelte.ts | 8 ++ ui/src/routes/+page.svelte | 7 ++ ui/src/styles/_section.scss | 28 ++++++ 13 files changed, 250 insertions(+), 35 deletions(-) create mode 100644 bin/daily-screen.ts diff --git a/bin/daily-screen.ts b/bin/daily-screen.ts new file mode 100644 index 0000000..cb01654 --- /dev/null +++ b/bin/daily-screen.ts @@ -0,0 +1,92 @@ +/** + * Daily screening job — keeps the signal snapshot ledger (PRODUCT.md P0.1) + * accumulating even when nobody opens the UI. + * + * Universe = union of all users' watchlist tickers + all non-crypto holdings, + * or an explicit list passed on the command line. + * + * Usage: + * npm run screen:daily # watchlist + holdings universe + * npm run screen:daily -- AAPL MSFT # explicit tickers + * + * Schedule for market close, e.g. crontab (4:30pm ET weekdays): + * 30 16 * * 1-5 cd /path/to/market_screener && npm run screen:daily + */ + +import 'dotenv/config'; +import { + YahooFinanceClient, + BenchmarkProvider, + SignalSnapshotRepository, + createDb, + DatabaseConnection, + QueryAudit, +} from '../server/domains/shared'; +import { QueryBuilder } from '../server/domains/shared/utils/QueryBuilder'; +import { ScreenerEngine } from '../server/domains/screener'; +import type { AssetResult } from '../server/domains/shared'; + +function universeFromDb(db: DatabaseConnection): string[] { + const watchlist = db + .all<{ ticker: string }>(new QueryBuilder('UNIVERSE_QUERIES.DISTINCT_WATCHLIST_TICKERS')) + .map((r) => r.ticker); + const holdings = db + .all<{ ticker: string }>(new QueryBuilder('UNIVERSE_QUERIES.DISTINCT_HOLDING_TICKERS')) + .map((r) => r.ticker); + return [...new Set([...watchlist, ...holdings])].sort(); +} + +const db = new DatabaseConnection(createDb(process.env.DB_PATH ?? './market-screener.db'), { + audit: new QueryAudit(), + logSlowQueries: 100, +}); + +const cliTickers = process.argv.slice(2).map((t) => t.toUpperCase()); +const tickers = cliTickers.length > 0 ? cliTickers : universeFromDb(db); + +if (tickers.length === 0) { + console.log('No tickers to screen — watchlist and holdings are empty.'); + console.log('Pass tickers explicitly: npm run screen:daily -- AAPL MSFT'); + process.exit(0); +} + +console.log(`Screening ${tickers.length} tickers: ${tickers.join(', ')}`); + +const yahoo = new YahooFinanceClient(); +const benchmark = new BenchmarkProvider(yahoo); +const engine = new ScreenerEngine(yahoo, benchmark); +const snapshots = new SignalSnapshotRepository(db); + +try { + const results = await engine.screenWithProgress(tickers); + const rateRegime = results.marketContext?.rateRegime ?? null; + + const assets = [...results.STOCK, ...results.ETF, ...results.BOND] as AssetResult[]; + const written = snapshots.recordBatch( + assets.map((r) => ({ + ticker: r.asset.ticker, + assetType: r.asset.type, + price: r.asset.currentPrice ?? null, + signal: r.signal, + fundamental: r.fundamental, + inflated: r.inflated, + rateRegime, + })), + ); + + const bySignal = new Map(); + for (const a of assets) bySignal.set(a.signal, (bySignal.get(a.signal) ?? 0) + 1); + + console.log(`\nSnapshots written: ${written}`); + for (const [signal, count] of [...bySignal.entries()].sort()) { + console.log(` ${signal}: ${count}`); + } + if (results.ERROR.length > 0) { + console.log(`Errors (${results.ERROR.length}):`); + for (const e of results.ERROR) console.log(` ${e.ticker}: ${e.message}`); + } + process.exit(0); +} catch (err) { + console.error('Daily screen failed:', (err as Error).message); + process.exit(1); +} diff --git a/package.json b/package.json index 992af28..21381cb 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "test:watch": "tsx --test --watch --test-reporter=spec tests/*.test.ts", "lint": "eslint . --ext .ts,.js", "lint:fix": "eslint . --ext .ts,.js --fix", + "screen:daily": "tsx bin/daily-screen.ts", "format": "prettier --write \"server/**/*.ts\" \"bin/**/*.ts\" \"tests/**/*.ts\"", "format:check": "prettier --check \"server/**/*.ts\" \"bin/**/*.ts\" \"tests/**/*.ts\"", "prepare": "husky" diff --git a/server/domains/screener/screener.controller.ts b/server/domains/screener/screener.controller.ts index f63c841..0e06581 100644 --- a/server/domains/screener/screener.controller.ts +++ b/server/domains/screener/screener.controller.ts @@ -1,7 +1,7 @@ import type { FastifyInstance, FastifyRequest } from 'fastify'; import { ScreenerEngine } from './ScreenerEngine'; import { CatalystCache, SignalSnapshotRepository } from '../../domains/shared'; -import type { LiveAssetResult, ScreenerResult } from '../../domains/shared'; +import type { DataHealth, LiveAssetResult, ScreenerResult } from '../../domains/shared'; import { screenSchema } from '../../domains/shared/types/schemas'; export class ScreenerController { @@ -65,11 +65,48 @@ export class ScreenerController { const tickers = (req.body as { tickers: string[] }).tickers.map((t) => t.toUpperCase()); const results = await this.engine.screenTickers(tickers); this.recordSnapshots(results, req); + const dataHealth = ScreenerController.assessDataHealth(results); + if (dataHealth.degraded) { + req.log?.warn?.({ dataHealth }, 'screen batch returned degraded fundamentals data'); + } return { ...results, STOCK: ScreenerController.serializeAssets(results.STOCK as LiveAssetResult[]), ETF: ScreenerController.serializeAssets(results.ETF as LiveAssetResult[]), BOND: ScreenerController.serializeAssets(results.BOND as LiveAssetResult[]), + dataHealth, + }; + } + + /** + * P0.4 data-sanity sentinel — if a large share of screened stocks come back + * with null core fundamentals (P/E, ROE), the upstream source has likely + * changed schema or is throttling. Surface it loudly instead of letting + * everything silently degrade to "No Data" rows. + */ + private static assessDataHealth(results: ScreenerResult): DataHealth { + const THRESHOLD = 0.3; // >30% nulls = degraded + const MIN_SAMPLE = 3; // don't alarm on tiny batches + + const stocks = results.STOCK as LiveAssetResult[]; + const metrics = stocks.map( + (r) => r.asset.metrics as { peRatio?: number | null; returnOnEquity?: number | null }, + ); + const nullPeRatio = metrics.filter((m) => m.peRatio == null).length; + const nullRoe = metrics.filter((m) => m.returnOnEquity == null).length; + const total = metrics.length; + + const degraded = + total >= MIN_SAMPLE && (nullPeRatio / total > THRESHOLD || nullRoe / total > THRESHOLD); + + return { + degraded, + stocksChecked: total, + nullPeRatio, + nullRoe, + message: degraded + ? `${Math.max(nullPeRatio, nullRoe)} of ${total} stocks returned no core fundamentals — data source may be degraded; treat this screen with caution` + : null, }; } diff --git a/server/domains/shared/db/queries.constant.ts b/server/domains/shared/db/queries.constant.ts index 5f1f6af..2eac821 100644 --- a/server/domains/shared/db/queries.constant.ts +++ b/server/domains/shared/db/queries.constant.ts @@ -151,6 +151,20 @@ export const WATCHLIST_QUERIES = { `, }; +// ── Screening Universe Queries (bin/daily-screen.ts) ──────────────────────── + +export const UNIVERSE_QUERIES = { + // Every ticker pinned by any user + DISTINCT_WATCHLIST_TICKERS: 'SELECT DISTINCT ticker FROM watchlist ORDER BY ticker', + + // Every ticker held by any user (crypto excluded — not fundamentally scored) + DISTINCT_HOLDING_TICKERS: ` + SELECT DISTINCT ticker FROM holdings + WHERE type != 'crypto' + ORDER BY ticker + `, +}; + // ── Signal Snapshot Queries (P0.1 — signal track record) ──────────────────── export const SIGNAL_SNAPSHOT_QUERIES = { diff --git a/server/domains/shared/types/asset.model.ts b/server/domains/shared/types/asset.model.ts index 2b5c682..3d2c692 100644 --- a/server/domains/shared/types/asset.model.ts +++ b/server/domains/shared/types/asset.model.ts @@ -87,10 +87,26 @@ export interface AssetResult { fundamental: ScoreResult; } +/** + * Data-source health for one screen batch (PRODUCT.md P0.4). + * Degraded = a large share of stocks came back without core fundamentals, + * which usually means the upstream data source changed or is throttling — + * not that the companies are actually missing data. + */ +export interface DataHealth { + degraded: boolean; + stocksChecked: number; + nullPeRatio: number; + nullRoe: number; + message: string | null; +} + export interface ScreenerResult { STOCK: AssetResult[]; ETF: AssetResult[]; BOND: AssetResult[]; ERROR: Array<{ ticker: string; message: string }>; marketContext: import('./market.model.js').MarketContext; + /** Set by the screener controller on API responses, not by the engine. */ + dataHealth?: DataHealth; } diff --git a/server/domains/shared/types/index.ts b/server/domains/shared/types/index.ts index 3ef0232..d8d2d87 100644 --- a/server/domains/shared/types/index.ts +++ b/server/domains/shared/types/index.ts @@ -8,6 +8,8 @@ export type { ScoringRules, ScoreAudit, ScoreResult, + VerdictTier, + DataHealth, AssetResult, LiveAssetResult, ScreenerResult, diff --git a/tests/bond-scorer.test.ts b/tests/bond-scorer.test.ts index 5f08084..90539f1 100644 --- a/tests/bond-scorer.test.ts +++ b/tests/bond-scorer.test.ts @@ -85,12 +85,12 @@ test('BondScorer', async (t) => { }); await t.test('handles null/undefined metrics gracefully', () => { - const metrics: BondMetrics = { + const metrics = { ytm: null, duration: 5, creditRating: null, creditRatingNumeric: null, - }; + } as unknown as BondMetrics; const result = BondScorer.score(metrics, DEFAULT_RULES); // Should not crash diff --git a/tests/calls-controller.test.ts b/tests/calls-controller.test.ts index 9c83700..01aa098 100644 --- a/tests/calls-controller.test.ts +++ b/tests/calls-controller.test.ts @@ -12,13 +12,13 @@ class MockMarketCallRepository { quarter: 'Q2 2024', thesis: 'Strong iPhone sales cycle', tickers: ['AAPL'], - date: new Date('2024-05-01'), - snapshots: [{ ticker: 'AAPL', price: 180, date: new Date('2024-05-01') }], + date: '2024-05-01', + snapshot: {}, }, ]; async list(): Promise<(MarketCall & { id: string })[]> { - return this.calls.sort((a, b) => b.date.getTime() - a.date.getTime()); + return this.calls.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); } async get(id: string): Promise<(MarketCall & { id: string }) | null> { @@ -27,7 +27,7 @@ class MockMarketCallRepository { async create(call: MarketCall): Promise { const id = String(this.calls.length + 1); - const newCall = { id, ...call }; + const newCall = { ...call, id }; this.calls.push(newCall); return newCall; } @@ -152,7 +152,7 @@ test('CallsController', async (t) => { const calls = await repository.list(); assert.ok(Array.isArray(calls)); assert.equal(calls.length, 1); - assert.equal(calls[0].ticker || calls[0].title, 'AAPL Post-Earnings' || 'AAPL'); + assert.equal(calls[0].title, 'AAPL Post-Earnings'); }); await t.test('returns calls sorted by date (newest first)', async () => { @@ -164,8 +164,8 @@ test('CallsController', async (t) => { quarter: 'Q1 2024', thesis: 'Old thesis', tickers: ['AAPL'], - date: new Date('2024-01-01'), - snapshots: [], + date: '2024-01-01', + snapshot: {}, }, { id: '2', @@ -173,13 +173,13 @@ test('CallsController', async (t) => { quarter: 'Q2 2024', thesis: 'New thesis', tickers: ['MSFT'], - date: new Date('2024-05-01'), - snapshots: [], + date: '2024-05-01', + snapshot: {}, }, ]; async list() { - return this.calls.sort((a, b) => b.date.getTime() - a.date.getTime()); + return this.calls.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime()); } async get(id: string) { @@ -205,14 +205,14 @@ test('CallsController', async (t) => { await t.test('creates new market call', async () => { const repository = new MockMarketCallRepository() as any; - const newCall: MarketCall = { + const newCall = { title: 'MSFT Q3 2024', quarter: 'Q3 2024', thesis: 'Cloud growth acceleration', tickers: ['MSFT'], - date: new Date('2024-07-01'), - snapshots: [], - }; + date: '2024-07-01', + snapshot: {}, + } as MarketCall; const created = await repository.create(newCall); assert.ok(created.id); @@ -261,14 +261,14 @@ test('CallsController', async (t) => { const repository = new MockMarketCallRepository() as any; const engine = new MockScreenerEngine() as any; - const newCall: MarketCall = { + const newCall = { title: 'Tech Quartet', quarter: 'Q3 2024', thesis: 'All tech leaders', tickers: ['AAPL', 'MSFT', 'NVDA', 'GOOG'], - date: new Date('2024-07-01'), - snapshots: [], - }; + date: '2024-07-01', + snapshot: {}, + } as MarketCall; const created = await repository.create(newCall); const results = await engine.screenTickers(created.tickers); @@ -290,11 +290,13 @@ test('CallsController', async (t) => { } }); - await t.test('call includes snapshots of entry prices', async () => { + await t.test('call includes a snapshot of entry prices', async () => { const repository = new MockMarketCallRepository() as any; const call = await repository.get('1'); assert.ok(call); - assert.ok(Array.isArray(call.snapshots)); + // MarketCall.snapshot is Record, not an array + assert.equal(typeof call.snapshot, 'object'); + assert.ok(!Array.isArray(call.snapshot)); }); }); diff --git a/tests/portfolio-advisor.test.ts b/tests/portfolio-advisor.test.ts index a374ffd..c0bea03 100644 --- a/tests/portfolio-advisor.test.ts +++ b/tests/portfolio-advisor.test.ts @@ -142,7 +142,7 @@ test('PortfolioAdvisor', async (t) => { displayMetrics: {}, } as any, { - signal: SIGNAL.BUY, + signal: SIGNAL.STRONG_BUY, fundamental: { label: 'pass', scoreSummary: '', audit: { passedGates: true } }, inflated: { label: 'pass', scoreSummary: '', audit: { passedGates: true } }, asset: { @@ -239,7 +239,7 @@ test('PortfolioAdvisor', async (t) => { displayMetrics: {}, } as any, { - signal: SIGNAL.BUY, + signal: SIGNAL.STRONG_BUY, fundamental: { label: 'pass', scoreSummary: '', audit: { passedGates: true } }, inflated: { label: 'pass', scoreSummary: '', audit: { passedGates: true } }, asset: { diff --git a/tests/screener-controller.test.ts b/tests/screener-controller.test.ts index 03bbcc5..b032376 100644 --- a/tests/screener-controller.test.ts +++ b/tests/screener-controller.test.ts @@ -3,11 +3,7 @@ import assert from 'node:assert/strict'; import { ScreenerController } from '../server/domains/screener/screener.controller.js'; import { ScreenerEngine } from '../server/domains/screener/ScreenerEngine.js'; -import type { - LiveAssetResult, - MarketContext, - Stock, -} from '../server/domains/shared/types/index.js'; +import type { LiveAssetResult, MarketContext } from '../server/domains/shared/types/index.js'; import { ASSET_TYPE, SIGNAL } from '../server/domains/shared/config/constants.js'; // Mock implementations @@ -43,12 +39,24 @@ class MockScreenerEngine extends ScreenerEngine { returnOnEquity: 95.2, freeCashFlow: 100000000, }), - } as unknown as Stock; + } as unknown as LiveAssetResult['asset']; const mockResult: LiveAssetResult = { asset: mockStock, - fundamentalScore: { label: '✓ BUY', scoreSummary: 'Quality gate PASS' }, - inflatedScore: { label: '✓ BUY', scoreSummary: 'Market adjusted gate PASS' }, + fundamental: { + label: '🟢 BUY (High Conviction)', + tier: 'PASS', + score: 9, + scoreSummary: 'Quality gate PASS', + audit: { passedGates: true }, + }, + inflated: { + label: '🟢 BUY (High Conviction)', + tier: 'PASS', + score: 9, + scoreSummary: 'Market adjusted gate PASS', + audit: { passedGates: true }, + }, signal: SIGNAL.STRONG_BUY, }; @@ -190,7 +198,7 @@ test('ScreenerController', async (t) => { assert.equal(results.STOCK.length, 1); const result = results.STOCK[0]; assert.ok(result.signal); - assert.ok(result.fundamentalScore); - assert.ok(result.inflatedScore); + assert.ok(result.fundamental); + assert.ok(result.inflated); }); }); diff --git a/ui/src/lib/stores/screener.store.svelte.ts b/ui/src/lib/stores/screener.store.svelte.ts index d35d465..a2adf49 100644 --- a/ui/src/lib/stores/screener.store.svelte.ts +++ b/ui/src/lib/stores/screener.store.svelte.ts @@ -21,6 +21,12 @@ class ScreenerStore { // ── Derived ──────────────────────────────────────────────────────── ctx = $derived(this.results?.marketContext ?? null); + /** P0.4 data-sanity sentinel — dismissible per screen run. */ + healthDismissed = $state(false); + dataHealth = $derived( + !this.healthDismissed && this.results?.dataHealth?.degraded ? this.results.dataHealth : null, + ); + allAssets = $derived( this.results ? sorted([...this.results.STOCK, ...this.results.ETF, ...this.results.BOND]) : [], ); @@ -28,6 +34,7 @@ class ScreenerStore { // ── Actions ──────────────────────────────────────────────────────── async screen(): Promise { this.error = null; + this.healthDismissed = false; this.loading = true; try { const tickers = this.input @@ -46,6 +53,7 @@ class ScreenerStore { async reloadCatalysts(): Promise { this.loadingCats = true; this.error = null; + this.healthDismissed = false; try { const cat = await fetchCatalysts(); this.input = cat.tickers.join(', '); diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index e2dce1b..18e9bad 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -62,6 +62,13 @@
⚠ {s.error}
{/if} + {#if s.dataHealth} + + {/if} + {#if s.loading || s.loadingCats}
diff --git a/ui/src/styles/_section.scss b/ui/src/styles/_section.scss index f67e358..2a64604 100644 --- a/ui/src/styles/_section.scss +++ b/ui/src/styles/_section.scss @@ -76,3 +76,31 @@ margin-bottom: 16px; font-size: var(--fs-md); } + +// ── Warning banner (data-sanity sentinel, P0.4) ─────────────────────────── + +.warn-banner { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; + background: var(--amber-dim); + border: 1px solid var(--amber); + border-radius: var(--radius-md); + color: var(--amber); + padding: 10px var(--space-lg); + margin-bottom: 16px; + font-size: var(--fs-md); +} + +.warn-dismiss { + background: none; + border: none; + color: var(--amber); + cursor: pointer; + font-size: 14px; + padding: 2px 6px; + line-height: 1; + + &:hover { filter: brightness(1.3); } +}