diff --git a/CLAUDE.md b/CLAUDE.md index 2796574..18dfe48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,6 +8,40 @@ Guidance for working in this repository. `market-screener` is a Node.js project consisting of a Fastify API server that powers the SvelteKit dashboard in the `ui/` subdirectory. **Evolved to support day trading**: real-time news webhooks, LLM-driven stock analysis with prompt caching, multi-user authentication, and Discord alerts for price movements. +--- + +## ✅ Status Update — Shipped June 2026 (post-Phase-10 sprint) + +See **PRODUCT.md** (priorities P0–P3) and **FREE-DATA-STACK.md** (zero-cost data architecture) for design rationale. Shipped since the Phase 9/10 reports below: + +**Correctness / foundations (PRODUCT.md P0):** +- Structured verdict tiers — scorers return `{ tier: PASS|HOLD|REJECT, score }`; `signal()` no longer string-matches emoji (P0.3) +- Signal snapshot ledger — `signal_snapshots` table, written on every screen; `GET /api/screen/history/:ticker`; `npm run screen:daily` (P0.1) +- Rate-regime hysteresis (±0.25% band, survives restarts) (P0.5) +- Data-sanity sentinel — `dataHealth` on `/api/screen` + UI banner when >30% of stocks return null fundamentals (P0.4) +- Bug-fix pass: ETF null-handling (missing data no longer auto-REJECTs), dividend-ETF↔bond classification via `fundProfile.categoryName`, zero-as-null sanitization, no-data verdicts labeled honestly, coverage field in every audit + +**Free-tier news pipeline (`server/domains/news/` — Phase-12-lite, $0):** +- EDGAR poller (8-K, SC 13D, S-4, DEFM14A; CIK→ticker map) + PR-wire RSS poller (GlobeNewswire, PR Newswire; exchange-tag ticker extraction) +- Shared pipeline: universe filter → noise blocklist → dedupe → keyword catalyst classifier → `news_articles` / `ticker_catalysts` tables; retention jobs +- In-server scheduler (EDGAR 10 min, PR 15 min; `NEWS_POLL=off` to disable) + `npm run news:poll` for cron +- `GET /api/news/:ticker` (stored + live Yahoo merge), `GET /api/news/recent` + +**Daily change digest (`server/domains/digest/` — PRODUCT P1.1, partial Phase 14):** +- Diffs today's snapshots vs previous, attaches news catalysts, M&A always surfaced +- `GET /api/digest`, `npm run digest:daily`, Discord webhook (forum-channel aware), `npm run discord:test` + +**UI (screener page):** +- Market Pulse header band — full-bleed sector bubbles (SPDR ETFs, 15-min cache), leader headline, loading/unavailable states; `GET /api/screen/sectors` +- Sector drill-down panel — top-10 ETF holdings screened on demand, Today/1Y gain sort, 3-day sector news; `GET /api/screen/sector/:sector` +- Ticker modal — company profile, range-switchable chart (1D…5Y, hover crosshair), Yahoo analyst target bar + Zacks link, latest news; `GET /api/screen/profile/:ticker`, `GET /api/screen/chart/:ticker?range=` +- Plain-language advice layer (`adviceFor`) — "Buy — stable growth" / "Buy, but expect dips" on Strong Buys + no-data honesty; full text in modal +- ↗ Turnaround watch + 💎 Quality dips filters (with live counts + self-explaining empty states) in the STOCK table header +- Score cell: negative-score fix, coverage chip, "No data" state + +**New env vars:** `EDGAR_USER_AGENT` (recommended), `DISCORD_WEBHOOK_URL`, `NEWS_PRWIRE_FEEDS`, `NEWS_POLL`. +**Pages NOT yet touched in this sprint:** Portfolio, Market Calls, Safe Buys (still at their Phase 7 state — see realigned roadmap in PHASES.md). + ### Two Scoring Lenses (Original) Every asset is scored under two lenses: @@ -1278,9 +1312,22 @@ lib/types/ | **Regime badge colors** | `HIGH` = amber, `NORMAL` = muted gray, `LOW` = blue (driven by `data-regime` CSS attribute) | | **Signal Summary hidden** | Removed from `+page.svelte` — table section no longer renders | -#### 🔲 Next Up (Phase 10.5 Remaining) +#### 🔲 Next Up (Phase 10.5 Remaining — status corrected June 2026) -These five items are the immediate next build targets, in priority order: +Item 1 (tearsheet) is **partially superseded**: the Ticker Modal now delivers +the chart, company profile, analyst targets, and news. Still genuinely +pending from the original list: + +1. **P/E + ROE + 52W columns in main table** (10.5c) — not started +2. **Valuation context / peer comparison** (10.5d §2) — modal has analyst + targets but no sector/S&P comparison table +3. **Numeric range filters for P/E and ROE** (10.5b) — price min/max and + score min exist; P/E-max / ROE-min do not +4. **Threshold sensitivity what-ifs** (10.5d §5) — not started +5. **Decision logging + backtest** (10.5e) — not started, but the snapshot + ledger (P0.1) now provides its data foundation + +Original spec below for reference: **1. Slide-in tearsheet panel** (`10.5d`) - Replace the current inline expand row with a 420px right-side slide-in panel (CSS `transform: translateX` animation, 0.2s) diff --git a/PHASES.md b/PHASES.md index de30f4a..b7dbf67 100644 --- a/PHASES.md +++ b/PHASES.md @@ -1,5 +1,38 @@ # PHASES.md +--- + +## 📍 Roadmap Status & Realignment — June 2026 + +Cross-reference: **PRODUCT.md** (P0–P3 priorities) and **FREE-DATA-STACK.md** ($0 data architecture). CLAUDE.md "Status Update" has the full shipped list. + +### Done ahead of schedule (was "future", now shipped) + +| Originally planned as | What actually shipped | Status | +|---|---|---| +| Phase 12 (news webhooks, ~$200/mo) | **Free-tier news pipeline**: EDGAR + PR-wire pollers → filter/dedupe/classify → SQLite; in-server scheduler + cron runner; `/api/news/*` | ✅ Free version shipped. Paid webhook spine = drop-in upgrade (same queue) | +| Phase 14 (real-time monitor + Discord) | **Daily change digest**: snapshot diff + catalyst join → Discord (forum-aware). EOD, not intraday | ✅ EOD version shipped. Real-time price feed still future | +| Phase 10.9 (dip opportunity monitor) | **💎 Quality dips filter**: quality-gate PASS + 10%+ off 52W high, in the STOCK table | ✅ v1 shipped. Dedicated daily monitor + dip attribution still future | +| Phase 10.5d tearsheet (partial) | **Ticker modal**: profile, 1D–5Y chart w/ crosshair, analyst target bar, news | ✅ Covers chart/profile/targets/news. Peer comparison + what-ifs pending | +| 10.5e backtest (foundation) | **Signal snapshot ledger** + `/api/screen/history/:ticker` | ✅ Data accumulating; dashboard pending | +| (unplanned) | Market Pulse band, sector drill-down panel, advice layer, turnaround watch, data sentinel, verdict tiers, regime hysteresis | ✅ | + +### Still at Phase-7 state (not touched this sprint) + +**Portfolio**, **Market Calls**, **Safe Buys** pages — work as before, none of the new +intelligence (advice layer, snapshots, news) is wired into them yet. + +### Realigned order of future work + +1. **Finish Phase 10.5** — P/E+ROE+52W columns, P/E/ROE range filters, peer-comparison + what-if sections in the ticker modal (items listed in CLAUDE.md) +2. **Phase 10.6 — Portfolio integration** ← biggest gap now: wire signals/advice/snapshots/news into the Portfolio page ("you own this, verdict changed, here's why") +3. **Safe Buys upgrade (10.9 v2)** — rebuild the Safe Buys page on quality-dips + snapshot history + news attribution +4. **10.8a — earnings dates in the ticker modal** (Finnhub free tier, per FREE-DATA-STACK §1.5) +5. **10.5e — decision log + backtest dashboard** (once the ledger has ~3 months of data) +6. **Phase 11 — auth** (already partially present: JWT login/watchlist exist) → then paid upgrades: **Phase 12** webhook spine, **Phase 13** prompt caching, **Phase 14** real-time monitor + +--- + Complete roadmap for market-screener evolution from Phase 9 through Phase 16+. ## Phase 9 — Subdomain Restructure: Server Layer Organization @@ -389,6 +422,8 @@ Consider consistency across three locations, visual hierarchy differences, and m ## Phase 10.9 — Strong Buys: Professional Dip Opportunity Monitor +> **June 2026:** v1 SHIPPED as the 💎 Quality dips filter (quality-gate PASS + 10%+ off 52W high). Remaining: dedicated daily monitor, dip-reason attribution, configurable universe/thresholds. + **Goal:** Flag quality stocks when they drop 5%+ from 52W high, with market analysis of why. ### 10.9a — Data Structure @@ -566,6 +601,8 @@ Create `lib/stores/auth.store.svelte.ts` for currentUser, JWT, login/logout. ## Phase 12 — Day Trading: News Webhooks +> **June 2026:** Free-tier equivalent SHIPPED (`server/domains/news/` — EDGAR + PR-wire pollers, same queue design). This phase now = adding the paid Polygon/Finnhub real-time spine as another producer. See FREE-DATA-STACK.md. + **Goal:** Ingest real-time market news via Polygon.io webhooks. **Timeline:** 2-3 weeks. @@ -670,6 +707,8 @@ Route to cheaper models (Sonnet) when cost-sensitive. Fallback to OpenAI if rate ## Phase 14 — Day Trading: Safe Buys Monitor with Discord Alerts +> **June 2026:** EOD version SHIPPED (`server/domains/digest/` — daily signal-flip digest with catalysts → Discord). This phase now = real-time price feed + intraday dip alerts. + **Goal:** Monitor safe-buy stocks in real-time, detect 5%+ dips, notify via Discord. **Timeline:** 3-4 weeks. diff --git a/README.md b/README.md index 2cd1058..86de564 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,37 @@ Defaults to `http://localhost:5173`. Change if the UI is served from a different CLIENT_ORIGIN=https://yourdomain.com ``` +### `EDGAR_USER_AGENT` — SEC filings poller *(recommended)* + +The news pipeline polls SEC EDGAR for 8-K / SC 13D / S-4 / DEFM14A filings. +The SEC requires a descriptive User-Agent with contact info: + +```env +EDGAR_USER_AGENT=market-screener/1.0 you@example.com +``` + +### `DISCORD_WEBHOOK_URL` — Daily digest alerts *(optional)* + +The daily change digest (`npm run digest:daily`) posts signal flips + their +news catalysts to Discord. Create: channel → Settings → Integrations → +Webhooks → New Webhook → copy URL. Paste it RAW (no quotes, no escaping). +Forum channels are supported (each digest becomes a dated post). +Test with `npm run discord:test`. + +```env +DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... +``` + +### `NEWS_PRWIRE_FEEDS` — Override press-release RSS feeds *(optional)* + +Comma-separated RSS URLs. Defaults to GlobeNewswire + PR Newswire. Only +needed if a default feed goes stale or you want to add one. + +### `NEWS_POLL` — Disable in-server news polling *(optional)* + +Set `NEWS_POLL=off` if you prefer running `npm run news:poll` from cron +instead of polling inside the server (EDGAR 10 min, PR-wire 15 min). + ### Complete `.env` example ```env @@ -109,6 +140,8 @@ ANTHROPIC_API_KEY=sk-ant-... SIMPLEFIN_SETUP_TOKEN=aHR0cHM6Ly... API_KEY=optional-secret CLIENT_ORIGIN=http://localhost:5173 +EDGAR_USER_AGENT=market-screener/1.0 you@example.com +DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/... ``` --- @@ -127,6 +160,16 @@ CLIENT_ORIGIN=http://localhost:5173 | `npm run format:check` | Check formatting without writing (used in CI) | | `npm run lint` | Run ESLint on all TypeScript files | | `npm run lint:fix` | Auto-fix ESLint issues | +| `npm run screen:daily` | Screen watchlist + holdings, write signal snapshots (cron at market close) | +| `npm run news:poll` | One-shot news poll: EDGAR + PR wires → news DB (cron alternative) | +| `npm run digest:daily` | Daily change digest: signal flips + catalysts → terminal/Discord (run after screen:daily) | +| `npm run discord:test` | Send a fake digest to verify the Discord webhook | + +**Recommended cron (weekdays, market close):** + +``` +30 16 * * 1-5 cd /path/to/market_screener && npm run screen:daily && npm run digest:daily +``` --- diff --git a/bin/daily-digest.ts b/bin/daily-digest.ts new file mode 100644 index 0000000..6e8361d --- /dev/null +++ b/bin/daily-digest.ts @@ -0,0 +1,85 @@ +/** + * Daily change digest (PRODUCT.md P1.1) — diff today's signal snapshots + * against the previous ones, join with stored news catalysts, and post to + * Discord (DISCORD_WEBHOOK_URL) or print to the terminal. + * + * RUN ORDER MATTERS — screen first, digest second: + * 30 16 * * 1-5 cd /path/to/app && npm run screen:daily && npm run digest:daily + * + * Usage: + * npm run digest:daily # today + * npm run digest:daily -- 2026-06-09 # specific day + */ + +import 'dotenv/config'; +import { + createDb, + DatabaseConnection, + QueryAudit, + SignalSnapshotRepository, +} from '../server/domains/shared'; +import { NewsRepository } from '../server/domains/news'; +import { DigestService, DiscordNotifier } from '../server/domains/digest'; + +const db = new DatabaseConnection(createDb(process.env.DB_PATH ?? './market-screener.db'), { + audit: new QueryAudit(), + logSlowQueries: 100, +}); + +const consoleLogger = { + log: (...args: unknown[]) => console.log(...args), // eslint-disable-line no-console + warn: (...args: unknown[]) => console.warn(...args), + write: (msg: string) => process.stdout.write(msg), +}; + +const dateArg = process.argv[2]; +const date = + dateArg && /^\d{4}-\d{2}-\d{2}$/.test(dateArg) ? dateArg : new Date().toISOString().slice(0, 10); + +const digest = new DigestService(new SignalSnapshotRepository(db), new NewsRepository(db)); +const report = digest.build(date); + +/* eslint-disable no-console */ +console.log(`\n📊 Daily Signal Digest — ${report.date}`); +console.log(`Tickers snapshotted: ${report.snapshotCount}`); + +if (report.snapshotCount === 0) { + console.log('\nNo snapshots for this date. Run `npm run screen:daily` first.'); + process.exit(0); +} + +if (report.changes.length === 0) { + console.log('No signal changes since the previous snapshots. Calm day.'); +} else { + console.log(`\nSignal changes (${report.changes.length}):`); + for (const c of report.changes) { + const delta = + c.scoreDelta != null ? ` (score ${c.scoreDelta > 0 ? '+' : ''}${c.scoreDelta})` : ''; + console.log(`\n ${c.ticker}: ${c.previousSignal} → ${c.newSignal}${delta}`); + if (c.catalysts.length === 0) { + console.log(' no catalyst found — moved on fundamentals/market data'); + } + for (const s of c.catalysts.slice(0, 3)) { + console.log(` [${s.catalyst ?? 'news'}] ${s.headline}`); + } + } +} + +if (report.maStories.length > 0) { + console.log(`\n🔱 M&A activity (${report.maStories.length}):`); + for (const s of report.maStories.slice(0, 5)) console.log(` • ${s.headline}`); +} + +if (report.newTickers.length > 0) { + console.log(`\nFirst-time snapshots (no baseline yet): ${report.newTickers.join(', ')}`); +} + +const notifier = new DiscordNotifier(consoleLogger); +if (notifier.enabled) { + const sent = await notifier.send(report); + console.log(sent ? '\nPosted to Discord ✓' : '\nDiscord post skipped/failed'); +} else { + console.log('\n(Set DISCORD_WEBHOOK_URL in .env to receive this as a Discord message.)'); +} +/* eslint-enable no-console */ +process.exit(0); diff --git a/bin/poll-news.ts b/bin/poll-news.ts new file mode 100644 index 0000000..3877cd3 --- /dev/null +++ b/bin/poll-news.ts @@ -0,0 +1,67 @@ +/** + * One-shot news poll — for cron users who don't run the server 24/7. + * Fetches EDGAR + PR-wire feeds once, runs the pipeline, runs retention, + * prints stats, exits. + * + * Usage: + * npm run news:poll + * + * Crontab example (every 15 min, market hours, weekdays): + * *\/15 9-16 * * 1-5 cd /path/to/market_screener && npm run news:poll + * + * If the server runs continuously, its built-in scheduler covers this — + * set NEWS_POLL=off on the server if you prefer cron-driven polling. + */ + +import 'dotenv/config'; +import { createDb, DatabaseConnection, QueryAudit, noopLogger } from '../server/domains/shared'; +import { + NewsRepository, + NewsPipeline, + UniverseProvider, + NewsScheduler, + EdgarPoller, + PrWirePoller, +} from '../server/domains/news'; + +const db = new DatabaseConnection(createDb(process.env.DB_PATH ?? './market-screener.db'), { + audit: new QueryAudit(), + logSlowQueries: 100, +}); + +const consoleLogger = { + log: (...args: unknown[]) => console.log(...args), // eslint-disable-line no-console + warn: (...args: unknown[]) => console.warn(...args), + write: (msg: string) => process.stdout.write(msg), +}; + +const universe = new UniverseProvider(db); +const pipeline = new NewsPipeline(new NewsRepository(db)); +const scheduler = new NewsScheduler( + pipeline, + universe, + new EdgarPoller(noopLogger), + new PrWirePoller(noopLogger), + consoleLogger, +); + +const size = universe.getUniverse().size; +if (size === 0) { + console.log('Universe is empty (no watchlist, holdings, or recent screens) — nothing to poll.'); // eslint-disable-line no-console + process.exit(0); +} +console.log(`Polling news for a ${size}-ticker universe…`); // eslint-disable-line no-console + +try { + const { edgar, prwire } = await scheduler.runOnce(); + const retention = pipeline.runRetention(); + /* eslint-disable no-console */ + console.log('\nEDGAR :', JSON.stringify(edgar)); + console.log('PR-wire:', JSON.stringify(prwire)); + console.log('Retention:', JSON.stringify(retention)); + /* eslint-enable no-console */ + process.exit(0); +} catch (err) { + console.error('News poll failed:', (err as Error).message); + process.exit(1); +} diff --git a/bin/test-discord.ts b/bin/test-discord.ts new file mode 100644 index 0000000..41aa8c0 --- /dev/null +++ b/bin/test-discord.ts @@ -0,0 +1,68 @@ +/** + * Discord webhook smoke test — sends a FAKE digest to DISCORD_WEBHOOK_URL + * so you can verify the integration without waiting for a real signal change. + * + * Usage: + * npm run discord:test + */ + +import 'dotenv/config'; +import { DiscordNotifier } from '../server/domains/digest/DiscordNotifier'; +import type { DigestReport } from '../server/domains/shared/types'; + +/* eslint-disable no-console */ +if (!process.env.DISCORD_WEBHOOK_URL) { + console.error('DISCORD_WEBHOOK_URL is not set in .env'); + console.error('Discord → channel → Settings → Integrations → Webhooks → New Webhook → Copy URL'); + process.exit(1); +} + +const fakeReport: DigestReport = { + date: new Date().toISOString().slice(0, 10), + snapshotCount: 3, + newTickers: [], + changes: [ + { + ticker: 'TEST', + previousSignal: '✅ Strong Buy', + newSignal: '🔄 Neutral', + previousDate: 'yesterday', + scoreDelta: -7, + price: 123.45, + catalysts: [ + { + headline: '🔧 This is a TEST message from market-screener — webhook works!', + catalyst: 'regulatory', + source: 'edgar', + url: 'https://example.com', + publishedAt: new Date().toISOString(), + }, + ], + }, + ], + maStories: [ + { + headline: '🔧 TEST: SC 13D filing example (M&A section renders like this)', + catalyst: 'ma', + source: 'edgar', + url: 'https://example.com', + publishedAt: new Date().toISOString(), + }, + ], +}; + +const logger = { + log: (...args: unknown[]) => console.log(...args), + warn: (...args: unknown[]) => console.warn(...args), + write: (msg: string) => process.stdout.write(msg), +}; + +const ok = await new DiscordNotifier(logger).send(fakeReport); +if (ok) { + console.log('✓ Test digest posted — check your Discord channel.'); + process.exit(0); +} else { + console.error('✗ Post failed. Check the webhook URL (it may have been deleted/regenerated).'); + process.exit(1); +} +/* eslint-enable no-console */ diff --git a/package.json b/package.json index 21381cb..9651b51 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,9 @@ "lint": "eslint . --ext .ts,.js", "lint:fix": "eslint . --ext .ts,.js --fix", "screen:daily": "tsx bin/daily-screen.ts", + "news:poll": "tsx bin/poll-news.ts", + "digest:daily": "tsx bin/daily-digest.ts", + "discord:test": "tsx bin/test-discord.ts", "format": "prettier --write \"server/**/*.ts\" \"bin/**/*.ts\" \"tests/**/*.ts\"", "format:check": "prettier --check \"server/**/*.ts\" \"bin/**/*.ts\" \"tests/**/*.ts\"", "prepare": "husky" diff --git a/server/app.ts b/server/app.ts index c3bdb69..86d5939 100644 --- a/server/app.ts +++ b/server/app.ts @@ -11,6 +11,16 @@ import { CallsController, CalendarService } from './domains/calls'; import { AuthController, AuthService, UserStore, verifyJwt } from './domains/auth'; import type { TokenPayload } from './domains/auth'; import { WatchlistController, WatchlistRepository } from './domains/watchlist'; +import { + NewsController, + NewsRepository, + NewsPipeline, + UniverseProvider, + NewsScheduler, + EdgarPoller, + PrWirePoller, +} from './domains/news'; +import { DigestController, DigestService } from './domains/digest'; // Shared infrastructure import { @@ -141,7 +151,14 @@ export async function buildApp({ logger = true, db: injectedDb }: BuildAppOption // Register controllers // Public routes (GET) remain open; write routes require JWT + trader role - new ScreenerController(engine, catalystCache, new SignalSnapshotRepository(db)).register(app); + const newsRepo = new NewsRepository(db); + new ScreenerController( + engine, + catalystCache, + new SignalSnapshotRepository(db), + yahoo, + newsRepo, + ).register(app); new FinanceController(engine, new PortfolioRepository(db), advisor, { authGuard, traderGuard, @@ -154,6 +171,31 @@ export async function buildApp({ logger = true, db: injectedDb }: BuildAppOption new WatchlistController(new WatchlistRepository(db), { authGuard }).register(app); + // ── News domain (FREE-DATA-STACK) — pipeline + read API + polling ──────── + new NewsController(newsRepo, yahoo).register(app); + + // ── Digest domain (P1.1) — snapshot diff + catalyst join, on demand ────── + new DigestController(new DigestService(new SignalSnapshotRepository(db), newsRepo)).register(app); + + // Polling runs inside the server unless NEWS_POLL=off (use bin/poll-news.ts + // from cron instead). Timers are unref'd and cleared on app.close(). + if (process.env.NEWS_POLL !== 'off') { + const newsLogger = { + log: (...args: unknown[]) => app.log.info(args.map(String).join(' ')), + warn: (...args: unknown[]) => app.log.warn(args.map(String).join(' ')), + write: () => {}, + }; + const newsScheduler = new NewsScheduler( + new NewsPipeline(newsRepo), + new UniverseProvider(db), + new EdgarPoller(newsLogger), + new PrWirePoller(newsLogger), + newsLogger, + ); + app.addHook('onReady', async () => newsScheduler.start()); + app.addHook('onClose', async () => newsScheduler.stop()); + } + app.get('/health', async () => ({ status: 'ok' })); return app; diff --git a/server/domains/digest/DigestService.ts b/server/domains/digest/DigestService.ts new file mode 100644 index 0000000..62b6eab --- /dev/null +++ b/server/domains/digest/DigestService.ts @@ -0,0 +1,110 @@ +import { SignalSnapshotRepository } from '../shared/persistence/SignalSnapshotRepository'; +import { NewsRepository } from '../news/NewsRepository'; +import { SIGNAL_ORDER } from '../shared/config/constants'; +import type { + DigestCatalyst, + DigestChange, + DigestReport, + NewsArticleRow, + SignalSnapshotRow, +} from '../shared/types'; + +/** + * Daily change digest (PRODUCT.md P1.1) — the step that makes the snapshot + * ledger and the news pipeline actionable together. + * + * For each ticker snapshotted today, diff against its most recent previous + * snapshot. A signal flip alone is just information; a signal flip WITH a + * known catalyst attached is the highest-value alert the free stack can + * produce. M&A stories are always surfaced, change or no change. + * + * Run order matters: screen first (writes today's snapshots), digest second. + */ +export class DigestService { + /** How many days back to look for catalyst stories per ticker. */ + private static readonly NEWS_LOOKBACK_DAYS = 2; + + constructor( + private readonly snapshots: SignalSnapshotRepository, + private readonly news: NewsRepository, + ) {} + + build(date = new Date().toISOString().slice(0, 10)): DigestReport { + const today = this.snapshots.byDate(date); + const previous = new Map(this.snapshots.latestBefore(date).map((r) => [r.ticker, r])); + + const newsSince = DigestService.daysBefore(date, DigestService.NEWS_LOOKBACK_DAYS); + const changes: DigestChange[] = []; + const newTickers: string[] = []; + const maStories = new Map(); // url → story, deduped + + for (const snap of today) { + const prev = previous.get(snap.ticker); + const catalysts = this.news + .newsForTicker(snap.ticker, newsSince) + .map(DigestService.toCatalyst); + + // Always collect M&A stories, even without a signal change + for (const c of catalysts) { + if (c.catalyst === 'ma') maStories.set(c.url, c); + } + + if (!prev) { + newTickers.push(snap.ticker); + continue; + } + if (prev.signal === snap.signal) continue; + + changes.push({ + ticker: snap.ticker, + previousSignal: prev.signal, + newSignal: snap.signal, + previousDate: prev.snapshot_date, + scoreDelta: DigestService.scoreDelta(prev, snap), + price: snap.price, + catalysts, + }); + } + + // Strongest impact first: biggest move across the signal ordering + changes.sort((a, b) => DigestService.impact(b) - DigestService.impact(a)); + + return { + date, + changes, + newTickers, + maStories: [...maStories.values()], + snapshotCount: today.length, + }; + } + + // ── Helpers ─────────────────────────────────────────────────────────────── + + private static toCatalyst(row: NewsArticleRow): DigestCatalyst { + return { + headline: row.headline, + catalyst: row.catalyst, + source: row.source, + url: row.url, + publishedAt: row.published_at, + }; + } + + private static scoreDelta(prev: SignalSnapshotRow, curr: SignalSnapshotRow): number | null { + if (prev.fundamental_score == null || curr.fundamental_score == null) return null; + return +(curr.fundamental_score - prev.fundamental_score).toFixed(1); + } + + /** Distance moved across the signal ordering (Strong Buy=0 … Avoid=4). */ + private static impact(change: DigestChange): number { + const ord = (s: string) => SIGNAL_ORDER[s] ?? 5; + return Math.abs(ord(change.newSignal) - ord(change.previousSignal)); + } + + /** YYYY-MM-DD `n` days before the given day. */ + private static daysBefore(date: string, n: number): string { + const d = new Date(`${date}T00:00:00.000Z`); + d.setUTCDate(d.getUTCDate() - n); + return d.toISOString().slice(0, 10); + } +} diff --git a/server/domains/digest/DiscordNotifier.ts b/server/domains/digest/DiscordNotifier.ts new file mode 100644 index 0000000..215d8b8 --- /dev/null +++ b/server/domains/digest/DiscordNotifier.ts @@ -0,0 +1,128 @@ +import type { DigestReport, Logger } from '../shared/types'; + +/** + * Posts the daily digest to a Discord webhook (DISCORD_WEBHOOK_URL in .env). + * When the env var is unset, send() is a no-op and the caller falls back to + * console output — the digest is still useful without Discord. + * + * Embed building is a pure static so it can be unit-tested without network. + */ +export class DiscordNotifier { + private static readonly MAX_FIELDS = 10; // Discord caps embeds at 25 fields; keep digests scannable + + constructor( + private readonly logger: Logger, + private readonly webhookUrl = process.env.DISCORD_WEBHOOK_URL, + ) {} + + get enabled(): boolean { + return Boolean(this.webhookUrl); + } + + async send(report: DigestReport): Promise { + if (!this.webhookUrl) return false; + const payload = DiscordNotifier.buildPayload(report); + if (!payload) { + this.logger.log('Digest: nothing to report — Discord post skipped'); + return false; + } + + let res = await this.post(payload); + + // Forum channels require a thread name (Discord error code 220001) — + // retry once, creating a post titled with the digest date. + if (res.status === 400 && (await DiscordNotifier.isForumError(res))) { + this.logger.log('Webhook targets a forum channel — retrying with thread_name'); + res = await this.post({ ...payload, thread_name: `Signal Digest ${report.date}` }); + } + + if (!res.ok) { + const body = await res.text().catch(() => ''); + this.logger.warn( + `Discord webhook failed: HTTP ${res.status} — ${body.slice(0, 200) || 'no response body'}`, + ); + if (res.status === 401 || res.status === 404) { + this.logger.warn( + 'Hint: the URL in .env must be the RAW webhook URL (no <>, no quotes, no HTML escaping), ' + + 'ending in a ~68-char token. Re-copy it: Channel Settings → Integrations → Webhooks.', + ); + } + return false; + } + return true; + } + + private post(payload: object): Promise { + return fetch(this.webhookUrl as string, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + } + + private static async isForumError(res: Response): Promise { + try { + const body = (await res.clone().json()) as { code?: number }; + return body.code === 220001; + } catch { + return false; + } + } + + /** Returns null when there is nothing worth posting. */ + static buildPayload(report: DigestReport): { embeds: unknown[] } | null { + if (report.changes.length === 0 && report.maStories.length === 0) return null; + + const fields: Array<{ name: string; value: string; inline: boolean }> = []; + + for (const c of report.changes.slice(0, DiscordNotifier.MAX_FIELDS)) { + const delta = + c.scoreDelta != null ? ` (score ${c.scoreDelta > 0 ? '+' : ''}${c.scoreDelta})` : ''; + const catalystLine = c.catalysts.length + ? c.catalysts + .slice(0, 2) + .map((s) => `• [${s.catalyst ?? 'news'}] ${DiscordNotifier.trim(s.headline, 80)}`) + .join('\n') + : '• no catalyst found — verdict moved on fundamentals/market data'; + fields.push({ + name: `${c.ticker}: ${c.previousSignal} → ${c.newSignal}${delta}`, + value: catalystLine, + inline: false, + }); + } + + if (report.changes.length > DiscordNotifier.MAX_FIELDS) { + fields.push({ + name: `…and ${report.changes.length - DiscordNotifier.MAX_FIELDS} more changes`, + value: 'See GET /api/digest for the full report', + inline: false, + }); + } + + if (report.maStories.length > 0) { + fields.push({ + name: `🔱 M&A activity (${report.maStories.length})`, + value: report.maStories + .slice(0, 5) + .map((s) => `• ${DiscordNotifier.trim(s.headline, 90)}`) + .join('\n'), + inline: false, + }); + } + + return { + embeds: [ + { + title: `📊 Daily Signal Digest — ${report.date}`, + description: `${report.snapshotCount} tickers screened · ${report.changes.length} signal change(s)`, + color: report.changes.length > 0 ? 0xf0b429 : 0x4ade80, // amber if changes, green if calm + fields, + }, + ], + }; + } + + private static trim(s: string, max: number): string { + return s.length <= max ? s : `${s.slice(0, max - 1)}…`; + } +} diff --git a/server/domains/digest/digest.controller.ts b/server/domains/digest/digest.controller.ts new file mode 100644 index 0000000..3e89456 --- /dev/null +++ b/server/domains/digest/digest.controller.ts @@ -0,0 +1,22 @@ +import type { FastifyInstance, FastifyRequest } from 'fastify'; +import { DigestService } from './DigestService'; + +/** + * On-demand digest read (P1.1). The scheduled path is bin/daily-digest.ts; + * this endpoint lets the UI (or curl) build the same report any time. + */ +export class DigestController { + constructor(private readonly digest: DigestService) {} + + register(app: FastifyInstance): void { + app.get('/api/digest', this.today.bind(this)); + } + + /** GET /api/digest?date=YYYY-MM-DD (defaults to today) */ + private async today(req: FastifyRequest) { + const { date } = req.query as { date?: string }; + const day = + date && /^\d{4}-\d{2}-\d{2}$/.test(date) ? date : new Date().toISOString().slice(0, 10); + return this.digest.build(day); + } +} diff --git a/server/domains/digest/index.ts b/server/domains/digest/index.ts new file mode 100644 index 0000000..f72cfe1 --- /dev/null +++ b/server/domains/digest/index.ts @@ -0,0 +1,5 @@ +// Digest domain — daily change detection (PRODUCT.md P1.1) + +export { DigestService } from './DigestService'; +export { DiscordNotifier } from './DiscordNotifier'; +export { DigestController } from './digest.controller'; diff --git a/server/domains/news/NewsPipeline.ts b/server/domains/news/NewsPipeline.ts new file mode 100644 index 0000000..f3d78a4 --- /dev/null +++ b/server/domains/news/NewsPipeline.ts @@ -0,0 +1,165 @@ +import { createHash } from 'crypto'; +import { NewsRepository } from './NewsRepository'; +import type { CatalystType, IngestStats, NormalizedStory } from '../shared/types'; + +/** + * Shared ingest pipeline (FREE-DATA-STACK §2) — every source flows through + * here: FILTER → DEDUPE → CLASSIFY → STORE. All drops happen BEFORE insert, + * cheapest check first, so the tables stay small by construction (§4). + */ +export class NewsPipeline { + /** §4.4 — max stories linked per ticker per day (filings exempt). */ + private static readonly DAILY_CAP = 25; + /** §4.3 — syndicated-copy window for title dedupe. */ + private static readonly TITLE_WINDOW_MS = 48 * 60 * 60 * 1000; + + /** §4.2 — headlines with no decision value are never stored. */ + private static readonly NOISE_PATTERNS: RegExp[] = [ + /\b\d+\s+(?:best|top|hot)\s+stocks?\b/i, + /\bstocks?\s+to\s+(?:watch|buy|sell)\b/i, + /\bprice\s+target\s+(?:raised|lowered|reiterated|maintained)\b/i, + /\b(?:premarket|after-?hours?)\s+movers?\b/i, + /\bwhy\s+.{0,40}\s+stock\s+(?:jumped|popped|soared|plunged|tanked)\b/i, + /\bmotley\s+fool\b/i, + ]; + + constructor(private readonly repo: NewsRepository) {} + + /** + * Run a batch of normalized stories through the pipeline. + * `universe` is the tracked-ticker set from UniverseProvider. + */ + ingest(stories: NormalizedStory[], universe: Set): IngestStats { + const stats: IngestStats = { + fetched: stories.length, + stored: 0, + droppedNoUniverseTicker: 0, + droppedNoise: 0, + droppedDuplicate: 0, + droppedCapped: 0, + }; + + for (const story of stories) { + this.ingestOne(story, universe, stats); + } + return stats; + } + + private ingestOne(story: NormalizedStory, universe: Set, stats: IngestStats): void { + const isFiling = story.source === 'edgar'; + + // 1. Universe filter — the big one (§4.1) + const tickers = [...new Set(story.tickers.map((t) => t.toUpperCase()))].filter((t) => + universe.has(t), + ); + if (tickers.length === 0) { + stats.droppedNoUniverseTicker++; + return; + } + + // 2. Noise blocklist (§4.2) — filings are never noise + if (!isFiling && NewsPipeline.isNoise(story.headline)) { + stats.droppedNoise++; + return; + } + + // 3. Dedupe (§4.3): url hash (storage-level PK) + recent title match + const urlHash = NewsPipeline.sha(story.url); + const titleHash = NewsPipeline.sha(NewsPipeline.normalizeTitle(story.headline)); + const titleCutoff = new Date(Date.now() - NewsPipeline.TITLE_WINDOW_MS).toISOString(); + if (this.repo.titleSeenSince(titleHash, titleCutoff)) { + stats.droppedDuplicate++; + return; + } + + // 4. Per-ticker daily cap (§4.4) — filings keep priority past the cap + const day = story.publishedAt.slice(0, 10); + const eligible = isFiling + ? tickers + : tickers.filter((t) => this.repo.countTickerDay(t, day) < NewsPipeline.DAILY_CAP); + if (eligible.length === 0) { + stats.droppedCapped++; + return; + } + + // 5. Classify + store + const catalyst = story.catalystHint ?? NewsPipeline.classify(story.headline); + const inserted = this.repo.insertArticle({ + urlHash, + titleHash, + tickers: eligible, + headline: story.headline.trim(), + body: story.body ?? null, + source: story.source, + catalyst, + url: story.url, + publishedAt: story.publishedAt, + }); + if (!inserted) { + stats.droppedDuplicate++; // url_hash collision — already stored + return; + } + + for (const ticker of eligible) { + this.repo.linkTicker(ticker, day, urlHash); + } + stats.stored++; + } + + /** Retention jobs (§5) — call once daily. */ + runRetention(now = new Date()): { bodiesPurged: number; rowsDeleted: number } { + const bodyCutoff = new Date(now.getTime() - 90 * 24 * 60 * 60 * 1000).toISOString(); + const rowCutoff = new Date(now.getTime() - 548 * 24 * 60 * 60 * 1000).toISOString(); // ~18mo + return { + bodiesPurged: this.repo.purgeBodiesBefore(bodyCutoff), + rowsDeleted: this.repo.deleteUnreferencedBefore(rowCutoff), + }; + } + + // ── Pure helpers (exposed for tests) ────────────────────────────────────── + + static isNoise(headline: string): boolean { + return NewsPipeline.NOISE_PATTERNS.some((re) => re.test(headline)); + } + + /** + * Keyword catalyst classifier. Order matters: M&A beats earnings + * ("acquisition closes in Q2" is an M&A story). + */ + static classify(headline: string): CatalystType | null { + const h = headline.toLowerCase(); + if ( + /\b(acqui[sr]|merger|takeover|buyout|tender offer|business combination|to be acquired)/.test( + h, + ) + ) + return 'ma'; + if (/\b(guidance|outlook|forecast|raises full[- ]year|lowers full[- ]year)/.test(h)) + return 'guidance'; + if ( + /\b(earnings|results|eps|quarterly report|q[1-4] (?:20\d\d|results)|fiscal (?:year|q[1-4]))/.test( + h, + ) + ) + return 'earnings'; + if ( + /\b(sec |fda|doj|ftc|antitrust|investigation|subpoena|lawsuit|settl|recall|approval)/.test(h) + ) + return 'regulatory'; + if (/\b(fed |fomc|inflation|cpi|jobs report|rate (?:cut|hike)|treasury yield)/.test(h)) + return 'macro'; + return null; + } + + static normalizeTitle(title: string): string { + return title + .toLowerCase() + .replace(/[^a-z0-9 ]/g, '') + .replace(/\s+/g, ' ') + .trim(); + } + + private static sha(input: string): string { + return createHash('sha256').update(input).digest('hex'); + } +} diff --git a/server/domains/news/NewsRepository.ts b/server/domains/news/NewsRepository.ts new file mode 100644 index 0000000..b3f4a6b --- /dev/null +++ b/server/domains/news/NewsRepository.ts @@ -0,0 +1,76 @@ +import { DatabaseConnection } from '../shared/db/index'; +import { QueryBuilder } from '../shared/utils/QueryBuilder'; +import type { NewsArticleRow } from '../shared/types'; + +/** + * Persistence for the free-tier news pipeline (FREE-DATA-STACK §3). + * Pure data access — all filtering/dedupe decisions live in NewsPipeline. + */ +export class NewsRepository { + constructor(private readonly db: DatabaseConnection) {} + + /** Returns true if the row was inserted (false = duplicate url_hash). */ + insertArticle(a: { + urlHash: string; + titleHash: string; + tickers: string[]; + headline: string; + body: string | null; + source: string; + catalyst: string | null; + url: string; + publishedAt: string; + }): boolean { + const qb = new QueryBuilder('NEWS_QUERIES.INSERT_ARTICLE', [ + a.urlHash, + a.titleHash, + JSON.stringify(a.tickers), + a.headline, + a.body, + a.source, + a.catalyst, + a.url, + a.publishedAt, + new Date().toISOString(), + ]); + return this.db.run(qb) > 0; + } + + titleSeenSince(titleHash: string, sinceIso: string): boolean { + const qb = new QueryBuilder('NEWS_QUERIES.TITLE_SEEN_SINCE', [titleHash, sinceIso]); + return this.db.get(qb) != null; + } + + linkTicker(ticker: string, day: string, urlHash: string): void { + const qb = new QueryBuilder('NEWS_QUERIES.INSERT_CATALYST_LINK', [ticker, day, urlHash]); + this.db.run(qb); + } + + countTickerDay(ticker: string, day: string): number { + const qb = new QueryBuilder('NEWS_QUERIES.COUNT_TICKER_DAY', [ticker, day]); + return this.db.get<{ n: number }>(qb)?.n ?? 0; + } + + newsForTicker(ticker: string, sinceDay: string): NewsArticleRow[] { + const qb = new QueryBuilder('NEWS_QUERIES.SELECT_TICKER_NEWS', [ + ticker.toUpperCase(), + sinceDay, + ]); + return this.db.all(qb); + } + + recent(limit: number): NewsArticleRow[] { + const qb = new QueryBuilder('NEWS_QUERIES.SELECT_RECENT', [limit]); + return this.db.all(qb); + } + + /** Retention: null out bodies older than cutoff. Returns rows changed. */ + purgeBodiesBefore(cutoffIso: string): number { + return this.db.run(new QueryBuilder('NEWS_QUERIES.PURGE_BODIES_BEFORE', [cutoffIso])); + } + + /** Retention: delete old rows no ticker references. Returns rows deleted. */ + deleteUnreferencedBefore(cutoffIso: string): number { + return this.db.run(new QueryBuilder('NEWS_QUERIES.DELETE_UNREFERENCED_BEFORE', [cutoffIso])); + } +} diff --git a/server/domains/news/NewsScheduler.ts b/server/domains/news/NewsScheduler.ts new file mode 100644 index 0000000..5b7255b --- /dev/null +++ b/server/domains/news/NewsScheduler.ts @@ -0,0 +1,106 @@ +import { NewsPipeline } from './NewsPipeline'; +import { UniverseProvider } from './UniverseProvider'; +import { EdgarPoller } from './pollers/EdgarPoller'; +import { PrWirePoller } from './pollers/PrWirePoller'; +import type { IngestStats, Logger } from '../shared/types'; + +/** + * In-process polling scheduler (FREE-DATA-STACK §2). No Redis/BullMQ at the + * free tier — plain intervals, unref'd so they never hold the process open. + * + * Cadences: EDGAR 10 min, PR-wire 15 min, retention daily. + * Disable entirely with NEWS_POLL=off (e.g. when running bin/poll-news.ts + * from cron instead of inside the server). + */ +export class NewsScheduler { + private static readonly EDGAR_INTERVAL_MS = 10 * 60 * 1000; + private static readonly PRWIRE_INTERVAL_MS = 15 * 60 * 1000; + private static readonly RETENTION_INTERVAL_MS = 24 * 60 * 60 * 1000; + + private timers: NodeJS.Timeout[] = []; + + constructor( + private readonly pipeline: NewsPipeline, + private readonly universe: UniverseProvider, + private readonly edgar: EdgarPoller, + private readonly prwire: PrWirePoller, + private readonly logger: Logger, + ) {} + + start(): void { + if (this.timers.length > 0) return; // already running + + const every = (ms: number, fn: () => void) => { + const t = setInterval(fn, ms); + t.unref(); // never keep the process alive just for polling + this.timers.push(t); + }; + + every(NewsScheduler.EDGAR_INTERVAL_MS, () => void this.runEdgar()); + every(NewsScheduler.PRWIRE_INTERVAL_MS, () => void this.runPrWire()); + every(NewsScheduler.RETENTION_INTERVAL_MS, () => this.runRetention()); + + // Prime once shortly after boot (delay keeps server startup fast) + const boot = setTimeout(() => void this.runOnce(), 15_000); + boot.unref(); + this.timers.push(boot); + + this.logger.log('News scheduler started (EDGAR 10m, PR-wire 15m, retention 24h)'); + } + + stop(): void { + for (const t of this.timers) clearInterval(t); + this.timers = []; + } + + /** One full cycle of everything — used at boot and by bin/poll-news.ts. */ + async runOnce(): Promise<{ edgar: IngestStats; prwire: IngestStats }> { + const edgar = await this.runEdgar(); + const prwire = await this.runPrWire(); + return { edgar, prwire }; + } + + private async runEdgar(): Promise { + try { + const stories = await this.edgar.poll(this.universe.getUniverse()); + const stats = this.pipeline.ingest(stories, this.universe.getUniverse()); + if (stats.stored > 0) this.logger.log(`EDGAR: stored ${stats.stored}/${stats.fetched}`); + return stats; + } catch (err) { + this.logger.warn('EDGAR poll cycle failed:', (err as Error).message); + return NewsScheduler.emptyStats(); + } + } + + private async runPrWire(): Promise { + try { + const stories = await this.prwire.poll(); + const stats = this.pipeline.ingest(stories, this.universe.getUniverse()); + if (stats.stored > 0) this.logger.log(`PR-wire: stored ${stats.stored}/${stats.fetched}`); + return stats; + } catch (err) { + this.logger.warn('PR-wire poll cycle failed:', (err as Error).message); + return NewsScheduler.emptyStats(); + } + } + + private runRetention(): void { + try { + const { bodiesPurged, rowsDeleted } = this.pipeline.runRetention(); + this.logger.log(`News retention: ${bodiesPurged} bodies purged, ${rowsDeleted} rows deleted`); + } catch (err) { + this.logger.warn('News retention failed:', (err as Error).message); + } + } + + private static emptyStats(): IngestStats { + return { + fetched: 0, + stored: 0, + droppedNoUniverseTicker: 0, + droppedNoise: 0, + droppedDuplicate: 0, + droppedCapped: 0, + }; + } +} diff --git a/server/domains/news/UniverseProvider.ts b/server/domains/news/UniverseProvider.ts new file mode 100644 index 0000000..0b65efe --- /dev/null +++ b/server/domains/news/UniverseProvider.ts @@ -0,0 +1,50 @@ +import { DatabaseConnection } from '../shared/db/index'; +import { QueryBuilder } from '../shared/utils/QueryBuilder'; + +/** + * The tracked-ticker universe (FREE-DATA-STACK §4.1): + * watchlist ∪ holdings ∪ tickers screened in the last 30 days. + * + * This is the news pipeline's first and biggest filter — stories about + * tickers outside the universe are never stored. Cached for 10 minutes; + * the universe changes slowly. + */ +export class UniverseProvider { + private static readonly CACHE_TTL_MS = 10 * 60 * 1000; + private static readonly SNAPSHOT_LOOKBACK_DAYS = 30; + + private cache: { universe: Set; expiresAt: number } = { + universe: new Set(), + expiresAt: 0, + }; + + constructor(private readonly db: DatabaseConnection) {} + + getUniverse(): Set { + if (Date.now() < this.cache.expiresAt) return this.cache.universe; + + const sinceDay = new Date( + Date.now() - UniverseProvider.SNAPSHOT_LOOKBACK_DAYS * 24 * 60 * 60 * 1000, + ) + .toISOString() + .slice(0, 10); + + const tickers = new Set(); + const add = (rows: { ticker: string }[]) => + rows.forEach((r) => tickers.add(r.ticker.toUpperCase())); + + add(this.db.all(new QueryBuilder('UNIVERSE_QUERIES.DISTINCT_WATCHLIST_TICKERS'))); + add(this.db.all(new QueryBuilder('UNIVERSE_QUERIES.DISTINCT_HOLDING_TICKERS'))); + add( + this.db.all(new QueryBuilder('UNIVERSE_QUERIES.DISTINCT_SNAPSHOT_TICKERS_SINCE', [sinceDay])), + ); + + this.cache = { universe: tickers, expiresAt: Date.now() + UniverseProvider.CACHE_TTL_MS }; + return tickers; + } + + /** Force next getUniverse() to re-read (e.g. after a watchlist change). */ + invalidate(): void { + this.cache.expiresAt = 0; + } +} diff --git a/server/domains/news/index.ts b/server/domains/news/index.ts new file mode 100644 index 0000000..aa5db07 --- /dev/null +++ b/server/domains/news/index.ts @@ -0,0 +1,10 @@ +// News domain — free-tier news ingestion pipeline (FREE-DATA-STACK.md) + +export { NewsController } from './news.controller'; +export { NewsRepository } from './NewsRepository'; +export { NewsPipeline } from './NewsPipeline'; +export { UniverseProvider } from './UniverseProvider'; +export { NewsScheduler } from './NewsScheduler'; +export { EdgarPoller } from './pollers/EdgarPoller'; +export { PrWirePoller } from './pollers/PrWirePoller'; +export { RssParser } from './rss'; diff --git a/server/domains/news/news.controller.ts b/server/domains/news/news.controller.ts new file mode 100644 index 0000000..fc09d1d --- /dev/null +++ b/server/domains/news/news.controller.ts @@ -0,0 +1,90 @@ +import type { FastifyInstance, FastifyRequest } from 'fastify'; +import { NewsRepository } from './NewsRepository'; +import { YahooFinanceClient } from '../shared'; +import type { NewsArticleRow } from '../shared/types'; + +interface StoryView { + headline: string; + tickers: string[]; + source: string; + catalyst: string | null; + url: string; + publishedAt: string; +} + +/** + * Read side of the news pipeline. Stored pipeline stories (curated, catalyst- + * tagged, historical) are merged with a live per-ticker Yahoo search on + * request — stored gives depth, live gives freshness. The RSS firehoses + * can't be queried per-ticker on demand, which is why they go through the + * polling pipeline instead. + */ +export class NewsController { + constructor( + private readonly repo: NewsRepository, + private readonly yahoo?: YahooFinanceClient, + ) {} + + register(app: FastifyInstance): void { + app.get('/api/news/recent', this.recent.bind(this)); + app.get('/api/news/:ticker', this.byTicker.bind(this)); + } + + /** GET /api/news/:ticker?days=7&live=1 (live Yahoo merge on by default) */ + private async byTicker(req: FastifyRequest) { + const ticker = (req.params as { ticker: string }).ticker.toUpperCase(); + const query = req.query as { days?: string; live?: string }; + const days = Math.min(Number(query.days ?? 7) || 7, 90); + const live = query.live !== '0'; + const sinceDay = new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); + + const stored = this.repo.newsForTicker(ticker, sinceDay).map(NewsController.serialize); + const fresh = live ? await this.fetchLive(ticker) : []; + + // Merge, dedupe by URL, newest first + const byUrl = new Map(); + for (const s of [...stored, ...fresh]) byUrl.set(s.url, byUrl.get(s.url) ?? s); + const stories = [...byUrl.values()].sort((a, b) => b.publishedAt.localeCompare(a.publishedAt)); + + return { ticker, days, stories }; + } + + /** Live per-ticker Yahoo news search — freshness layer, best-effort. */ + private async fetchLive(ticker: string): Promise { + if (!this.yahoo) return []; + try { + const items = await this.yahoo.search(ticker, { newsCount: 8 }); + return items + .filter((n) => n.title && n.link) + .map((n) => ({ + headline: n.title as string, + tickers: [ticker], + source: 'yahoo', + catalyst: null, + url: n.link as string, + publishedAt: n.providerPublishTime + ? new Date(n.providerPublishTime).toISOString() + : new Date().toISOString(), + })); + } catch { + return []; + } + } + + /** GET /api/news/recent?limit=50 */ + private async recent(req: FastifyRequest) { + const limit = Math.min(Number((req.query as { limit?: string }).limit ?? 50) || 50, 200); + return { stories: this.repo.recent(limit).map(NewsController.serialize) }; + } + + private static serialize(row: NewsArticleRow) { + return { + headline: row.headline, + tickers: JSON.parse(row.ticker_list) as string[], + source: row.source, + catalyst: row.catalyst, + url: row.url, + publishedAt: row.published_at, + }; + } +} diff --git a/server/domains/news/pollers/EdgarPoller.ts b/server/domains/news/pollers/EdgarPoller.ts new file mode 100644 index 0000000..92adabd --- /dev/null +++ b/server/domains/news/pollers/EdgarPoller.ts @@ -0,0 +1,122 @@ +import { RssParser } from '../rss'; +import type { CatalystType, Logger, NormalizedStory } from '../../shared/types'; + +/** + * SEC EDGAR poller (FREE-DATA-STACK §1.3 / P1.2 Tier 2). Free forever, and + * the highest-value source: filings frequently precede the headline. + * + * Strategy: poll the site-wide "current filings" atom feed once per form + * type (4 requests/cycle total, well inside SEC fair use), map filer CIK → + * ticker via the daily-cached company_tickers.json, and emit stories only + * for universe tickers. The pipeline applies its own universe filter again — + * defense in depth. + * + * SEC requires a descriptive User-Agent with contact info: set + * EDGAR_USER_AGENT in .env (e.g. "market-screener/1.0 you@example.com"). + */ +export class EdgarPoller { + private static readonly TICKER_MAP_URL = 'https://www.sec.gov/files/company_tickers.json'; + private static readonly TICKER_MAP_TTL_MS = 24 * 60 * 60 * 1000; + + /** form type → catalyst classification (overrides keyword classify). */ + private static readonly FORMS: Array<{ form: string; catalyst: CatalystType }> = [ + { form: '8-K', catalyst: 'regulatory' }, // material events + { form: 'SC 13D', catalyst: 'ma' }, // activist stake >5% — classic pre-M&A tell + { form: 'S-4', catalyst: 'ma' }, // merger registration + { form: 'DEFM14A', catalyst: 'ma' }, // merger proxy + ]; + + private cikToTicker: Map = new Map(); + private mapExpiresAt = 0; + + constructor( + private readonly logger: Logger, + private readonly userAgent = process.env.EDGAR_USER_AGENT ?? + 'market-screener/1.0 (set EDGAR_USER_AGENT in .env)', + ) {} + + /** Fetch all form feeds and return normalized stories for universe tickers. */ + async poll(universe: Set): Promise { + if (universe.size === 0) return []; + await this.refreshTickerMap(); + + const stories: NormalizedStory[] = []; + for (const { form, catalyst } of EdgarPoller.FORMS) { + try { + const xml = await this.fetchText(EdgarPoller.feedUrl(form)); + stories.push(...this.parseFeed(xml, form, catalyst, universe)); + } catch (err) { + this.logger.warn(`EDGAR ${form} feed failed:`, (err as Error).message); + } + } + return stories; + } + + /** Parse one atom feed. Public for fixture tests. */ + parseFeed( + xml: string, + form: string, + catalyst: CatalystType, + universe: Set, + ): NormalizedStory[] { + const stories: NormalizedStory[] = []; + for (const entry of RssParser.blocks(xml, 'entry')) { + const title = RssParser.tag(entry, 'title') ?? ''; + const updated = RssParser.tag(entry, 'updated'); + const url = RssParser.link(entry); + if (!title || !url || !updated) continue; + + // Title format: "8-K - APPLE INC (0000320193) (Filer)" + const cikMatch = title.match(/\((\d{10})\)/); + if (!cikMatch) continue; + const ticker = this.cikToTicker.get(cikMatch[1]); + if (!ticker || !universe.has(ticker)) continue; + + const company = title + .replace(/^[^-]+-\s*/, '') + .replace(/\(\d{10}\)/g, '') + .replace(/\((Filer|Subject|Reporting)\)/gi, '') + .trim(); + + stories.push({ + tickers: [ticker], + headline: `${form} filing: ${company}`, + body: null, + source: 'edgar', + url, + publishedAt: new Date(updated).toISOString(), + catalystHint: catalyst, + }); + } + return stories; + } + + /** Inject a CIK→ticker map directly (tests). CIKs are 10-digit zero-padded. */ + setTickerMap(map: Map): void { + this.cikToTicker = map; + this.mapExpiresAt = Date.now() + EdgarPoller.TICKER_MAP_TTL_MS; + } + + private async refreshTickerMap(): Promise { + if (Date.now() < this.mapExpiresAt && this.cikToTicker.size > 0) return; + const raw = await this.fetchText(EdgarPoller.TICKER_MAP_URL); + const data = JSON.parse(raw) as Record; + const map = new Map(); + for (const entry of Object.values(data)) { + map.set(String(entry.cik_str).padStart(10, '0'), entry.ticker.toUpperCase()); + } + this.setTickerMap(map); + this.logger.log(`EDGAR ticker map refreshed: ${map.size} companies`); + } + + private static feedUrl(form: string): string { + const type = encodeURIComponent(form); + return `https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent&type=${type}&company=&dateb=&owner=include&count=100&output=atom`; + } + + private async fetchText(url: string): Promise { + const res = await fetch(url, { headers: { 'User-Agent': this.userAgent } }); + if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`); + return res.text(); + } +} diff --git a/server/domains/news/pollers/PrWirePoller.ts b/server/domains/news/pollers/PrWirePoller.ts new file mode 100644 index 0000000..2777f10 --- /dev/null +++ b/server/domains/news/pollers/PrWirePoller.ts @@ -0,0 +1,91 @@ +import { RssParser } from '../rss'; +import type { Logger, NormalizedStory } from '../../shared/types'; + +/** + * PR-wire RSS poller (FREE-DATA-STACK §1.4 / P1.2 Tier 3) — press releases + * that the other free feeds miss, mostly small-caps. + * + * Ticker extraction relies on the wire convention of exchange tags in the + * text: "(NYSE: ABC)", "(Nasdaq: XYZ)". Stories without an exchange tag + * produce no tickers and are dropped by the pipeline's universe filter — + * that's intentional; untagged wire stories are rarely decision-grade. + * + * Feed list is overridable: NEWS_PRWIRE_FEEDS="url1,url2" in .env + * (wire RSS URLs change occasionally — if a feed 404s, update the env var). + */ +export class PrWirePoller { + private static readonly DEFAULT_FEEDS = [ + // GlobeNewswire — public-company news + 'https://www.globenewswire.com/RssFeed/orgclass/1/feedTitle/GlobeNewswire%20-%20News%20about%20Public%20Companies', + // PR Newswire — all news releases + 'https://www.prnewswire.com/rss/news-releases-list.rss', + ]; + + private static readonly EXCHANGE_TAG = + /\((?:NYSE(?:\s+American)?|NASDAQ|Nasdaq|AMEX|CBOE|OTC(?:QB|QX|MKTS)?)\s*:\s*([A-Za-z][A-Za-z.]{0,5})\)/g; + + private readonly feeds: string[]; + + constructor( + private readonly logger: Logger, + feeds?: string[], + ) { + const env = process.env.NEWS_PRWIRE_FEEDS; + this.feeds = feeds ?? (env ? env.split(',').map((s) => s.trim()) : PrWirePoller.DEFAULT_FEEDS); + } + + async poll(): Promise { + const stories: NormalizedStory[] = []; + for (const feed of this.feeds) { + try { + const xml = await this.fetchText(feed); + stories.push(...PrWirePoller.parseFeed(xml)); + } catch (err) { + this.logger.warn(`PR-wire feed failed (${feed}):`, (err as Error).message); + } + } + return stories; + } + + /** Parse one RSS feed. Public static for fixture tests. */ + static parseFeed(xml: string): NormalizedStory[] { + const stories: NormalizedStory[] = []; + for (const item of RssParser.blocks(xml, 'item')) { + const title = RssParser.tag(item, 'title'); + const url = RssParser.link(item); + const pubDate = RssParser.tag(item, 'pubDate'); + if (!title || !url) continue; + + const description = RssParser.tag(item, 'description') ?? ''; + const tickers = PrWirePoller.extractTickers(`${title} ${description}`); + if (tickers.length === 0) continue; // no exchange tag → skip early + + stories.push({ + tickers, + headline: title, + body: description || null, + source: 'prwire', + url, + publishedAt: pubDate ? new Date(pubDate).toISOString() : new Date().toISOString(), + }); + } + return stories; + } + + /** "(NYSE: ABC)" / "(Nasdaq: XYZ)" → ['ABC', 'XYZ']. Public for tests. */ + static extractTickers(text: string): string[] { + const out = new Set(); + for (const m of text.matchAll(PrWirePoller.EXCHANGE_TAG)) { + out.add(m[1].toUpperCase()); + } + return [...out]; + } + + private async fetchText(url: string): Promise { + const res = await fetch(url, { + headers: { 'User-Agent': 'market-screener/1.0 (+rss reader)' }, + }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return res.text(); + } +} diff --git a/server/domains/news/rss.ts b/server/domains/news/rss.ts new file mode 100644 index 0000000..eb2179e --- /dev/null +++ b/server/domains/news/rss.ts @@ -0,0 +1,43 @@ +/** + * Minimal RSS/Atom extraction — enough for EDGAR atom feeds and PR-wire RSS. + * Deliberately dependency-free; if a feed outgrows this, swap in + * fast-xml-parser without touching the pollers' output shape. + */ +export class RssParser { + /** Extract raw … or … blocks. */ + static blocks(xml: string, tag: 'item' | 'entry'): string[] { + const re = new RegExp(`<${tag}[\\s>][\\s\\S]*?<\\/${tag}>`, 'g'); + return xml.match(re) ?? []; + } + + /** First occurrence of a simple tag's text content, entity-decoded. */ + static tag(block: string, name: string): string | null { + const re = new RegExp(`<${name}[^>]*>([\\s\\S]*?)<\\/${name}>`, 'i'); + const m = block.match(re); + return m ? RssParser.clean(m[1]) : null; + } + + /** Atom-style (self-closing) or RSS …. */ + static link(block: string): string | null { + const href = block.match(/]*href="([^"]+)"/i); + if (href) return RssParser.decode(href[1].trim()); + return RssParser.tag(block, 'link'); + } + + private static clean(raw: string): string { + const noCdata = raw.replace(//g, '$1'); + const noTags = noCdata.replace(/<[^>]+>/g, ' '); + return RssParser.decode(noTags).replace(/\s+/g, ' ').trim(); + } + + private static decode(s: string): string { + return s + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/�?39;/g, "'") + .replace(/'/g, "'") + .replace(/&#(\d+);/g, (_, n) => String.fromCharCode(Number(n))); + } +} diff --git a/server/domains/screener/screener.controller.ts b/server/domains/screener/screener.controller.ts index 0e06581..22d2d2d 100644 --- a/server/domains/screener/screener.controller.ts +++ b/server/domains/screener/screener.controller.ts @@ -1,15 +1,42 @@ import type { FastifyInstance, FastifyRequest } from 'fastify'; import { ScreenerEngine } from './ScreenerEngine'; -import { CatalystCache, SignalSnapshotRepository } from '../../domains/shared'; +import { CatalystCache, SignalSnapshotRepository, YahooFinanceClient } from '../../domains/shared'; import type { DataHealth, LiveAssetResult, ScreenerResult } from '../../domains/shared'; +import type { NewsRepository } from '../news/NewsRepository'; import { screenSchema } from '../../domains/shared/types/schemas'; export class ScreenerController { + /** Company profiles change rarely — cache for an hour. */ + private static readonly PROFILE_TTL_MS = 60 * 60 * 1000; + private profileCache = new Map(); + + /** Sector pulse — SPDR sector ETFs as the standard proxy, cached 15 min. */ + private static readonly SECTOR_TTL_MS = 15 * 60 * 1000; + private static readonly SECTOR_ETFS: Array<{ etf: string; sector: string; name: string }> = [ + { etf: 'XLK', sector: 'TECHNOLOGY', name: 'Technology' }, + { etf: 'XLF', sector: 'FINANCIAL', name: 'Financials' }, + { etf: 'XLE', sector: 'ENERGY', name: 'Energy' }, + { etf: 'XLV', sector: 'HEALTHCARE', name: 'Healthcare' }, + { etf: 'XLC', sector: 'COMMUNICATION', name: 'Communication' }, + { etf: 'XLP', sector: 'CONSUMER_STAPLES', name: 'Staples' }, + { etf: 'XLY', sector: 'CONSUMER_DISCRETIONARY', name: 'Discretionary' }, + { etf: 'XLRE', sector: 'REIT', name: 'Real Estate' }, + { etf: 'XLI', sector: 'GENERAL', name: 'Industrials' }, + { etf: 'XLU', sector: 'GENERAL', name: 'Utilities' }, + ]; + private sectorCache: { data: unknown; expiresAt: number } | null = null; + + /** Sector drill-down (holdings + screen + news) — cached 30 min per sector. */ + private static readonly SECTOR_DETAIL_TTL_MS = 30 * 60 * 1000; + private sectorDetailCache = new Map(); + constructor( private readonly engine: ScreenerEngine, private readonly catalystCache: CatalystCache, // Optional so tests and minimal setups work without a database. private readonly snapshots?: SignalSnapshotRepository, + private readonly yahoo?: YahooFinanceClient, + private readonly news?: NewsRepository, ) {} register(app: FastifyInstance): void { @@ -24,6 +51,161 @@ export class ScreenerController { this.catalysts.bind(this), ); app.get('/api/screen/history/:ticker', this.history.bind(this)); + app.get('/api/screen/profile/:ticker', this.profile.bind(this)); + app.get('/api/screen/chart/:ticker', this.chart.bind(this)); + app.get('/api/screen/sectors', this.sectors.bind(this)); + app.get('/api/screen/sector/:sector', this.sectorDetail.bind(this)); + } + + /** + * Sector drill-down: the sector ETF's top 10 holdings, freshly screened + * (signal + advice-ready rows), plus recent news for those tickers and + * macro stories — "what's in this sector and why is it moving". + */ + private async sectorDetail(req: FastifyRequest) { + const sector = (req.params as { sector: string }).sector.toUpperCase(); + const entry = ScreenerController.SECTOR_ETFS.find((s) => s.sector === sector); + if (!entry || !this.yahoo) return { sector, etf: null, stocks: [], news: [] }; + + const cached = this.sectorDetailCache.get(sector); + if (cached && Date.now() < cached.expiresAt) return cached.data; + + const holdings = await this.yahoo.fetchTopHoldings(entry.etf, 10); + const results = holdings.length > 0 ? await this.engine.screenTickers(holdings) : null; + const stocks = results + ? ScreenerController.serializeAssets(results.STOCK as LiveAssetResult[]) + : []; + + // News: stored stories for these tickers (last 3 days), deduped by URL + const newsSince = new Date(Date.now() - 3 * 24 * 60 * 60 * 1000).toISOString().slice(0, 10); + const byUrl = new Map(); + if (this.news) { + for (const ticker of holdings) { + for (const row of this.news.newsForTicker(ticker, newsSince)) { + byUrl.set(row.url, { + headline: row.headline, + tickers: JSON.parse(row.ticker_list), + source: row.source, + catalyst: row.catalyst, + url: row.url, + publishedAt: row.published_at, + }); + } + } + } + + const data = { + sector, + etf: entry.etf, + name: entry.name, + stocks, + news: [...byUrl.values()], + }; + this.sectorDetailCache.set(sector, { + data, + expiresAt: Date.now() + ScreenerController.SECTOR_DETAIL_TTL_MS, + }); + return data; + } + + /** + * Sector pulse — today's % change per sector via SPDR sector ETFs (the + * standard proxy). Returns sectors sorted best→worst plus the leader. + */ + private async sectors() { + if (this.sectorCache && Date.now() < this.sectorCache.expiresAt) { + return this.sectorCache.data; + } + if (!this.yahoo) return { asOf: null, leader: null, sectors: [] }; + + const results = await Promise.all( + ScreenerController.SECTOR_ETFS.map(async ({ etf, sector, name }) => { + try { + const summary = await this.yahoo!.fetchSummary(etf); + const pr = summary?.price ?? {}; + const price = pr.regularMarketPrice ?? null; + const prev = pr.regularMarketPreviousClose ?? null; + const changePct = + price != null && prev != null && prev > 0 + ? +(((price - prev) / prev) * 100).toFixed(2) + : null; + return { etf, sector, name, changePct }; + } catch { + return { etf, sector, name, changePct: null }; + } + }), + ); + + const sectors = results + .filter((s) => s.changePct != null) + .sort((a, b) => (b.changePct as number) - (a.changePct as number)); + + const data = { + asOf: new Date().toISOString(), + leader: sectors[0] ?? null, + sectors, + }; + this.sectorCache = { data, expiresAt: Date.now() + ScreenerController.SECTOR_TTL_MS }; + return data; + } + + /** Company profile for the ticker modal — name, description, sector. */ + private async profile(req: FastifyRequest) { + const ticker = (req.params as { ticker: string }).ticker.toUpperCase(); + if (!this.yahoo) return { ticker, profile: null }; + + const cached = this.profileCache.get(ticker); + if (cached && Date.now() < cached.expiresAt) return cached.data; + + try { + const summary = await this.yahoo.fetchSummary(ticker); + const ap = summary?.assetProfile ?? {}; + const pr = summary?.price ?? {}; + const fd = summary?.financialData ?? {}; + const price = pr.regularMarketPrice ?? null; + const targetMean = fd.targetMeanPrice ?? null; + const data = { + ticker, + profile: { + name: pr.longName ?? pr.shortName ?? ticker, + summary: ap.longBusinessSummary ?? null, + sector: ap.sector ?? null, + industry: ap.industry ?? null, + website: ap.website ?? null, + employees: ap.fullTimeEmployees ?? null, + marketCap: pr.marketCap ?? null, + currentPrice: price, + // Analyst price targets (Yahoo sell-side consensus) + targets: { + mean: targetMean, + high: fd.targetHighPrice ?? null, + low: fd.targetLowPrice ?? null, + analysts: fd.numberOfAnalystOpinions ?? null, + recommendationMean: fd.recommendationMean ?? null, // 1=Strong Buy … 5=Strong Sell + upsidePct: + targetMean != null && price != null && price > 0 + ? +(((targetMean - price) / price) * 100).toFixed(1) + : null, + }, + }, + }; + this.profileCache.set(ticker, { + data, + expiresAt: Date.now() + ScreenerController.PROFILE_TTL_MS, + }); + return data; + } catch { + return { ticker, profile: null }; + } + } + + /** Closes for the ticker modal chart. ?range=1d|5d|1mo|3mo|6mo|1y. */ + private async chart(req: FastifyRequest) { + const ticker = (req.params as { ticker: string }).ticker.toUpperCase(); + const raw = (req.query as { range?: string }).range ?? '6mo'; + const range = raw in YahooFinanceClient.CHART_RANGES ? raw : '6mo'; + if (!this.yahoo) return { ticker, range, points: [] }; + return { ticker, range, points: await this.yahoo.fetchCloses(ticker, range) }; } /** Signal snapshot history for one ticker (P0.1 ledger read side). */ @@ -65,6 +247,7 @@ export class ScreenerController { const tickers = (req.body as { tickers: string[] }).tickers.map((t) => t.toUpperCase()); const results = await this.engine.screenTickers(tickers); this.recordSnapshots(results, req); + this.flagTurnarounds(results); const dataHealth = ScreenerController.assessDataHealth(results); if (dataHealth.degraded) { req.log?.warn?.({ dataHealth }, 'screen batch returned degraded fundamentals data'); @@ -78,6 +261,35 @@ export class ScreenerController { }; } + /** + * Turnaround-watch (candidate flag, NOT a prediction): the stock's style is + * already Turnaround (earnings down, revenue holding) AND its fundamental + * score improved vs the previous snapshot in the ledger. Both legs must + * hold — style alone is static, improvement alone is noise. + */ + private flagTurnarounds(results: ScreenerResult): void { + if (!this.snapshots) return; + for (const row of results.STOCK as LiveAssetResult[]) { + const metrics = row.asset.metrics as { growthCategory?: string }; + if (metrics?.growthCategory !== 'Turnaround') continue; + if (row.fundamental.tier === 'REJECT' || row.fundamental.score == null) continue; + + try { + // History includes today's snapshot (recorded just above) — compare + // today's score against the most recent prior day with a score. + const history = this.snapshots.history(row.asset.ticker); + const prior = [...history] + .reverse() + .find((h) => h.snapshot_date < history[history.length - 1]?.snapshot_date); + if (prior?.fundamental_score != null && row.fundamental.score > prior.fundamental_score) { + row.turnaroundWatch = true; + } + } catch { + // best-effort — never fail the screen for a highlight + } + } + } + /** * P0.4 data-sanity sentinel — if a large share of screened stocks come back * with null core fundamentals (P/E, ROE), the upstream source has likely diff --git a/server/domains/screener/transform/DataMapper.ts b/server/domains/screener/transform/DataMapper.ts index f956758..2d6f3d9 100644 --- a/server/domains/screener/transform/DataMapper.ts +++ b/server/domains/screener/transform/DataMapper.ts @@ -40,6 +40,13 @@ export class DataMapper { const currentPrice = pr.regularMarketPrice ?? 0; const sharesOutstanding = ks.sharesOutstanding ?? 0; + + // Today's % change — powers the sector drill-down "Today" sort + const prevClose = pr.regularMarketPreviousClose ?? null; + const dayChangePct = + prevClose != null && prevClose > 0 && (currentPrice as number) > 0 + ? +((((currentPrice as number) - prevClose) / prevClose) * 100).toFixed(2) + : null; const operatingCashflow = fd.operatingCashflow ?? 0; const freeCashflow = fd.freeCashflow ?? 0; @@ -131,6 +138,7 @@ export class DataMapper { ? (sd.trailingAnnualDividendYield as number) * 100 : null, beta: sd.beta ?? null, + dayChangePct, week52High, week52Low, week52Change, diff --git a/server/domains/shared/adapters/YahooFinanceClient.ts b/server/domains/shared/adapters/YahooFinanceClient.ts index 2729fb1..9931bcb 100644 --- a/server/domains/shared/adapters/YahooFinanceClient.ts +++ b/server/domains/shared/adapters/YahooFinanceClient.ts @@ -1,5 +1,5 @@ import YahooFinance from 'yahoo-finance2'; -import type { YahooNewsItem, YahooSearchOptions, YahooFinanceLib } from '../types'; +import type { YahooNewsItem, YahooSearchOptions, YahooFinanceLib, PricePoint } from '../types'; import { YAHOO_MODULES } from '../config/constants'; export class YahooFinanceClient { @@ -49,4 +49,71 @@ export class YahooFinanceClient { const { news = [] } = await this.lib.search(query, opts); return news; } + + /** + * Top holdings of an ETF (ticker symbols, largest weight first). + * Used for sector drill-down. Returns [] on any failure. + */ + async fetchTopHoldings(etf: string, limit = 10): Promise { + try { + const result = await this.lib.quoteSummary( + YahooFinanceClient.normalise(etf), + { modules: ['topHoldings'] }, + { validateResult: false }, + ); + const holdings = (result?.topHoldings?.holdings ?? []) as Array<{ symbol?: string }>; + return holdings + .map((h) => h.symbol) + .filter((s): s is string => Boolean(s)) + .slice(0, limit) + .map((s) => s.toUpperCase()); + } catch { + return []; + } + } + + /** Chart range presets — Robinhood/Yahoo-style. Intraday for short ranges. */ + static readonly CHART_RANGES: Record = { + '1d': { days: 1, interval: '5m' }, + '5d': { days: 5, interval: '30m' }, + '1mo': { days: 30, interval: '1d' }, + '3mo': { days: 91, interval: '1d' }, + '6mo': { days: 182, interval: '1d' }, + ytd: { days: 0, interval: '1d' }, // days computed dynamically (Jan 1 → now) + '1y': { days: 365, interval: '1d' }, + '5y': { days: 1826, interval: '1wk' }, // weekly bars keep ~260 points + }; + + /** + * Closing prices for a named range (ticker modal chart). Intraday ranges + * keep the full timestamp; daily ranges keep the date only. + * Returns [] on any failure — the chart is a nice-to-have, never a blocker. + */ + async fetchCloses(ticker: string, range = '6mo'): Promise { + const preset = YahooFinanceClient.CHART_RANGES[range] ?? YahooFinanceClient.CHART_RANGES['6mo']; + try { + const period1 = + range === 'ytd' + ? new Date(Date.UTC(new Date().getUTCFullYear(), 0, 1)) + : new Date(Date.now() - preset.days * 24 * 60 * 60 * 1000); + const result = await this.lib.chart( + YahooFinanceClient.normalise(ticker), + { period1, interval: preset.interval }, + { validateResult: false }, + ); + const quotes = (result?.quotes ?? []) as Array<{ date?: string | Date; close?: number }>; + const intraday = preset.interval !== '1d'; + return quotes + .filter((q) => q.close != null && q.date != null) + .map((q) => { + const iso = new Date(q.date as string | Date).toISOString(); + return { + date: intraday ? iso : iso.slice(0, 10), + close: +(q.close as number).toFixed(2), + }; + }); + } catch { + return []; + } + } } diff --git a/server/domains/shared/db/queries.constant.ts b/server/domains/shared/db/queries.constant.ts index 2eac821..bc3db29 100644 --- a/server/domains/shared/db/queries.constant.ts +++ b/server/domains/shared/db/queries.constant.ts @@ -163,6 +163,68 @@ export const UNIVERSE_QUERIES = { WHERE type != 'crypto' ORDER BY ticker `, + + // Every ticker screened recently (snapshot ledger) — part of the news universe + DISTINCT_SNAPSHOT_TICKERS_SINCE: ` + SELECT DISTINCT ticker FROM signal_snapshots + WHERE snapshot_date >= ? + ORDER BY ticker + `, +}; + +// ── News Queries (FREE-DATA-STACK §2–5 — free-tier news pipeline) ─────────── + +export const NEWS_QUERIES = { + // INSERT OR IGNORE — url_hash PK is the first dedupe line (returns 0 changes on dup) + INSERT_ARTICLE: ` + INSERT OR IGNORE INTO news_articles + (url_hash, title_hash, ticker_list, headline, body, source, catalyst, url, published_at, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + + // Second dedupe line: same (normalized) title seen recently → syndicated copy + TITLE_SEEN_SINCE: ` + SELECT 1 FROM news_articles + WHERE title_hash = ? AND published_at >= ? + LIMIT 1 + `, + + INSERT_CATALYST_LINK: ` + INSERT OR IGNORE INTO ticker_catalysts (ticker, day, url_hash) + VALUES (?, ?, ?) + `, + + // Per-ticker daily cap check (FREE-DATA-STACK §4.4) + COUNT_TICKER_DAY: ` + SELECT COUNT(*) AS n FROM ticker_catalysts + WHERE ticker = ? AND day = ? + `, + + // Stories for one ticker since a given day — what the UI reads (never Yahoo live) + SELECT_TICKER_NEWS: ` + SELECT a.* FROM ticker_catalysts c + JOIN news_articles a ON a.url_hash = c.url_hash + WHERE c.ticker = ? AND c.day >= ? + ORDER BY a.published_at DESC + `, + + SELECT_RECENT: ` + SELECT * FROM news_articles + ORDER BY published_at DESC + LIMIT ? + `, + + // Retention (FREE-DATA-STACK §5): purge bodies after 90d, drop unreferenced after 18mo + PURGE_BODIES_BEFORE: ` + UPDATE news_articles SET body = NULL + WHERE body IS NOT NULL AND published_at < ? + `, + + DELETE_UNREFERENCED_BEFORE: ` + DELETE FROM news_articles + WHERE published_at < ? + AND url_hash NOT IN (SELECT url_hash FROM ticker_catalysts) + `, }; // ── Signal Snapshot Queries (P0.1 — signal track record) ──────────────────── @@ -287,6 +349,31 @@ export const DDL = ` CREATE INDEX IF NOT EXISTS idx_snapshots_date ON signal_snapshots(snapshot_date); CREATE INDEX IF NOT EXISTS idx_snapshots_signal ON signal_snapshots(signal, snapshot_date); + + CREATE TABLE IF NOT EXISTS news_articles ( + url_hash TEXT PRIMARY KEY, -- sha256(url) + title_hash TEXT NOT NULL, -- sha256(normalized headline) — syndication dedupe + ticker_list TEXT NOT NULL, -- JSON array of matched universe tickers + headline TEXT NOT NULL, + body TEXT, -- nullable; purged after 90 days (retention job) + source TEXT NOT NULL, -- 'edgar' | 'prwire' | 'yahoo' + catalyst TEXT, -- 'earnings'|'ma'|'guidance'|'regulatory'|'macro'|NULL + url TEXT NOT NULL, + published_at TEXT NOT NULL, -- ISO timestamp + created_at TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS idx_news_published ON news_articles(published_at DESC); + CREATE INDEX IF NOT EXISTS idx_news_title ON news_articles(title_hash, published_at); + + CREATE TABLE IF NOT EXISTS ticker_catalysts ( + ticker TEXT NOT NULL, + day TEXT NOT NULL, -- YYYY-MM-DD (published date) + url_hash TEXT NOT NULL REFERENCES news_articles(url_hash), + PRIMARY KEY (ticker, day, url_hash) + ); + + CREATE INDEX IF NOT EXISTS idx_catalysts_ticker ON ticker_catalysts(ticker, day DESC); `; // ── Runtime migrations (ALTER TABLE for existing DBs) ──────────────────────── diff --git a/server/domains/shared/entities/Stock.ts b/server/domains/shared/entities/Stock.ts index fa120a2..c1a470d 100644 --- a/server/domains/shared/entities/Stock.ts +++ b/server/domains/shared/entities/Stock.ts @@ -34,6 +34,7 @@ export class Stock extends Asset { pFFO: data.pFFO ?? null, dividendYield: data.dividendYield ?? null, beta: data.beta ?? null, + dayChangePct: data.dayChangePct ?? null, week52High: data.week52High ?? null, week52Low: data.week52Low ?? null, week52Change: data.week52Change ?? null, @@ -192,7 +193,8 @@ export class Stock extends Asset { if (m.quickRatio != null) display['Quick'] = fmt(m.quickRatio, 2); if (m.beta != null) display['Beta'] = fmt(m.beta, 2); - // 52-week movement + // Movement + if (m.dayChangePct != null) display['Day %'] = fmtSign(m.dayChangePct, '%'); if (w52pos != null) display['52W Pos'] = w52pos; if (m.week52Change != null) display['52W Chg'] = fmtSign(m.week52Change, '%'); if (m.week52FromHigh != null) display['From High'] = fmtSign(m.week52FromHigh, '%'); diff --git a/server/domains/shared/types/asset.model.ts b/server/domains/shared/types/asset.model.ts index 3d2c692..35e99ca 100644 --- a/server/domains/shared/types/asset.model.ts +++ b/server/domains/shared/types/asset.model.ts @@ -85,6 +85,12 @@ export interface AssetResult { signal: Signal; inflated: ScoreResult; fundamental: ScoreResult; + /** + * Turnaround-watch highlight: style is Turnaround AND the fundamental + * score improved vs the previous snapshot. A candidate flag, not a + * prediction — set by the screener controller, absent for ETFs/bonds. + */ + turnaroundWatch?: boolean; } /** diff --git a/server/domains/shared/types/digest.model.ts b/server/domains/shared/types/digest.model.ts new file mode 100644 index 0000000..ec0e9d7 --- /dev/null +++ b/server/domains/shared/types/digest.model.ts @@ -0,0 +1,30 @@ +/** + * Daily change digest types (PRODUCT.md P1.1). + */ + +export interface DigestCatalyst { + headline: string; + catalyst: string | null; // 'earnings' | 'ma' | 'guidance' | 'regulatory' | 'macro' | null + source: string; // 'edgar' | 'prwire' | 'yahoo' + url: string; + publishedAt: string; +} + +/** A ticker whose signal changed since the previous snapshot. */ +export interface DigestChange { + ticker: string; + previousSignal: string; + newSignal: string; + previousDate: string; // day of the previous snapshot + scoreDelta: number | null; // fundamental score change, when both sides have one + price: number | null; + catalysts: DigestCatalyst[]; // recent stories for this ticker (the "why", maybe) +} + +export interface DigestReport { + date: string; // YYYY-MM-DD the digest covers + changes: DigestChange[]; // signal flips, strongest-impact first + newTickers: string[]; // first-ever snapshot today (no baseline to diff) + maStories: DigestCatalyst[]; // all M&A-classified stories in the window, always surfaced + snapshotCount: number; // tickers snapshotted today +} diff --git a/server/domains/shared/types/finance.model.ts b/server/domains/shared/types/finance.model.ts index 31079c3..aca8153 100644 --- a/server/domains/shared/types/finance.model.ts +++ b/server/domains/shared/types/finance.model.ts @@ -50,6 +50,7 @@ export interface YahooNewsItem { publisher: string; link: string; relatedTickers?: string[]; + providerPublishTime?: string | number | Date; } export interface YahooSearchOptions { @@ -66,6 +67,17 @@ export interface YahooFinanceLib { queryOpts?: { validateResult?: boolean }, ): Promise; search(query: string, opts?: YahooSearchOptions): Promise<{ news?: YahooNewsItem[] }>; + chart( + ticker: string, + opts: { period1: Date | string; interval?: string }, + queryOpts?: { validateResult?: boolean }, + ): Promise; +} + +/** One point of daily price history (ticker modal chart). */ +export interface PricePoint { + date: string; // YYYY-MM-DD + close: number; } // ── SimpleFIN client types ───────────────────────────────────────────────── diff --git a/server/domains/shared/types/index.ts b/server/domains/shared/types/index.ts index d8d2d87..c27fbd1 100644 --- a/server/domains/shared/types/index.ts +++ b/server/domains/shared/types/index.ts @@ -32,6 +32,7 @@ export type { YahooNewsItem, YahooSearchOptions, YahooFinanceLib, + PricePoint, SimpleFINOptions, SimpleFINTransaction, SimpleFINAccount, @@ -55,6 +56,14 @@ export type { HoldingRow, SignalSnapshotRow, } from './repositories.model'; +export type { + NewsSource, + CatalystType, + NormalizedStory, + NewsArticleRow, + IngestStats, +} from './news.model'; +export type { DigestCatalyst, DigestChange, DigestReport } from './digest.model'; export type { NumVal, SanitizedMetrics, SanitizedBondMetrics } from './scorers.model'; export type { BenchmarkProviderOptions, diff --git a/server/domains/shared/types/models.model.ts b/server/domains/shared/types/models.model.ts index e2dd4e9..f2beab1 100644 --- a/server/domains/shared/types/models.model.ts +++ b/server/domains/shared/types/models.model.ts @@ -32,6 +32,7 @@ export interface StockData { pFFO?: number | null; dividendYield?: number | null; beta?: number | null; + dayChangePct?: number | null; week52High?: number | null; week52Low?: number | null; week52Change?: number | null; @@ -66,6 +67,7 @@ export interface StockMetrics { pFFO: number | null; dividendYield: number | null; beta: number | null; + dayChangePct: number | null; week52High: number | null; week52Low: number | null; week52Change: number | null; diff --git a/server/domains/shared/types/news.model.ts b/server/domains/shared/types/news.model.ts new file mode 100644 index 0000000..219b1b5 --- /dev/null +++ b/server/domains/shared/types/news.model.ts @@ -0,0 +1,43 @@ +/** + * News pipeline types (FREE-DATA-STACK.md). + */ + +export type NewsSource = 'edgar' | 'prwire' | 'yahoo'; + +export type CatalystType = 'earnings' | 'ma' | 'guidance' | 'regulatory' | 'macro'; + +/** One story after a poller has normalized it — the only shape the pipeline accepts. */ +export interface NormalizedStory { + tickers: string[]; + headline: string; + body?: string | null; + source: NewsSource; + url: string; + publishedAt: string; // ISO timestamp + /** Poller-supplied classification (e.g. EDGAR form type); overrides keyword classify. */ + catalystHint?: CatalystType | null; +} + +/** Raw row from news_articles (snake_case, as stored). */ +export interface NewsArticleRow { + url_hash: string; + title_hash: string; + ticker_list: string; // JSON array stringified + headline: string; + body: string | null; + source: string; + catalyst: string | null; + url: string; + published_at: string; + created_at: string; +} + +/** What one ingest run did — logged by pollers and bin/poll-news. */ +export interface IngestStats { + fetched: number; + stored: number; + droppedNoUniverseTicker: number; + droppedNoise: number; + droppedDuplicate: number; + droppedCapped: number; +} diff --git a/tests/digest.test.ts b/tests/digest.test.ts new file mode 100644 index 0000000..2583e51 --- /dev/null +++ b/tests/digest.test.ts @@ -0,0 +1,191 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { DigestService } from '../server/domains/digest/DigestService.js'; +import { DiscordNotifier } from '../server/domains/digest/DiscordNotifier.js'; +import type { SignalSnapshotRepository } from '../server/domains/shared/persistence/SignalSnapshotRepository.js'; +import type { NewsRepository } from '../server/domains/news/NewsRepository.js'; +import type { NewsArticleRow, SignalSnapshotRow } from '../server/domains/shared/types/index.js'; + +function snap(over: Partial): SignalSnapshotRow { + return { + ticker: 'AAPL', + snapshot_date: '2026-06-09', + asset_type: 'STOCK', + price: 189.5, + signal: '✅ Strong Buy', + fundamental_tier: 'PASS', + fundamental_score: 9, + fundamental_label: '🟢 BUY (High Conviction)', + inflated_tier: 'PASS', + inflated_score: 9, + inflated_label: '🟢 BUY (High Conviction)', + coverage_active: 8, + coverage_total: 11, + risk_flags: null, + rate_regime: 'NORMAL', + created_at: '2026-06-09T21:00:00.000Z', + ...over, + }; +} + +function article(over: Partial): NewsArticleRow { + return { + url_hash: 'h1', + title_hash: 't1', + ticker_list: '["AAPL"]', + headline: '8-K filing: APPLE INC', + body: null, + source: 'edgar', + catalyst: 'regulatory', + url: 'https://sec.gov/x', + published_at: '2026-06-08T20:00:00.000Z', + created_at: '2026-06-08T20:01:00.000Z', + ...over, + }; +} + +function makeService( + today: SignalSnapshotRow[], + prev: SignalSnapshotRow[], + newsByTicker: Record = {}, +): DigestService { + const snapshots = { + byDate: () => today, + latestBefore: () => prev, + } as unknown as SignalSnapshotRepository; + const news = { + newsForTicker: (t: string) => newsByTicker[t] ?? [], + } as unknown as NewsRepository; + return new DigestService(snapshots, news); +} + +test('DigestService', async (t) => { + await t.test('detects signal change and attaches catalysts', () => { + const service = makeService( + [snap({ signal: '🔄 Neutral', fundamental_score: 2 })], + [snap({ snapshot_date: '2026-06-08', signal: '✅ Strong Buy', fundamental_score: 9 })], + { AAPL: [article({})] }, + ); + const report = service.build('2026-06-09'); + assert.equal(report.changes.length, 1); + const c = report.changes[0]; + assert.equal(c.previousSignal, '✅ Strong Buy'); + assert.equal(c.newSignal, '🔄 Neutral'); + assert.equal(c.scoreDelta, -7); + assert.equal(c.catalysts.length, 1); + assert.equal(c.catalysts[0].catalyst, 'regulatory'); + }); + + await t.test('no change → empty digest', () => { + const service = makeService([snap({})], [snap({ snapshot_date: '2026-06-08' })]); + const report = service.build('2026-06-09'); + assert.equal(report.changes.length, 0); + assert.equal(report.snapshotCount, 1); + }); + + await t.test('first-ever snapshot lands in newTickers, not changes', () => { + const service = makeService([snap({ ticker: 'NVDA' })], []); + const report = service.build('2026-06-09'); + assert.equal(report.changes.length, 0); + assert.deepEqual(report.newTickers, ['NVDA']); + }); + + await t.test('M&A stories surface even without a signal change', () => { + const service = makeService( + [snap({})], + [snap({ snapshot_date: '2026-06-08' })], // same signal — no change + { + AAPL: [ + article({ + catalyst: 'ma', + headline: 'SC 13D filing: APPLE INC', + url_hash: 'h2', + url: 'https://sec.gov/13d', + }), + ], + }, + ); + const report = service.build('2026-06-09'); + assert.equal(report.changes.length, 0); + assert.equal(report.maStories.length, 1); + assert.ok(report.maStories[0].headline.includes('SC 13D')); + }); + + await t.test('sorts changes by signal-distance impact', () => { + const service = makeService( + [ + snap({ ticker: 'SMALL', signal: '⚡ Momentum' }), // Strong Buy(0) → Momentum(1): impact 1 + snap({ ticker: 'BIG', signal: '❌ Avoid' }), // Strong Buy(0) → Avoid(4): impact 4 + ], + [ + snap({ ticker: 'SMALL', snapshot_date: '2026-06-08', signal: '✅ Strong Buy' }), + snap({ ticker: 'BIG', snapshot_date: '2026-06-08', signal: '✅ Strong Buy' }), + ], + ); + const report = service.build('2026-06-09'); + assert.equal(report.changes[0].ticker, 'BIG'); + assert.equal(report.changes[1].ticker, 'SMALL'); + }); +}); + +test('DiscordNotifier.buildPayload', async (t) => { + await t.test('returns null when nothing to report', () => { + assert.equal( + DiscordNotifier.buildPayload({ + date: '2026-06-09', + changes: [], + newTickers: [], + maStories: [], + snapshotCount: 5, + }), + null, + ); + }); + + await t.test('builds embed with change fields and M&A section', () => { + const payload = DiscordNotifier.buildPayload({ + date: '2026-06-09', + changes: [ + { + ticker: 'AAPL', + previousSignal: '✅ Strong Buy', + newSignal: '🔄 Neutral', + previousDate: '2026-06-08', + scoreDelta: -7, + price: 189.5, + catalysts: [ + { + headline: '8-K filing: APPLE INC', + catalyst: 'regulatory', + source: 'edgar', + url: 'https://sec.gov/x', + publishedAt: '2026-06-08T20:00:00.000Z', + }, + ], + }, + ], + newTickers: [], + maStories: [ + { + headline: 'SC 13D filing: APPLE INC', + catalyst: 'ma', + source: 'edgar', + url: 'https://sec.gov/13d', + publishedAt: '2026-06-08T21:00:00.000Z', + }, + ], + snapshotCount: 12, + }); + assert.ok(payload); + const embed = payload.embeds[0] as { + title: string; + fields: Array<{ name: string; value: string }>; + }; + assert.ok(embed.title.includes('2026-06-09')); + assert.equal(embed.fields.length, 2); // 1 change + 1 M&A section + assert.ok(embed.fields[0].name.includes('AAPL')); + assert.ok(embed.fields[0].name.includes('score -7')); + assert.ok(embed.fields[0].value.includes('regulatory')); + assert.ok(embed.fields[1].name.includes('M&A')); + }); +}); diff --git a/tests/news-pipeline.test.ts b/tests/news-pipeline.test.ts new file mode 100644 index 0000000..41cbdf5 --- /dev/null +++ b/tests/news-pipeline.test.ts @@ -0,0 +1,129 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { NewsPipeline } from '../server/domains/news/NewsPipeline.js'; +import type { NewsRepository } from '../server/domains/news/NewsRepository.js'; +import type { NormalizedStory } from '../server/domains/shared/types/index.js'; + +/** In-memory stub that records what the pipeline stores. */ +class StubRepo { + articles: Array<{ urlHash: string; tickers: string[]; catalyst: string | null }> = []; + links: Array<{ ticker: string; day: string }> = []; + seenTitles = new Set(); + capCounts = new Map(); // `${ticker}|${day}` → count + + insertArticle(a: { urlHash: string; tickers: string[]; catalyst: string | null }): boolean { + if (this.articles.some((x) => x.urlHash === a.urlHash)) return false; + this.articles.push(a); + return true; + } + titleSeenSince(titleHash: string): boolean { + return this.seenTitles.has(titleHash); + } + linkTicker(ticker: string, day: string): void { + this.links.push({ ticker, day }); + } + countTickerDay(ticker: string, day: string): number { + return this.capCounts.get(`${ticker}|${day}`) ?? 0; + } + purgeBodiesBefore(): number { + return 0; + } + deleteUnreferencedBefore(): number { + return 0; + } +} + +const UNIVERSE = new Set(['AAPL', 'MSFT']); + +function story(overrides: Partial = {}): NormalizedStory { + return { + tickers: ['AAPL'], + headline: 'Apple announces quarterly results beat estimates', + source: 'prwire', + url: `https://example.com/${Math.random()}`, + publishedAt: '2026-06-09T14:00:00.000Z', + ...overrides, + }; +} + +function makePipeline(repo: StubRepo): NewsPipeline { + return new NewsPipeline(repo as unknown as NewsRepository); +} + +test('NewsPipeline', async (t) => { + await t.test('stores universe stories and links tickers', () => { + const repo = new StubRepo(); + const stats = makePipeline(repo).ingest([story()], UNIVERSE); + assert.equal(stats.stored, 1); + assert.equal(repo.links.length, 1); + assert.equal(repo.links[0].ticker, 'AAPL'); + assert.equal(repo.links[0].day, '2026-06-09'); + }); + + await t.test('drops stories with no universe ticker (§4.1)', () => { + const repo = new StubRepo(); + const stats = makePipeline(repo).ingest([story({ tickers: ['ZZZZ'] })], UNIVERSE); + assert.equal(stats.stored, 0); + assert.equal(stats.droppedNoUniverseTicker, 1); + assert.equal(repo.articles.length, 0); + }); + + await t.test('drops noise headlines, but never filings (§4.2)', () => { + const repo = new StubRepo(); + const noise = story({ headline: '5 best stocks to buy now including Apple' }); + const filing = story({ + headline: '8-K filing: 5 best stocks edge case', + source: 'edgar', + catalystHint: 'regulatory', + }); + const stats = makePipeline(repo).ingest([noise, filing], UNIVERSE); + assert.equal(stats.droppedNoise, 1); + assert.equal(stats.stored, 1); + assert.equal(repo.articles[0].catalyst, 'regulatory'); + }); + + await t.test('drops syndicated duplicates by normalized title (§4.3)', () => { + const repo = new StubRepo(); + const pipeline = makePipeline(repo); + // First copy stored; mark its normalized-title hash as seen + pipeline.ingest([story({ headline: 'Apple Beats Q2 Estimates!' })], UNIVERSE); + repo.seenTitles.add(sha256(NewsPipeline.normalizeTitle('Apple Beats Q2 Estimates!'))); + // Same story, different casing/punctuation/URL → syndicated copy + const stats = pipeline.ingest( + [story({ headline: 'APPLE BEATS Q2 ESTIMATES', url: 'https://other.com/copy' })], + UNIVERSE, + ); + assert.equal(stats.droppedDuplicate, 1); + }); + + await t.test('enforces per-ticker daily cap, filings exempt (§4.4)', () => { + const repo = new StubRepo(); + repo.capCounts.set('AAPL|2026-06-09', 25); // at cap + const wire = story(); + const filing = story({ source: 'edgar', catalystHint: 'ma', url: 'https://sec.gov/x' }); + const stats = makePipeline(repo).ingest([wire, filing], UNIVERSE); + assert.equal(stats.droppedCapped, 1); + assert.equal(stats.stored, 1); // the filing + }); + + await t.test('classifies catalysts with M&A taking priority', () => { + assert.equal(NewsPipeline.classify('Acme to be acquired by MegaCorp in Q2 deal'), 'ma'); + assert.equal(NewsPipeline.classify('Acme reports record quarterly results'), 'earnings'); + assert.equal(NewsPipeline.classify('Acme raises full-year guidance'), 'guidance'); + assert.equal(NewsPipeline.classify('FDA approval granted for Acme drug'), 'regulatory'); + assert.equal(NewsPipeline.classify('Fed holds rates steady amid CPI data'), 'macro'); + assert.equal(NewsPipeline.classify('Acme appoints new CMO'), null); + }); + + await t.test('noise detector catches listicles and target reiterations', () => { + assert.ok(NewsPipeline.isNoise('3 Top Stocks to Watch This Week')); + assert.ok(NewsPipeline.isNoise('Analyst price target raised on momentum')); + assert.ok(!NewsPipeline.isNoise('Apple announces $90B buyback')); + }); +}); + +// Helper mirroring NewsPipeline's title hashing for the dedupe test +import { createHash } from 'crypto'; +function sha256(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} diff --git a/tests/news-pollers.test.ts b/tests/news-pollers.test.ts new file mode 100644 index 0000000..723fe33 --- /dev/null +++ b/tests/news-pollers.test.ts @@ -0,0 +1,85 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { EdgarPoller } from '../server/domains/news/pollers/EdgarPoller.js'; +import { PrWirePoller } from '../server/domains/news/pollers/PrWirePoller.js'; +import { RssParser } from '../server/domains/news/rss.js'; +import { noopLogger } from '../server/domains/shared/utils/logger.js'; + +const EDGAR_ATOM = ` + + Latest Filings + + 8-K - APPLE INC (0000320193) (Filer) + + 2026-06-09T13:01:02-04:00 + urn:tag:sec.gov,2008:accession-number=0000320193-26-000001 + + + 8-K - UNKNOWN CO (0009999999) (Filer) + + 2026-06-09T13:05:00-04:00 + urn:tag:sec.gov,2008:accession-number=x + +`; + +const PRWIRE_RSS = ` + + + Acme Corp (NYSE: ACME) Announces Record Q2 Results + https://www.example.com/acme-q2 + Tue, 09 Jun 2026 12:00:00 GMT + + + + Local bakery wins award + https://www.example.com/bakery + Tue, 09 Jun 2026 11:00:00 GMT + No public companies here. + +`; + +test('news pollers', async (t) => { + await t.test('EdgarPoller maps CIK to ticker and filters by universe', () => { + const poller = new EdgarPoller(noopLogger, 'test-agent'); + poller.setTickerMap(new Map([['0000320193', 'AAPL']])); + + const stories = poller.parseFeed(EDGAR_ATOM, '8-K', 'regulatory', new Set(['AAPL'])); + assert.equal(stories.length, 1); // unknown CIK dropped + assert.deepEqual(stories[0].tickers, ['AAPL']); + assert.equal(stories[0].source, 'edgar'); + assert.equal(stories[0].catalystHint, 'regulatory'); + assert.ok(stories[0].headline.startsWith('8-K filing:')); + assert.ok(stories[0].headline.includes('APPLE INC')); + assert.ok(stories[0].url.includes('sec.gov')); + }); + + await t.test('EdgarPoller drops universe misses', () => { + const poller = new EdgarPoller(noopLogger, 'test-agent'); + poller.setTickerMap(new Map([['0000320193', 'AAPL']])); + const stories = poller.parseFeed(EDGAR_ATOM, '8-K', 'regulatory', new Set(['MSFT'])); + assert.equal(stories.length, 0); + }); + + await t.test('PrWirePoller extracts exchange-tagged tickers', () => { + const stories = PrWirePoller.parseFeed(PRWIRE_RSS); + assert.equal(stories.length, 1); // bakery story has no tickers → skipped + assert.deepEqual(stories[0].tickers.sort(), ['ACME', 'BETA']); + assert.equal(stories[0].source, 'prwire'); + assert.ok(stories[0].publishedAt.startsWith('2026-06-09')); + }); + + await t.test('extractTickers handles exchange tag variants', () => { + assert.deepEqual(PrWirePoller.extractTickers('(NYSE: ABC)'), ['ABC']); + assert.deepEqual(PrWirePoller.extractTickers('(Nasdaq: xyz)'), ['XYZ']); + assert.deepEqual(PrWirePoller.extractTickers('(NYSE American: BRK.B)'), ['BRK.B']); + assert.deepEqual(PrWirePoller.extractTickers('(OTCQB: TINY)'), ['TINY']); + assert.deepEqual(PrWirePoller.extractTickers('no tags here'), []); + }); + + await t.test('RssParser decodes entities and strips CDATA', () => { + const block = 'A & B say "hi"'; + assert.equal(RssParser.tag(block, 'title'), 'A & B say "hi"'); + const cdata = 'bold here]]>'; + assert.equal(RssParser.tag(cdata, 'description'), 'Text bold here'); + }); +}); diff --git a/ui/src/lib/api/index.ts b/ui/src/lib/api/index.ts index 248934d..970675e 100644 --- a/ui/src/lib/api/index.ts +++ b/ui/src/lib/api/index.ts @@ -3,6 +3,21 @@ // Existing imports from '$lib/api.js' continue to work via api.ts re-export. export { screenTickers, fetchCatalysts, analyzeTickers } from './screener.js'; +export { + fetchProfile, + fetchChart, + fetchTickerNews, + fetchSectorPulse, + fetchSectorDetail, +} from './screener.js'; +export type { + CompanyProfile, + PricePoint, + TickerNewsStory, + SectorPulse, + SectorPulseEntry, + SectorDetail, +} from './screener.js'; export { fetchPortfolio, addHolding, removeHolding, fetchMarketContext } from './finance.js'; export { fetchCalls, fetchCall, createCall, deleteCall, fetchCallsCalendar } from './calls.js'; export { login, register, authFetch } from './auth.js'; diff --git a/ui/src/lib/api/screener.ts b/ui/src/lib/api/screener.ts index 224d503..83630cd 100644 --- a/ui/src/lib/api/screener.ts +++ b/ui/src/lib/api/screener.ts @@ -19,6 +19,99 @@ export async function fetchCatalysts(): Promise<{ tickers: string[]; stories: Ca return res.json(); } +// ── Ticker modal data (profile + chart + news) ───────────────────────────── + +export interface AnalystTargets { + mean: number | null; + high: number | null; + low: number | null; + analysts: number | null; + recommendationMean: number | null; // 1=Strong Buy … 5=Strong Sell + upsidePct: number | null; +} + +export interface CompanyProfile { + name: string; + summary: string | null; + sector: string | null; + industry: string | null; + website: string | null; + employees: number | null; + marketCap: number | null; + currentPrice: number | null; + targets?: AnalystTargets; +} + +export type ChartRange = '1d' | '5d' | '1mo' | '3mo' | '6mo' | 'ytd' | '1y' | '5y'; + +export interface PricePoint { + date: string; + close: number; +} + +export interface TickerNewsStory { + headline: string; + tickers: string[]; + source: string; + catalyst: string | null; + url: string; + publishedAt: string; +} + +export async function fetchProfile(ticker: string): Promise { + const res = await fetch(`${BASE}/screen/profile/${encodeURIComponent(ticker)}`); + if (!res.ok) return null; + const body = (await res.json()) as { profile: CompanyProfile | null }; + return body.profile; +} + +export async function fetchChart(ticker: string, range: ChartRange = '6mo'): Promise { + const res = await fetch(`${BASE}/screen/chart/${encodeURIComponent(ticker)}?range=${range}`); + if (!res.ok) return []; + const body = (await res.json()) as { points: PricePoint[] }; + return body.points ?? []; +} + +export interface SectorPulseEntry { + etf: string; + sector: string; // internal constant: TECHNOLOGY, FINANCIAL, … + name: string; // display name + changePct: number | null; +} + +export interface SectorPulse { + asOf: string | null; + leader: SectorPulseEntry | null; + sectors: SectorPulseEntry[]; +} + +export async function fetchSectorPulse(): Promise { + const res = await fetch(`${BASE}/screen/sectors`); + if (!res.ok) return null; + return res.json(); +} + +export interface SectorDetail { + sector: string; + etf: string | null; + name?: string; + stocks: import('$lib/types.js').AssetResult[]; + news: TickerNewsStory[]; +} + +export async function fetchSectorDetail(sector: string): Promise { + const res = await fetch(`${BASE}/screen/sector/${encodeURIComponent(sector)}`); + if (!res.ok) return null; + return res.json(); +} + +export async function fetchTickerNews(ticker: string, days = 14): Promise { + const res = await fetch(`${BASE}/news/${encodeURIComponent(ticker)}?days=${days}`); + if (!res.ok) return []; + const body = (await res.json()) as { stories: TickerNewsStory[] }; + return body.stories ?? []; +} + export async function analyzeTickers( tickers: string[], ): Promise<{ analysis: LLMAnalysis | null; reason?: string | null }> { diff --git a/ui/src/lib/components/screener/AssetTable.svelte b/ui/src/lib/components/screener/AssetTable.svelte index 40b5828..a4d10da 100644 --- a/ui/src/lib/components/screener/AssetTable.svelte +++ b/ui/src/lib/components/screener/AssetTable.svelte @@ -1,8 +1,9 @@ + +{#if s.sectorFilter} +
+
+

{(s.sectorDetail?.name ?? s.sectorFilter).toUpperCase()} — TOP HOLDINGS

+ {sortedStocks.length} + {#if pulseEntry?.changePct != null} + = 0} class:neg={pulseEntry.changePct < 0}> + {pulseEntry.changePct >= 0 ? '+' : ''}{pulseEntry.changePct.toFixed(2)}% today + + {/if} + {#if s.sectorDetail?.etf} + via {s.sectorDetail.etf} + {/if} + +
+ + +
+ + +
+ + {#if s.sectorDetailLoading} +
+ {:else if s.sectorDetail} + {#if sortedStocks.length > 0} +
+ + + + + + + + + + + + + {#each sortedStocks as r} + {@const m = r.asset.displayMetrics ?? {}} + {@const adv = adviceFor(r)} + + + + + + + + + {/each} + +
TickerPriceToday1YSignalAdvice
+ + {m['Price'] ?? '—'}{m['Day %'] ?? '—'}{m['52W Chg'] ?? '—'} + + {(r.signal ?? '—').replace(/^[^\w\s]+\s*/, '').trim()} + + {adv.text}
+
+ {:else} +
Couldn't load holdings for this sector right now.
+ {/if} + +
+
Recent sector news (3 days)
+ {#if s.sectorDetail.news.length > 0} +
    + {#each s.sectorDetail.news.slice(0, 6) as story} +
  • + {story.headline} + {story.tickers.join(', ')} · {fmtDate(story.publishedAt)} +
    • + {/each} +
+ {:else} +
+ No stored stories for these tickers in the last 3 days — often the honest answer is + "no sector-specific catalyst; the whole market moved." News accumulates as the pollers run. +
+ {/if} +
+ {/if} +
+{/if} + +{#if tickerModal} + (tickerModal = null)} + /> +{/if} + + diff --git a/ui/src/lib/components/screener/SectorPulse.svelte b/ui/src/lib/components/screener/SectorPulse.svelte new file mode 100644 index 0000000..9301fd4 --- /dev/null +++ b/ui/src/lib/components/screener/SectorPulse.svelte @@ -0,0 +1,194 @@ + + +
+ {#if s.sectorPulseLoading} +
+ Market pulse + loading sector data… +
+ {:else if !s.sectorPulse || s.sectorPulse.sectors.length === 0} +
+ Market pulse + sector data unavailable right now — retrying on next page load +
+ {:else} + {@const pulse = s.sectorPulse} +
+ Market pulse + {#if pulse.leader} + + {pulse.leader.name} leads today + {fmtPct(pulse.leader.changePct)} + + {/if} + {#if asOfLabel} + sector ETFs · {asOfLabel} + {/if} + {#if s.sectorFilter} + + {/if} +
+ +
+ {#each pulse.sectors as sec} + + {/each} +
+ {/if} +
+ + diff --git a/ui/src/lib/components/screener/TickerModal.svelte b/ui/src/lib/components/screener/TickerModal.svelte new file mode 100644 index 0000000..30442b8 --- /dev/null +++ b/ui/src/lib/components/screener/TickerModal.svelte @@ -0,0 +1,589 @@ + + + + + + + + diff --git a/ui/src/lib/stores/screener.store.svelte.ts b/ui/src/lib/stores/screener.store.svelte.ts index a2adf49..ca160ae 100644 --- a/ui/src/lib/stores/screener.store.svelte.ts +++ b/ui/src/lib/stores/screener.store.svelte.ts @@ -1,4 +1,11 @@ -import { fetchCatalysts, screenTickers, analyzeTickers } from '$lib/api.js'; +import { + fetchCatalysts, + screenTickers, + analyzeTickers, + fetchSectorPulse, + fetchSectorDetail, +} from '$lib/api.js'; +import type { SectorPulse, SectorDetail } from '$lib/api/screener.js'; import { sorted } from '$lib/utils.js'; import type { ScreenerResult, AssetType, SidebarState } from '$lib/types.js'; @@ -31,6 +38,42 @@ class ScreenerStore { this.results ? sorted([...this.results.STOCK, ...this.results.ETF, ...this.results.BOND]) : [], ); + // ── Sector pulse (daily % change per sector via SPDR ETFs) ────────────── + sectorPulse = $state(null); + sectorPulseLoading = $state(true); + /** Selected sector — drives the sector drill-down panel only. */ + sectorFilter = $state(null); + + // Sector drill-down panel (top holdings screened + sector news) + sectorDetail = $state(null); + sectorDetailLoading = $state(false); + + async loadSectorPulse(): Promise { + this.sectorPulseLoading = true; + try { + this.sectorPulse = await fetchSectorPulse(); + } catch { + this.sectorPulse = null; + } finally { + this.sectorPulseLoading = false; + } + } + + /** Select a sector: filter the table and load the drill-down panel. */ + async selectSector(sector: string | null): Promise { + this.sectorFilter = sector; + this.sectorDetail = null; + if (!sector) return; + this.sectorDetailLoading = true; + try { + this.sectorDetail = await fetchSectorDetail(sector); + } catch { + this.sectorDetail = null; + } finally { + this.sectorDetailLoading = false; + } + } + // ── Actions ──────────────────────────────────────────────────────── async screen(): Promise { this.error = null; diff --git a/ui/src/lib/utils/advice.ts b/ui/src/lib/utils/advice.ts new file mode 100644 index 0000000..055cb3e --- /dev/null +++ b/ui/src/lib/utils/advice.ts @@ -0,0 +1,140 @@ +/** + * Plain-language advice line (personal-use layer). + * + * Translates the screener's signal + volatility markers into one sentence a + * human can act on. Deterministic and derived ONLY from data already on the + * row — it adds wording, never new judgment. Tones: + * buy → green "buy — stable growth" + * mindful → amber "buy, but expect dips" + * caution → orange "risky / expensive" + * wait → blue "no edge right now" + * skip → red "fundamentals don't support it" + * unknown → gray "not enough data" + */ + +import type { AssetResult } from '$lib/types.js'; + +export type AdviceTone = 'buy' | 'mindful' | 'caution' | 'wait' | 'skip' | 'unknown'; + +export interface Advice { + text: string; + tone: AdviceTone; + detail: string; // tooltip — why this advice + /** True when the advice says something the signal pill doesn't already convey. */ + addsInfo: boolean; +} + +/** Parse "1.85" / "-23.4%" / "—" out of a display metric. */ +function num(v: string | number | null | undefined): number | null { + if (v == null || v === '—') return null; + const n = parseFloat(String(v).replace(/[%$,x+]/g, '')); + return Number.isFinite(n) ? n : null; +} + +export function adviceFor(row: AssetResult): Advice { + const m = row.asset.displayMetrics ?? {}; + const signal = row.signal ?? ''; + const coverage = row.fundamental?.audit?.coverage; + + // Not enough data → say so, don't fake confidence + if (coverage && coverage.active === 0) { + return { + text: "Can't judge — not enough data", + tone: 'unknown', + detail: 'No scoring factors had data for this asset. Treat any verdict as meaningless.', + addsInfo: true, + }; + } + + // Volatility / drawdown markers (any one makes a buy "bumpy") + const beta = num(m['Beta']); + const fromHigh = num(m['From High']); + const chg52 = num(m['52W Chg']); + const style = String(m['Style'] ?? ''); + const bumpy = + (beta != null && beta > 1.25) || + (fromHigh != null && fromHigh <= -20) || + (chg52 != null && chg52 <= -25) || + style === 'High Growth' || + style === 'Turnaround'; + + if (signal.includes('Strong Buy')) { + return bumpy + ? { + text: 'Buy, but expect dips — long-term growth', + tone: 'mindful', + detail: + 'Passes both the strict value gates and market-adjusted gates, but it moves sharply ' + + `(beta ${beta ?? '—'}, ${fromHigh ?? '—'}% off its high). Falls are likely along the way; ` + + 'the fundamentals say the trend supports holding through them.', + addsInfo: true, + } + : { + text: 'Buy — stable growth', + tone: 'buy', + detail: + 'Passes both the strict value gates and market-adjusted gates with calm price behavior. ' + + 'The closest thing this screener has to a steady compounder.', + addsInfo: true, + }; + } + + if (signal.includes('Momentum')) { + return { + text: 'Be mindful — rising on momentum, can fall fast', + tone: 'mindful', + detail: + 'Acceptable at today’s market prices but does not pass the strict value gates. ' + + 'Fine while the market is kind; expect sharper falls when it isn’t.', + addsInfo: false, + }; + } + + if (signal.includes('Speculation')) { + return { + text: 'Caution — priced for perfection', + tone: 'caution', + detail: + 'Only passes the loosened market-adjusted gates and fails fundamentals. ' + + 'Buy only with money you can watch swing hard.', + addsInfo: false, + }; + } + + if (signal.includes('Neutral')) { + return { + text: 'Wait — no clear edge right now', + tone: 'wait', + detail: + 'Neither clearly cheap nor clearly strong. Nothing here argues for buying today; ' + + 'keep it on the watchlist and let the daily digest tell you if that changes.', + addsInfo: false, + }; + } + + if (signal.includes('Avoid')) { + return { + text: 'Skip — fundamentals don’t support it', + tone: 'skip', + detail: 'Fails both the strict and the market-adjusted gates. The data says no.', + addsInfo: false, + }; + } + + return { + text: '—', + tone: 'unknown', + detail: 'No signal computed for this asset.', + addsInfo: false, + }; +} + +/** + * 💎 Quality dip: passes strict OR market-adjusted quality gates AND trades + * 10%+ below its 52-week high. A dip with a sound base — candidate to recover. + */ +export function isQualityDip(row: AssetResult): boolean { + const fromHigh = num(row.asset.displayMetrics?.['From High'] as string | undefined); + const quality = row.fundamental?.tier === 'PASS' || row.inflated?.tier === 'PASS'; + return quality && fromHigh != null && fromHigh <= -10; +} diff --git a/ui/src/lib/utils/index.ts b/ui/src/lib/utils/index.ts index 8e77ff7..504cd1f 100644 --- a/ui/src/lib/utils/index.ts +++ b/ui/src/lib/utils/index.ts @@ -1,3 +1,4 @@ export * from './sorting.js'; export * from './verdicts.js'; export * from './formatting.js'; +export * from './advice.js'; diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index 18e9bad..9c8f822 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -5,6 +5,8 @@ import AssetTable from '$lib/components/screener/AssetTable.svelte'; import AnalysisSidebar from '$lib/components/screener/AnalysisSidebar.svelte'; import WatchlistPanel from '$lib/components/screener/WatchlistPanel.svelte'; + import SectorPulse from '$lib/components/screener/SectorPulse.svelte'; + import SectorPanel from '$lib/components/screener/SectorPanel.svelte'; const s = screenerStore; @@ -19,11 +21,16 @@ if (_booted) return; _booted = true; s.reloadCatalysts(); + s.loadSectorPulse(); });
+ + + +
diff --git a/ui/src/routes/auth/forgot-password/+page.svelte b/ui/src/routes/auth/forgot-password/+page.svelte index f446b72..2b3b29d 100644 --- a/ui/src/routes/auth/forgot-password/+page.svelte +++ b/ui/src/routes/auth/forgot-password/+page.svelte @@ -71,16 +71,17 @@