feat: add Hono backend API server with tiered scheduler
Standalone Node.js backend that owns the SQLite database and serves composed data via REST endpoints. Replaces the shell-scheduled scraper with in-process node-cron tiered scheduling. Backend structure: - Hono HTTP server on port 3001 with CORS and request logging - Singleton SQLite connection with WAL mode - In-memory TTL cache for Queue-Times and fetchToday responses - Comparison check on fetchToday (read-before-write, only upserts on change) API endpoints: - GET /api/calendar/week — week schedule + live ride counts for all parks - GET /api/calendar/:parkId/month — month calendar for one park - GET /api/parks — park list with metadata - GET /api/parks/:id — single park detail - GET /api/parks/:id/rides — live rides with Queue-Times/schedule fallback - GET /api/status — health check, scrape stats - POST /api/scrape/trigger — manual scrape (scope: today/month/upcoming/full) Scheduler tiers: - Tier 1: today — hourly (Mar-Dec) - Tier 2: current month — every 6 hours - Tier 3: upcoming — twice daily (3 AM + 3 PM) - Tier 4: full year — daily at 3 AM Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,143 @@
|
||||
import { PARKS } from "../../../lib/parks";
|
||||
import { scrapeMonth, fetchToday, RateLimitError } from "../../../lib/scrapers/sixflags";
|
||||
import { upsertDay, isMonthScraped, getDayData, transact } from "../db/queries";
|
||||
import { parseStalenessHours } from "../../../lib/env";
|
||||
|
||||
const DELAY_MS = 1000;
|
||||
const STALE_AFTER_MS = parseStalenessHours(process.env.PARK_HOURS_STALENESS_HOURS, 72) * 60 * 60 * 1000;
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise<void>((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
export interface ScrapeResult {
|
||||
scope: string;
|
||||
fetched: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
updated: number;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
let lastScrapeResult: ScrapeResult | null = null;
|
||||
|
||||
export function getLastScrapeResult(): ScrapeResult | null {
|
||||
return lastScrapeResult;
|
||||
}
|
||||
|
||||
export async function scrapeToday(): Promise<ScrapeResult> {
|
||||
const startedAt = new Date().toISOString();
|
||||
let fetched = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let updated = 0;
|
||||
|
||||
for (const park of PARKS) {
|
||||
try {
|
||||
const live = await fetchToday(park.apiId);
|
||||
if (!live) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
fetched++;
|
||||
|
||||
const existing = getDayData(park.id, live.date);
|
||||
if (
|
||||
existing &&
|
||||
existing.isOpen === live.isOpen &&
|
||||
existing.hoursLabel === (live.hoursLabel ?? null) &&
|
||||
existing.specialType === (live.specialType ?? null)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
upsertDay(park.id, live.date, live.isOpen, live.hoursLabel, live.specialType);
|
||||
updated++;
|
||||
console.log(`[today] ${park.shortName}: updated (${live.isOpen ? "open" : "closed"}${live.hoursLabel ? " " + live.hoursLabel : ""})`);
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
|
||||
const result: ScrapeResult = {
|
||||
scope: "today",
|
||||
fetched,
|
||||
skipped,
|
||||
errors,
|
||||
updated,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
};
|
||||
lastScrapeResult = result;
|
||||
console.log(`[today] done: ${fetched} fetched, ${updated} updated, ${skipped} skipped, ${errors} errors`);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function scrapeMonths(monthList: { year: number; month: number }[], force = false): Promise<ScrapeResult> {
|
||||
const startedAt = new Date().toISOString();
|
||||
let fetched = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const park of PARKS) {
|
||||
for (const { year, month } of monthList) {
|
||||
if (!force && isMonthScraped(park.id, year, month, STALE_AFTER_MS)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const days = await scrapeMonth(park.apiId, year, month);
|
||||
transact(() => {
|
||||
for (const d of days) {
|
||||
upsertDay(park.id, d.date, d.isOpen, d.hoursLabel, d.specialType);
|
||||
}
|
||||
});
|
||||
fetched++;
|
||||
console.log(`[month] ${park.shortName} ${year}-${String(month).padStart(2, "0")}: ${days.filter((d) => d.isOpen).length} open days`);
|
||||
} catch (err) {
|
||||
if (err instanceof RateLimitError) {
|
||||
console.log(`[month] ${park.shortName}: rate limited`);
|
||||
} else {
|
||||
console.error(`[month] ${park.shortName}: error — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
errors++;
|
||||
}
|
||||
await sleep(DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
const result: ScrapeResult = {
|
||||
scope: `months(${monthList.map((m) => `${m.year}-${String(m.month).padStart(2, "0")}`).join(",")})`,
|
||||
fetched,
|
||||
skipped,
|
||||
errors,
|
||||
updated: fetched,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
};
|
||||
lastScrapeResult = result;
|
||||
console.log(`[month] done: ${fetched} fetched, ${skipped} skipped, ${errors} errors`);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function scrapeCurrentMonth(): Promise<ScrapeResult> {
|
||||
const now = new Date();
|
||||
return scrapeMonths([{ year: now.getFullYear(), month: now.getMonth() + 1 }]);
|
||||
}
|
||||
|
||||
export async function scrapeUpcomingMonths(): Promise<ScrapeResult> {
|
||||
const now = new Date();
|
||||
const current = { year: now.getFullYear(), month: now.getMonth() + 1 };
|
||||
const next = new Date(now.getFullYear(), now.getMonth() + 1, 1);
|
||||
const nextMonth = { year: next.getFullYear(), month: next.getMonth() + 1 };
|
||||
return scrapeMonths([current, nextMonth]);
|
||||
}
|
||||
|
||||
export async function scrapeFullYear(force = false): Promise<ScrapeResult> {
|
||||
const year = new Date().getFullYear();
|
||||
const months = Array.from({ length: 12 }, (_, i) => ({ year, month: i + 1 }));
|
||||
return scrapeMonths(months, force);
|
||||
}
|
||||
Reference in New Issue
Block a user