refactor: production-essentials hardening pass
Backend: structured logger, env-validated config, graceful SIGTERM/SIGINT shutdown, per-IP rate limiter, per-tier scheduler concurrency latch, error context on previously-silent catches, compiled-JS Dockerfile stage. Frontend: lib/api.ts consolidates BACKEND_URL with lazy production-required check, root + per-segment error.tsx / not-found.tsx / loading.tsx, generateMetadata on park and ride pages, graceful fallback when backend is unreachable, Plausible script gated on env vars. Infra: CI runs lint + typecheck + tests on both packages before docker build, compose adds healthchecks, log rotation, and memory limits; .env.example documents every variable. Cleanup: removed empty app/api/parks/ dir and 0-byte root parks.db, moved wait-times-urls.txt into docs/, dropped an `as any` cast. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -5,7 +5,7 @@
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"start": "node dist/backend/src/index.js",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "tsx --test tests/*.test.ts"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Centralized env parsing — validate at startup, fail fast on bad config.
|
||||
* Other modules read from the frozen `config` object instead of process.env
|
||||
* so misconfiguration shows up here, not deep in a request handler.
|
||||
*/
|
||||
|
||||
import { parseStalenessHours } from "../../lib/env";
|
||||
|
||||
function parsePort(raw: string | undefined, fallback: number): number {
|
||||
if (!raw) return fallback;
|
||||
const n = parseInt(raw, 10);
|
||||
if (!Number.isFinite(n) || n < 1 || n > 65535) {
|
||||
throw new Error(`Invalid PORT=${raw}: must be an integer in 1..65535`);
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
export const config = Object.freeze({
|
||||
port: parsePort(process.env.PORT, 3001),
|
||||
parkHoursStalenessHours: parseStalenessHours(process.env.PARK_HOURS_STALENESS_HOURS, 72),
|
||||
nodeEnv: process.env.NODE_ENV ?? "development",
|
||||
rateLimitPerMin: parsePort(process.env.RATE_LIMIT_PER_MIN, 60),
|
||||
});
|
||||
|
||||
export type Config = typeof config;
|
||||
+58
-12
@@ -1,10 +1,12 @@
|
||||
import { Hono } from "hono";
|
||||
import { serve } from "@hono/node-server";
|
||||
import { cors } from "hono/cors";
|
||||
import { logger } from "hono/logger";
|
||||
|
||||
import { getDb } from "./db/index";
|
||||
import { config } from "./config";
|
||||
import { log } from "./log";
|
||||
import { getDb, closeDb } from "./db/index";
|
||||
import { startScheduler } from "./services/scheduler";
|
||||
import { rateLimit } from "./middleware/rate-limit";
|
||||
|
||||
import calendarRoutes from "./routes/calendar";
|
||||
import parksRoutes from "./routes/parks";
|
||||
@@ -13,12 +15,18 @@ import rideHistoryRoutes from "./routes/ride-history";
|
||||
import statusRoutes from "./routes/status";
|
||||
import scrapeRoutes from "./routes/scrape";
|
||||
|
||||
const PORT = parseInt(process.env.PORT ?? "3001", 10);
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.use("*", logger());
|
||||
app.use("*", async (c, next) => {
|
||||
const start = Date.now();
|
||||
await next();
|
||||
log.info("http", `${c.req.method} ${c.req.path}`, {
|
||||
status: c.res.status,
|
||||
ms: Date.now() - start,
|
||||
});
|
||||
});
|
||||
app.use("*", cors());
|
||||
app.use("*", rateLimit(config.rateLimitPerMin));
|
||||
|
||||
app.route("/api/calendar", calendarRoutes);
|
||||
app.route("/api/parks", parksRoutes);
|
||||
@@ -27,14 +35,52 @@ app.route("/api/parks", rideHistoryRoutes);
|
||||
app.route("/api/status", statusRoutes);
|
||||
app.route("/api/scrape", scrapeRoutes);
|
||||
|
||||
// Initialize database on startup
|
||||
getDb();
|
||||
console.log("[backend] database initialized");
|
||||
log.info("startup", "config loaded", {
|
||||
port: config.port,
|
||||
nodeEnv: config.nodeEnv,
|
||||
parkHoursStalenessHours: config.parkHoursStalenessHours,
|
||||
rateLimitPerMin: config.rateLimitPerMin,
|
||||
});
|
||||
|
||||
getDb();
|
||||
log.info("startup", "database initialized");
|
||||
|
||||
// Start cron scheduler
|
||||
startScheduler();
|
||||
|
||||
// Start HTTP server
|
||||
serve({ fetch: app.fetch, port: PORT }, (info) => {
|
||||
console.log(`[backend] listening on http://localhost:${info.port}`);
|
||||
const server = serve({ fetch: app.fetch, port: config.port }, (info) => {
|
||||
log.info("startup", "listening", { url: `http://localhost:${info.port}` });
|
||||
});
|
||||
|
||||
let shuttingDown = false;
|
||||
function shutdown(signal: string): void {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
log.info("shutdown", "signal received", { signal });
|
||||
|
||||
const forceExit = setTimeout(() => {
|
||||
log.error("shutdown", "force-exiting after 5s grace period");
|
||||
process.exit(1);
|
||||
}, 5000);
|
||||
forceExit.unref();
|
||||
|
||||
server.close((err) => {
|
||||
if (err) log.error("shutdown", "http server close error", { err: err.message });
|
||||
else log.info("shutdown", "http server closed");
|
||||
try {
|
||||
closeDb();
|
||||
log.info("shutdown", "database closed");
|
||||
} catch (err) {
|
||||
log.error("shutdown", "database close error", { err: (err as Error).message });
|
||||
}
|
||||
process.exit(0);
|
||||
});
|
||||
}
|
||||
|
||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||
process.on("unhandledRejection", (reason) => {
|
||||
log.error("process", "unhandledRejection", { reason: String(reason) });
|
||||
});
|
||||
process.on("uncaughtException", (err) => {
|
||||
log.error("process", "uncaughtException", { err: err.message, stack: err.stack });
|
||||
});
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Tiny structured logger. Emits `[ISO] [LEVEL] [tag] msg key=value...` so logs
|
||||
* are searchable and grep-friendly without dragging in pino/winston.
|
||||
*/
|
||||
|
||||
type Meta = Record<string, unknown>;
|
||||
type Level = "INFO" | "WARN" | "ERROR";
|
||||
|
||||
function formatMeta(meta?: Meta): string {
|
||||
if (!meta) return "";
|
||||
const parts: string[] = [];
|
||||
for (const [k, v] of Object.entries(meta)) {
|
||||
if (v === undefined) continue;
|
||||
const s = typeof v === "string" ? v : JSON.stringify(v);
|
||||
parts.push(`${k}=${s}`);
|
||||
}
|
||||
return parts.length ? " " + parts.join(" ") : "";
|
||||
}
|
||||
|
||||
function emit(level: Level, tag: string, msg: string, meta?: Meta): void {
|
||||
const line = `${new Date().toISOString()} [${level}] [${tag}] ${msg}${formatMeta(meta)}`;
|
||||
if (level === "ERROR") console.error(line);
|
||||
else console.log(line);
|
||||
}
|
||||
|
||||
export const log = {
|
||||
info: (tag: string, msg: string, meta?: Meta) => emit("INFO", tag, msg, meta),
|
||||
warn: (tag: string, msg: string, meta?: Meta) => emit("WARN", tag, msg, meta),
|
||||
error: (tag: string, msg: string, meta?: Meta) => emit("ERROR", tag, msg, meta),
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* Simple IP-based rate limiter. Fixed-window counter in a Map, swept on
|
||||
* each request — no external dependency, sufficient for a single-instance
|
||||
* public site. Keys honour x-forwarded-for so a reverse proxy doesn't
|
||||
* collapse every client to one bucket.
|
||||
*/
|
||||
|
||||
import type { MiddlewareHandler } from "hono";
|
||||
import { log } from "../log";
|
||||
|
||||
interface Bucket {
|
||||
count: number;
|
||||
resetAt: number;
|
||||
}
|
||||
|
||||
const WINDOW_MS = 60_000;
|
||||
|
||||
function clientIp(c: Parameters<MiddlewareHandler>[0]): string {
|
||||
const fwd = c.req.header("x-forwarded-for");
|
||||
if (fwd) return fwd.split(",")[0].trim();
|
||||
const real = c.req.header("x-real-ip");
|
||||
if (real) return real.trim();
|
||||
// @hono/node-server attaches the raw connection on c.env.incoming
|
||||
const incoming = (c.env as { incoming?: { socket?: { remoteAddress?: string } } } | undefined)?.incoming;
|
||||
return incoming?.socket?.remoteAddress ?? "unknown";
|
||||
}
|
||||
|
||||
export function rateLimit(limitPerMin: number): MiddlewareHandler {
|
||||
const buckets = new Map<string, Bucket>();
|
||||
|
||||
return async (c, next) => {
|
||||
const now = Date.now();
|
||||
const ip = clientIp(c);
|
||||
let bucket = buckets.get(ip);
|
||||
|
||||
if (!bucket || bucket.resetAt <= now) {
|
||||
bucket = { count: 0, resetAt: now + WINDOW_MS };
|
||||
buckets.set(ip, bucket);
|
||||
}
|
||||
|
||||
bucket.count++;
|
||||
|
||||
if (bucket.count > limitPerMin) {
|
||||
const retryAfter = Math.max(1, Math.ceil((bucket.resetAt - now) / 1000));
|
||||
log.warn("rate-limit", "blocked", { ip, count: bucket.count, retryAfter });
|
||||
c.header("Retry-After", String(retryAfter));
|
||||
return c.json({ error: "Too many requests" }, 429);
|
||||
}
|
||||
|
||||
// Opportunistic cleanup so the Map doesn't grow unbounded.
|
||||
if (buckets.size > 10_000) {
|
||||
for (const [k, v] of buckets) {
|
||||
if (v.resetAt <= now) buckets.delete(k);
|
||||
}
|
||||
}
|
||||
|
||||
await next();
|
||||
};
|
||||
}
|
||||
@@ -7,8 +7,14 @@ import { fetchToday } from "../../../lib/scrapers/sixflags";
|
||||
import { fetchLiveRides } from "../../../lib/scrapers/queuetimes";
|
||||
import { getDateRange, getParkMonthData, type DayData } from "../db/queries";
|
||||
import { TtlCache } from "../services/cache";
|
||||
import { log } from "../log";
|
||||
|
||||
const todayCache = new TtlCache<{ date: string; isOpen: boolean; hoursLabel?: string; specialType?: string } | null>(5 * 60 * 1000);
|
||||
type TodayCacheValue = { date: string; isOpen: boolean; hoursLabel?: string; specialType?: string } | null;
|
||||
const todayCache = new TtlCache<TodayCacheValue>(5 * 60 * 1000);
|
||||
// Tracks parks we've already attempted this TTL window so a null cache hit
|
||||
// doesn't re-fetch on every request. Same TTL as todayCache so they expire
|
||||
// together.
|
||||
const todayChecked = new TtlCache<true>(5 * 60 * 1000);
|
||||
const ridesCache = new TtlCache<{ openRides: number; openCoasters: number } | null>(5 * 60 * 1000);
|
||||
|
||||
const app = new Hono();
|
||||
@@ -34,10 +40,13 @@ app.get("/week", async (c) => {
|
||||
await Promise.all(
|
||||
PARKS.map(async (p) => {
|
||||
let live = todayCache.get(p.id);
|
||||
if (live === null && !todayCache.get(p.id + "_checked")) {
|
||||
live = await fetchToday(p.apiId).catch(() => null);
|
||||
if (live === null && todayChecked.get(p.id) === null) {
|
||||
live = await fetchToday(p.apiId).catch((err: Error) => {
|
||||
log.warn("calendar.week", "fetchToday failed", { park: p.id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
todayCache.set(p.id, live);
|
||||
todayCache.set(p.id + "_checked", true as any);
|
||||
todayChecked.set(p.id, true);
|
||||
}
|
||||
if (!live) return;
|
||||
if (!data[p.id]) data[p.id] = {};
|
||||
@@ -84,7 +93,10 @@ app.get("/week", async (c) => {
|
||||
let cached = ridesCache.get(p.id);
|
||||
if (cached === null) {
|
||||
const coasterSet = getCoasterSet(p.id);
|
||||
const result = await fetchLiveRides(QUEUE_TIMES_IDS[p.id], coasterSet).catch(() => null);
|
||||
const result = await fetchLiveRides(QUEUE_TIMES_IDS[p.id], coasterSet).catch((err: Error) => {
|
||||
log.warn("calendar.week", "fetchLiveRides failed", { park: p.id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
cached = result
|
||||
? {
|
||||
openRides: result.rides.filter((r) => r.isOpen).length,
|
||||
@@ -143,7 +155,10 @@ app.get("/:parkId/month", async (c) => {
|
||||
// Merge live today if viewing current month
|
||||
const park = PARKS.find((p) => p.id === parkId);
|
||||
if (park) {
|
||||
const liveToday = await fetchToday(park.apiId).catch(() => null);
|
||||
const liveToday = await fetchToday(park.apiId).catch((err: Error) => {
|
||||
log.warn("calendar.month", "fetchToday failed", { park: park.id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
if (liveToday) {
|
||||
monthData[today] = {
|
||||
isOpen: liveToday.isOpen,
|
||||
|
||||
@@ -10,6 +10,7 @@ import { getDayData } from "../db/queries";
|
||||
import { liveRidesCache, fastLaneCache } from "../services/live-cache";
|
||||
import { slugifyRideName } from "../../../lib/ride-slug";
|
||||
import type { LiveRidesResult } from "../../../lib/scrapers/queuetimes";
|
||||
import { log } from "../log";
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
@@ -31,7 +32,10 @@ app.get("/:id/rides", async (c) => {
|
||||
liveRides = liveRidesCache.get(id);
|
||||
if (liveRides === null) {
|
||||
const coasterSet = getCoasterSet(id);
|
||||
liveRides = await fetchLiveRides(queueTimesId, coasterSet).catch(() => null);
|
||||
liveRides = await fetchLiveRides(queueTimesId, coasterSet).catch((err: Error) => {
|
||||
log.warn("rides", "fetchLiveRides failed", { park: id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
if (liveRides) liveRidesCache.set(id, liveRides);
|
||||
}
|
||||
|
||||
@@ -46,7 +50,10 @@ app.get("/:id/rides", async (c) => {
|
||||
if (liveRides) {
|
||||
let fastLane = fastLaneCache.get(id);
|
||||
if (fastLane === null) {
|
||||
fastLane = await fetchFastLaneWaits(park.apiId).catch(() => null);
|
||||
fastLane = await fetchFastLaneWaits(park.apiId).catch((err: Error) => {
|
||||
log.warn("rides", "fetchFastLaneWaits failed", { park: id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
if (fastLane) fastLaneCache.set(id, fastLane);
|
||||
}
|
||||
if (fastLane) {
|
||||
@@ -81,7 +88,10 @@ app.get("/:id/rides", async (c) => {
|
||||
|
||||
let scheduleFallback = null;
|
||||
if (!liveRides) {
|
||||
scheduleFallback = await scrapeRidesForDay(park.apiId, today).catch(() => null);
|
||||
scheduleFallback = await scrapeRidesForDay(park.apiId, today).catch((err: Error) => {
|
||||
log.warn("rides", "scrapeRidesForDay failed", { park: id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
c.header("Cache-Control", "public, max-age=60, stale-while-revalidate=120");
|
||||
|
||||
@@ -2,68 +2,100 @@ import cron from "node-cron";
|
||||
import { scrapeToday, scrapeCurrentMonth, scrapeUpcomingMonths, scrapeFullYear } from "./scraper";
|
||||
import { sampleAllOpenParks } from "./wait-sampler";
|
||||
import { getParkDayCount } from "../db/queries";
|
||||
import { log } from "../log";
|
||||
|
||||
let initialized = false;
|
||||
|
||||
/**
|
||||
* Wrap a cron handler so a still-running prior tick is skipped instead of
|
||||
* racing it. Each tier gets its own latch — better-sqlite3's per-statement
|
||||
* locking handles row-level safety, but skipping overlap avoids redundant
|
||||
* upstream API calls and the resulting rate-limit risk.
|
||||
*/
|
||||
function withLatch(tag: string, fn: () => Promise<void>): () => Promise<void> {
|
||||
let running = false;
|
||||
return async () => {
|
||||
if (running) {
|
||||
log.warn(tag, "previous run still in progress, skipping tick");
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
try {
|
||||
await fn();
|
||||
} catch (err) {
|
||||
log.error(tag, "tick failed", { err: (err as Error).message });
|
||||
} finally {
|
||||
running = false;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function startScheduler(): void {
|
||||
if (initialized) return;
|
||||
initialized = true;
|
||||
|
||||
// Tier 1: Today — every hour during operating season (Mar-Dec)
|
||||
cron.schedule("0 * * 3-12 *", async () => {
|
||||
console.log(`[scheduler] tier-1: scraping today @ ${new Date().toISOString()}`);
|
||||
await scrapeToday().catch((err) => console.error("[scheduler] tier-1 error:", err));
|
||||
});
|
||||
cron.schedule(
|
||||
"0 * * 3-12 *",
|
||||
withLatch("scheduler.tier1", async () => {
|
||||
log.info("scheduler.tier1", "scraping today");
|
||||
await scrapeToday();
|
||||
}),
|
||||
);
|
||||
|
||||
// Tier 2: This week — every 6 hours, current month for all parks
|
||||
cron.schedule("0 */6 * * *", async () => {
|
||||
console.log(`[scheduler] tier-2: scraping current month @ ${new Date().toISOString()}`);
|
||||
await scrapeCurrentMonth().catch((err) => console.error("[scheduler] tier-2 error:", err));
|
||||
});
|
||||
cron.schedule(
|
||||
"0 */6 * * *",
|
||||
withLatch("scheduler.tier2", async () => {
|
||||
log.info("scheduler.tier2", "scraping current month");
|
||||
await scrapeCurrentMonth();
|
||||
}),
|
||||
);
|
||||
|
||||
// Tier 3: Upcoming — twice daily (3 AM, 3 PM), current + next month
|
||||
cron.schedule("0 3,15 * * *", async () => {
|
||||
console.log(`[scheduler] tier-3: scraping upcoming months @ ${new Date().toISOString()}`);
|
||||
await scrapeUpcomingMonths().catch((err) => console.error("[scheduler] tier-3 error:", err));
|
||||
});
|
||||
cron.schedule(
|
||||
"0 3,15 * * *",
|
||||
withLatch("scheduler.tier3", async () => {
|
||||
log.info("scheduler.tier3", "scraping upcoming months");
|
||||
await scrapeUpcomingMonths();
|
||||
}),
|
||||
);
|
||||
|
||||
// Tier 4: Full season — once daily at 3 AM
|
||||
cron.schedule("0 3 * * *", async () => {
|
||||
console.log(`[scheduler] tier-4: scraping full year @ ${new Date().toISOString()}`);
|
||||
await scrapeFullYear().catch((err) => console.error("[scheduler] tier-4 error:", err));
|
||||
});
|
||||
cron.schedule(
|
||||
"0 3 * * *",
|
||||
withLatch("scheduler.tier4", async () => {
|
||||
log.info("scheduler.tier4", "scraping full year");
|
||||
await scrapeFullYear();
|
||||
}),
|
||||
);
|
||||
|
||||
// Tier 5: Wait-time samples — every 5 minutes for parks open today
|
||||
cron.schedule("*/5 * * * *", async () => {
|
||||
try {
|
||||
cron.schedule(
|
||||
"*/5 * * * *",
|
||||
withLatch("scheduler.tier5", async () => {
|
||||
const r = await sampleAllOpenParks();
|
||||
console.log(
|
||||
`[scheduler] tier-5: sampled ${r.parksSampled} parks, ${r.samplesWritten} samples, ` +
|
||||
`${r.weatherDelayed} weather-delayed, ${r.errors} errors`,
|
||||
);
|
||||
} catch (err) {
|
||||
console.error("[scheduler] tier-5 error:", err);
|
||||
}
|
||||
});
|
||||
log.info("scheduler.tier5", "sample run complete", {
|
||||
parksSampled: r.parksSampled,
|
||||
samplesWritten: r.samplesWritten,
|
||||
weatherDelayed: r.weatherDelayed,
|
||||
errors: r.errors,
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
console.log("[scheduler] cron jobs registered");
|
||||
console.log(" tier-1: today — hourly (Mar-Dec)");
|
||||
console.log(" tier-2: current month — every 6h");
|
||||
console.log(" tier-3: upcoming — 3 AM + 3 PM");
|
||||
console.log(" tier-4: full year — 3 AM daily");
|
||||
console.log(" tier-5: wait samples — every 5 min");
|
||||
log.info("scheduler", "cron jobs registered", {
|
||||
tiers: "tier1=hourly(Mar-Dec) tier2=6h tier3=3am+3pm tier4=3am-daily tier5=5min",
|
||||
});
|
||||
|
||||
const existingRows = getParkDayCount();
|
||||
if (existingRows < 50) {
|
||||
console.log(`[scheduler] DB has ${existingRows} rows — running startup scrape`);
|
||||
log.info("scheduler", "running startup scrape", { existingRows });
|
||||
scrapeToday()
|
||||
.then((r) => {
|
||||
console.log(`[scheduler] startup today: ${r.fetched} fetched, ${r.updated} updated, ${r.errors} errors`);
|
||||
log.info("scheduler.startup", "today done", { fetched: r.fetched, updated: r.updated, errors: r.errors });
|
||||
return scrapeFullYear();
|
||||
})
|
||||
.then((r) => console.log(`[scheduler] startup full-year: ${r.fetched} fetched, ${r.skipped} skipped, ${r.errors} errors`))
|
||||
.catch((err) => console.error("[scheduler] startup scrape error:", err));
|
||||
.then((r) => {
|
||||
log.info("scheduler.startup", "full-year done", { fetched: r.fetched, skipped: r.skipped, errors: r.errors });
|
||||
})
|
||||
.catch((err) => log.error("scheduler.startup", "scrape failed", { err: (err as Error).message }));
|
||||
} else {
|
||||
console.log(`[scheduler] DB has ${existingRows} rows — skipping startup scrape, relying on cron`);
|
||||
log.info("scheduler", "skipping startup scrape — relying on cron", { existingRows });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import { PARKS } from "../../../lib/parks";
|
||||
import { scrapeMonth, fetchToday, RateLimitError } from "../../../lib/scrapers/sixflags";
|
||||
import { upsertDay, isMonthScraped, getDayData, transact } from "../db/queries";
|
||||
import { parseStalenessHours } from "../../../lib/env";
|
||||
import { config } from "../config";
|
||||
import { log } from "../log";
|
||||
|
||||
const DELAY_MS = 1000;
|
||||
const STALE_AFTER_MS = parseStalenessHours(process.env.PARK_HOURS_STALENESS_HOURS, 72) * 60 * 60 * 1000;
|
||||
const STALE_AFTER_MS = config.parkHoursStalenessHours * 60 * 60 * 1000;
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise<void>((r) => setTimeout(r, ms));
|
||||
@@ -54,9 +55,17 @@ export async function scrapeToday(): Promise<ScrapeResult> {
|
||||
|
||||
upsertDay(park.id, live.date, live.isOpen, live.hoursLabel, live.specialType);
|
||||
updated++;
|
||||
console.log(`[today] ${park.shortName}: updated (${live.isOpen ? "open" : "closed"}${live.hoursLabel ? " " + live.hoursLabel : ""})`);
|
||||
} catch {
|
||||
log.info("scrape.today", "updated", {
|
||||
park: park.shortName,
|
||||
isOpen: live.isOpen,
|
||||
hours: live.hoursLabel ?? null,
|
||||
});
|
||||
} catch (err) {
|
||||
errors++;
|
||||
log.warn("scrape.today", "park failed", {
|
||||
park: park.shortName,
|
||||
err: (err as Error).message,
|
||||
});
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
@@ -71,7 +80,7 @@ export async function scrapeToday(): Promise<ScrapeResult> {
|
||||
finishedAt: new Date().toISOString(),
|
||||
};
|
||||
lastScrapeResult = result;
|
||||
console.log(`[today] done: ${fetched} fetched, ${updated} updated, ${skipped} skipped, ${errors} errors`);
|
||||
log.info("scrape.today", "done", { fetched, updated, skipped, errors });
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -96,14 +105,22 @@ export async function scrapeMonths(monthList: { year: number; month: number }[],
|
||||
}
|
||||
});
|
||||
fetched++;
|
||||
console.log(`[month] ${park.shortName} ${year}-${String(month).padStart(2, "0")}: ${days.filter((d) => d.isOpen).length} open days`);
|
||||
log.info("scrape.month", "scraped", {
|
||||
park: park.shortName,
|
||||
month: `${year}-${String(month).padStart(2, "0")}`,
|
||||
openDays: days.filter((d) => d.isOpen).length,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof RateLimitError) {
|
||||
console.log(`[month] ${park.shortName}: rate limited`);
|
||||
} else {
|
||||
console.error(`[month] ${park.shortName}: error — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
errors++;
|
||||
if (err instanceof RateLimitError) {
|
||||
log.warn("scrape.month", "rate limited", { park: park.shortName });
|
||||
} else {
|
||||
log.error("scrape.month", "failed", {
|
||||
park: park.shortName,
|
||||
month: `${year}-${String(month).padStart(2, "0")}`,
|
||||
err: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
await sleep(DELAY_MS);
|
||||
}
|
||||
@@ -119,7 +136,7 @@ export async function scrapeMonths(monthList: { year: number; month: number }[],
|
||||
finishedAt: new Date().toISOString(),
|
||||
};
|
||||
lastScrapeResult = result;
|
||||
console.log(`[month] done: ${fetched} fetched, ${skipped} skipped, ${errors} errors`);
|
||||
log.info("scrape.month", "done", { fetched, skipped, errors });
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import { formatLocalDate, formatLocalTime } from "../../../lib/timezone";
|
||||
import { isWithinOperatingWindow } from "../../../lib/env";
|
||||
import { liveRidesCache, fastLaneCache } from "./live-cache";
|
||||
import { getDayData, upsertRide, insertSample, transact } from "../db/queries";
|
||||
import { log } from "../log";
|
||||
|
||||
const PARALLEL_CHUNK = 6;
|
||||
|
||||
@@ -54,7 +55,10 @@ async function samplePark(park: Park, now: Date): Promise<{
|
||||
let liveRides = liveRidesCache.get(park.id);
|
||||
if (liveRides === null) {
|
||||
const coasterSet = getCoasterSet(park.id);
|
||||
liveRides = await fetchLiveRides(queueTimesId, coasterSet).catch(() => null);
|
||||
liveRides = await fetchLiveRides(queueTimesId, coasterSet).catch((err: Error) => {
|
||||
log.warn("wait-sampler", "fetchLiveRides failed", { park: park.id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
if (liveRides) liveRidesCache.set(park.id, liveRides);
|
||||
}
|
||||
if (!liveRides || liveRides.rides.length === 0) {
|
||||
@@ -70,7 +74,10 @@ async function samplePark(park: Park, now: Date): Promise<{
|
||||
// Fast Lane — reuse cache; fetch on miss.
|
||||
let fastLane = fastLaneCache.get(park.id);
|
||||
if (fastLane === null) {
|
||||
fastLane = await fetchFastLaneWaits(park.apiId).catch(() => null);
|
||||
fastLane = await fetchFastLaneWaits(park.apiId).catch((err: Error) => {
|
||||
log.warn("wait-sampler", "fetchFastLaneWaits failed", { park: park.id, err: err.message });
|
||||
return null;
|
||||
});
|
||||
if (fastLane) fastLaneCache.set(park.id, fastLane);
|
||||
}
|
||||
|
||||
@@ -117,7 +124,10 @@ async function samplePark(park: Park, now: Date): Promise<{
|
||||
|
||||
return { ridesUpserted, samplesWritten, weatherDelayed: false, error: false };
|
||||
} catch (err) {
|
||||
console.error(`[wait-sampler] error sampling ${park.id}:`, err);
|
||||
log.error("wait-sampler", "sampling failed", {
|
||||
park: park.id,
|
||||
err: (err as Error).message,
|
||||
});
|
||||
return { ridesUpserted: 0, samplesWritten: 0, weatherDelayed: false, error: true };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,5 +17,5 @@
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "../lib/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
"exclude": ["node_modules", "dist", "../lib/api.ts"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user