refactor: hardcode API IDs and coaster lists, remove Playwright discovery
Embed Six Flags API IDs directly in the park registry and snapshot coaster lists from park-meta.json into a TypeScript module. This eliminates the Playwright-based discovery script, RCDB scraper, and runtime dependency on park-meta.json — preparing for the backend API transition. - Add apiId field to Park type and all 24 park entries - Create lib/coaster-data.ts with hardcoded coaster lists - Update page components to use park.apiId and new getCoasterSet() - Remove scripts/discover.ts, lib/scrapers/rcdb.ts, lib/park-meta.ts - Remove data/park-meta.json from shared volume - Remove playwright devDependency and discover npm script - Simplify scripts/scrape.ts (no RCDB, no discovery checks) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
+7
-64
@@ -1,17 +1,13 @@
|
||||
/**
|
||||
* Scrape job — fetches 2026 operating hours for all parks from the Six Flags API.
|
||||
*
|
||||
* Prerequisite: run `npm run discover` first to populate API IDs.
|
||||
*
|
||||
* npm run scrape — skips months scraped within the last 7 days
|
||||
* npm run scrape — skips months scraped within the last 72h
|
||||
* npm run scrape:force — re-scrapes everything
|
||||
*/
|
||||
|
||||
import { openDb, upsertDay, getApiId, isMonthScraped } from "../lib/db";
|
||||
import { openDb, upsertDay, isMonthScraped } from "../lib/db";
|
||||
import { PARKS } from "../lib/parks";
|
||||
import { scrapeMonth, fetchToday, RateLimitError } from "../lib/scrapers/sixflags";
|
||||
import { readParkMeta, writeParkMeta, areCoastersStale } from "../lib/park-meta";
|
||||
import { scrapeRcdbCoasters } from "../lib/scrapers/rcdb";
|
||||
|
||||
const YEAR = 2026;
|
||||
const MONTHS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
|
||||
@@ -25,29 +21,13 @@ async function sleep(ms: number) {
|
||||
async function main() {
|
||||
const db = openDb();
|
||||
|
||||
const ready = PARKS.filter((p) => getApiId(db, p.id) !== null);
|
||||
const needsDiscovery = PARKS.filter((p) => getApiId(db, p.id) === null);
|
||||
|
||||
if (needsDiscovery.length > 0) {
|
||||
console.log(
|
||||
`⚠ ${needsDiscovery.length} park(s) need discovery first: ${needsDiscovery.map((p) => p.id).join(", ")}\n`
|
||||
);
|
||||
}
|
||||
|
||||
if (ready.length === 0) {
|
||||
console.log("No parks ready — run: npm run discover");
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Scraping ${YEAR} — ${ready.length} parks\n`);
|
||||
console.log(`Scraping ${YEAR} — ${PARKS.length} parks\n`);
|
||||
|
||||
let totalFetched = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
for (const park of ready) {
|
||||
const apiId = getApiId(db, park.id)!;
|
||||
for (const park of PARKS) {
|
||||
const label = park.shortName.padEnd(22);
|
||||
|
||||
let openDays = 0;
|
||||
@@ -65,7 +45,7 @@ async function main() {
|
||||
}
|
||||
|
||||
try {
|
||||
const days = await scrapeMonth(apiId, YEAR, month);
|
||||
const days = await scrapeMonth(park.apiId, YEAR, month);
|
||||
db.transaction(() => {
|
||||
for (const d of days) upsertDay(db, park.id, d.date, d.isOpen, d.hoursLabel, d.specialType);
|
||||
})();
|
||||
@@ -102,11 +82,10 @@ async function main() {
|
||||
|
||||
// ── Today scrape (always fresh — dateless endpoint returns current day) ────
|
||||
console.log("\n── Today's data ──");
|
||||
for (const park of ready) {
|
||||
const apiId = getApiId(db, park.id)!;
|
||||
for (const park of PARKS) {
|
||||
process.stdout.write(` ${park.shortName.padEnd(22)} `);
|
||||
try {
|
||||
const today = await fetchToday(apiId);
|
||||
const today = await fetchToday(park.apiId);
|
||||
if (today) {
|
||||
upsertDay(db, park.id, today.date, today.isOpen, today.hoursLabel, today.specialType);
|
||||
console.log(today.isOpen ? `open ${today.hoursLabel ?? ""}` : "closed");
|
||||
@@ -120,42 +99,6 @@ async function main() {
|
||||
}
|
||||
|
||||
db.close();
|
||||
|
||||
// ── RCDB coaster scrape (30-day staleness) ────────────────────────────────
|
||||
const meta = readParkMeta();
|
||||
const rcdbParks = PARKS.filter((p) => {
|
||||
const entry = meta[p.id];
|
||||
return entry?.rcdb_id && (FORCE || areCoastersStale(entry));
|
||||
});
|
||||
|
||||
if (rcdbParks.length === 0) {
|
||||
console.log("\nCoaster data up to date.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n── RCDB coaster scrape — ${rcdbParks.length} park(s) ──`);
|
||||
|
||||
for (const park of rcdbParks) {
|
||||
const entry = meta[park.id];
|
||||
const rcdbId = entry.rcdb_id!;
|
||||
process.stdout.write(` ${park.shortName.padEnd(30)} `);
|
||||
|
||||
const coasters = await scrapeRcdbCoasters(rcdbId);
|
||||
if (coasters === null) {
|
||||
console.log("FAILED");
|
||||
continue;
|
||||
}
|
||||
|
||||
entry.coasters = coasters;
|
||||
entry.coasters_scraped_at = new Date().toISOString();
|
||||
console.log(`${coasters.length} coasters`);
|
||||
|
||||
// Polite delay between RCDB requests
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
|
||||
writeParkMeta(meta);
|
||||
console.log(" Saved to data/park-meta.json");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
|
||||
Reference in New Issue
Block a user