Compare commits
5 Commits
757c2a8d4f
...
c5c9f750a3
| Author | SHA1 | Date | |
|---|---|---|---|
| c5c9f750a3 | |||
| 3815da2d3f | |||
| ccd35c4648 | |||
| 70b56158d4 | |||
| 4652a92c29 |
@@ -5,6 +5,7 @@ node_modules
|
||||
data/*.db
|
||||
data/*.db-shm
|
||||
data/*.db-wal
|
||||
backend/data
|
||||
.env*
|
||||
npm-debug.log*
|
||||
.DS_Store
|
||||
|
||||
@@ -27,10 +27,10 @@ jobs:
|
||||
push: true
|
||||
tags: ${{ vars.REGISTRY }}/${{ gitea.repository_owner }}/sixflagssupercalendar:web
|
||||
|
||||
- name: Build and push scraper image
|
||||
- name: Build and push backend image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
target: scraper
|
||||
target: backend
|
||||
push: true
|
||||
tags: ${{ vars.REGISTRY }}/${{ gitea.repository_owner }}/sixflagssupercalendar:scraper
|
||||
tags: ${{ vars.REGISTRY }}/${{ gitea.repository_owner }}/sixflagssupercalendar:backend
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# dependencies
|
||||
/node_modules
|
||||
/backend/node_modules
|
||||
/backend/dist
|
||||
/.pnp
|
||||
.pnp.*
|
||||
.yarn/*
|
||||
@@ -33,6 +35,7 @@ yarn-error.log*
|
||||
/data/*.db
|
||||
/data/*.db-shm
|
||||
/data/*.db-wal
|
||||
/backend/data/
|
||||
parks.db
|
||||
|
||||
# env files
|
||||
|
||||
+24
-32
@@ -1,19 +1,21 @@
|
||||
# Stage 1: Install all dependencies (dev included — scraper needs tsx + playwright)
|
||||
FROM node:22-bookworm-slim AS deps
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends python3 make g++ && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
# ── builder: Next.js production build ────────────────────────────────────────
|
||||
FROM node:22-bookworm-slim AS builder
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json* ./
|
||||
RUN npm ci
|
||||
|
||||
# Stage 2: Build the Next.js app
|
||||
FROM deps AS builder
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
|
||||
# ── backend-deps: backend node_modules (better-sqlite3 needs build tools) ────
|
||||
FROM node:22-bookworm-slim AS backend-deps
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends python3 make g++ && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY backend/package.json backend/package-lock.json* ./
|
||||
RUN npm ci
|
||||
|
||||
# ── web ──────────────────────────────────────────────────────────────────────
|
||||
# Minimal Next.js runner. No playwright, no tsx, no scripts.
|
||||
# next build --output standalone bundles its own node_modules (incl. better-sqlite3).
|
||||
# Minimal Next.js standalone runner. No database, no native modules.
|
||||
FROM node:22-bookworm-slim AS web
|
||||
WORKDIR /app
|
||||
|
||||
@@ -27,44 +29,34 @@ COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/public ./public
|
||||
|
||||
RUN mkdir -p /app/data && chown nextjs:nodejs /app/data
|
||||
VOLUME ["/app/data"]
|
||||
|
||||
USER nextjs
|
||||
EXPOSE 3000
|
||||
ENV PORT=3000
|
||||
ENV HOSTNAME="0.0.0.0"
|
||||
CMD ["node", "server.js"]
|
||||
|
||||
# ── scraper ───────────────────────────────────────────────────────────────────
|
||||
# Scraper-only image. No Next.js output. Runs on a nightly schedule via
|
||||
# scripts/scrape-schedule.sh. Staleness windows are configurable via env vars:
|
||||
# PARK_HOURS_STALENESS_HOURS (default: 72)
|
||||
# COASTER_STALENESS_HOURS (default: 720 = 30 days)
|
||||
FROM node:22-bookworm-slim AS scraper
|
||||
# ── backend ──────────────────────────────────────────────────────────────────
|
||||
# Hono API server + node-cron scheduler. Owns the SQLite database exclusively.
|
||||
FROM node:22-bookworm-slim AS backend
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/app/.playwright
|
||||
|
||||
RUN addgroup --system --gid 1001 nodejs && \
|
||||
adduser --system --uid 1001 nextjs
|
||||
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/scripts ./scripts
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/lib ./lib
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/tests ./tests
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/package.json ./package.json
|
||||
COPY --from=builder --chown=nextjs:nodejs /app/tsconfig.json ./tsconfig.json
|
||||
|
||||
# Full node_modules — includes tsx, playwright, better-sqlite3, all devDeps
|
||||
COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules
|
||||
|
||||
# Install Playwright Chromium + system libraries (runs as root, then fixes ownership)
|
||||
RUN npx playwright install --with-deps chromium && \
|
||||
chown -R nextjs:nodejs /app/.playwright
|
||||
COPY --from=backend-deps --chown=nextjs:nodejs /app/node_modules ./backend/node_modules
|
||||
COPY --chown=nextjs:nodejs backend/src ./backend/src
|
||||
COPY --chown=nextjs:nodejs backend/package.json ./backend/package.json
|
||||
COPY --chown=nextjs:nodejs backend/tsconfig.json ./backend/tsconfig.json
|
||||
COPY --chown=nextjs:nodejs lib ./lib
|
||||
|
||||
RUN mkdir -p /app/data && chown nextjs:nodejs /app/data
|
||||
VOLUME ["/app/data"]
|
||||
|
||||
USER nextjs
|
||||
CMD ["sh", "/app/scripts/scrape-schedule.sh"]
|
||||
EXPOSE 3001
|
||||
ENV PORT=3001
|
||||
|
||||
WORKDIR /app/backend
|
||||
CMD ["npx", "tsx", "src/index.ts"]
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { PARKS } from "@/lib/parks";
|
||||
import { openDb, getMonthCalendar } from "@/lib/db";
|
||||
import type { Park } from "@/lib/scrapers/types";
|
||||
|
||||
export interface ParksApiResponse {
|
||||
parks: Park[];
|
||||
calendar: Record<string, boolean[]>;
|
||||
month: string;
|
||||
daysInMonth: number;
|
||||
}
|
||||
|
||||
function getDaysInMonth(year: number, month: number): number {
|
||||
return new Date(year, month, 0).getDate();
|
||||
}
|
||||
|
||||
function parseMonthParam(
|
||||
monthParam: string | null
|
||||
): { year: number; month: number } | null {
|
||||
if (!monthParam) return null;
|
||||
const match = monthParam.match(/^(\d{4})-(\d{2})$/);
|
||||
if (!match) return null;
|
||||
const year = parseInt(match[1], 10);
|
||||
const month = parseInt(match[2], 10);
|
||||
if (month < 1 || month > 12) return null;
|
||||
return { year, month };
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest): Promise<NextResponse> {
|
||||
const monthParam = request.nextUrl.searchParams.get("month");
|
||||
const parsed = parseMonthParam(monthParam);
|
||||
|
||||
if (!parsed) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid or missing ?month=YYYY-MM parameter" },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const { year, month } = parsed;
|
||||
const daysInMonth = getDaysInMonth(year, month);
|
||||
|
||||
const db = openDb();
|
||||
const calendar = getMonthCalendar(db, year, month);
|
||||
db.close();
|
||||
|
||||
const response: ParksApiResponse = {
|
||||
parks: PARKS,
|
||||
calendar,
|
||||
month: `${year}-${String(month).padStart(2, "0")}`,
|
||||
daysInMonth,
|
||||
};
|
||||
|
||||
return NextResponse.json(response, {
|
||||
headers: {
|
||||
"Cache-Control": "public, s-maxage=3600, stale-while-revalidate=86400",
|
||||
},
|
||||
});
|
||||
}
|
||||
+8
-124
@@ -1,12 +1,7 @@
|
||||
import { HomePageClient } from "@/components/HomePageClient";
|
||||
import { PARKS } from "@/lib/parks";
|
||||
import { openDb, getDateRange, getApiId } from "@/lib/db";
|
||||
import { getTodayLocal, isWithinOperatingWindow, getOperatingStatus } from "@/lib/env";
|
||||
import { fetchLiveRides } from "@/lib/scrapers/queuetimes";
|
||||
import { fetchToday } from "@/lib/scrapers/sixflags";
|
||||
import { QUEUE_TIMES_IDS } from "@/lib/queue-times-map";
|
||||
import { readParkMeta, getCoasterSet } from "@/lib/park-meta";
|
||||
import type { DayData } from "@/lib/db";
|
||||
import { getTodayLocal } from "@/lib/env";
|
||||
|
||||
const BACKEND_URL = process.env.BACKEND_URL ?? "http://localhost:3001";
|
||||
|
||||
interface PageProps {
|
||||
searchParams: Promise<{ week?: string }>;
|
||||
@@ -26,125 +21,14 @@ function getWeekStart(param: string | undefined): string {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function getWeekDates(sundayIso: string): string[] {
|
||||
return Array.from({ length: 7 }, (_, i) => {
|
||||
const d = new Date(sundayIso + "T00:00:00");
|
||||
d.setDate(d.getDate() + i);
|
||||
return d.toISOString().slice(0, 10);
|
||||
});
|
||||
}
|
||||
|
||||
function getCurrentWeekStart(): string {
|
||||
const todayIso = getTodayLocal();
|
||||
const d = new Date(todayIso + "T00:00:00");
|
||||
d.setDate(d.getDate() - d.getDay());
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
export default async function HomePage({ searchParams }: PageProps) {
|
||||
const params = await searchParams;
|
||||
const weekStart = getWeekStart(params.week);
|
||||
const weekDates = getWeekDates(weekStart);
|
||||
const endDate = weekDates[6];
|
||||
const today = getTodayLocal();
|
||||
const isCurrentWeek = weekStart === getCurrentWeekStart();
|
||||
|
||||
const db = openDb();
|
||||
const data = getDateRange(db, weekStart, endDate);
|
||||
const data = await fetch(
|
||||
`${BACKEND_URL}/api/calendar/week?start=${weekStart}`,
|
||||
{ next: { revalidate: 120 } },
|
||||
).then((r) => r.json());
|
||||
|
||||
// Merge live today data from the Six Flags API (dateless endpoint, 5-min ISR cache).
|
||||
// This ensures weather delays, early closures, and hour changes surface within 5 minutes
|
||||
// without waiting for the next scheduled scrape. Only fetched when viewing the current week.
|
||||
if (weekDates.includes(today)) {
|
||||
const todayResults = await Promise.all(
|
||||
PARKS.map(async (p) => {
|
||||
const apiId = getApiId(db, p.id);
|
||||
if (!apiId) return null;
|
||||
const live = await fetchToday(apiId, 300); // 5-min ISR cache
|
||||
return live ? { parkId: p.id, live } : null;
|
||||
})
|
||||
);
|
||||
for (const result of todayResults) {
|
||||
if (!result) continue;
|
||||
const { parkId, live } = result;
|
||||
if (!data[parkId]) data[parkId] = {};
|
||||
data[parkId][today] = {
|
||||
isOpen: live.isOpen,
|
||||
hoursLabel: live.hoursLabel ?? null,
|
||||
specialType: live.specialType ?? null,
|
||||
} satisfies DayData;
|
||||
}
|
||||
}
|
||||
|
||||
db.close();
|
||||
|
||||
const scrapedCount = Object.values(data).reduce(
|
||||
(sum, parkData) => sum + Object.keys(parkData).length,
|
||||
0
|
||||
);
|
||||
|
||||
// Always fetch both ride and coaster counts — the client decides which to display.
|
||||
const parkMeta = readParkMeta();
|
||||
const hasCoasterData = PARKS.some((p) => (parkMeta[p.id]?.coasters.length ?? 0) > 0);
|
||||
|
||||
let rideCounts: Record<string, number> = {};
|
||||
let coasterCounts: Record<string, number> = {};
|
||||
let closingParkIds: string[] = [];
|
||||
let openParkIds: string[] = [];
|
||||
let weatherDelayParkIds: string[] = [];
|
||||
if (weekDates.includes(today)) {
|
||||
// Parks within operating hours right now (for open dot — independent of ride counts)
|
||||
const openTodayParks = PARKS.filter((p) => {
|
||||
const dayData = data[p.id]?.[today];
|
||||
if (!dayData?.isOpen || !dayData.hoursLabel) return false;
|
||||
return isWithinOperatingWindow(dayData.hoursLabel, p.timezone);
|
||||
});
|
||||
openParkIds = openTodayParks.map((p) => p.id);
|
||||
closingParkIds = openTodayParks
|
||||
.filter((p) => {
|
||||
const dayData = data[p.id]?.[today];
|
||||
return dayData?.hoursLabel
|
||||
? getOperatingStatus(dayData.hoursLabel, p.timezone) === "closing"
|
||||
: false;
|
||||
})
|
||||
.map((p) => p.id);
|
||||
// Only fetch ride counts for parks that have queue-times coverage
|
||||
const trackedParks = openTodayParks.filter((p) => QUEUE_TIMES_IDS[p.id]);
|
||||
const results = await Promise.all(
|
||||
trackedParks.map(async (p) => {
|
||||
const coasterSet = getCoasterSet(p.id, parkMeta);
|
||||
const result = await fetchLiveRides(QUEUE_TIMES_IDS[p.id], coasterSet, 300);
|
||||
const rideCount = result ? result.rides.filter((r) => r.isOpen).length : null;
|
||||
const coasterCount = result ? result.rides.filter((r) => r.isOpen && r.isCoaster).length : 0;
|
||||
return { id: p.id, rideCount, coasterCount };
|
||||
})
|
||||
);
|
||||
// Parks with queue-times coverage but 0 open rides = likely weather delay
|
||||
weatherDelayParkIds = results
|
||||
.filter(({ rideCount }) => rideCount === 0)
|
||||
.map(({ id }) => id);
|
||||
rideCounts = Object.fromEntries(
|
||||
results.filter(({ rideCount }) => rideCount != null && rideCount > 0).map(({ id, rideCount }) => [id, rideCount!])
|
||||
);
|
||||
coasterCounts = Object.fromEntries(
|
||||
results.filter(({ coasterCount }) => coasterCount > 0).map(({ id, coasterCount }) => [id, coasterCount])
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<HomePageClient
|
||||
weekStart={weekStart}
|
||||
weekDates={weekDates}
|
||||
today={today}
|
||||
isCurrentWeek={isCurrentWeek}
|
||||
data={data}
|
||||
rideCounts={rideCounts}
|
||||
coasterCounts={coasterCounts}
|
||||
openParkIds={openParkIds}
|
||||
closingParkIds={closingParkIds}
|
||||
weatherDelayParkIds={weatherDelayParkIds}
|
||||
hasCoasterData={hasCoasterData}
|
||||
scrapedCount={scrapedCount}
|
||||
/>
|
||||
);
|
||||
return <HomePageClient {...data} />;
|
||||
}
|
||||
|
||||
+31
-82
@@ -1,33 +1,27 @@
|
||||
import Link from "next/link";
|
||||
import { BackToCalendarLink } from "@/components/BackToCalendarLink";
|
||||
import { notFound } from "next/navigation";
|
||||
import { PARK_MAP } from "@/lib/parks";
|
||||
import { openDb, getParkMonthData, getApiId } from "@/lib/db";
|
||||
import { scrapeRidesForDay } from "@/lib/scrapers/sixflags";
|
||||
import { fetchLiveRides } from "@/lib/scrapers/queuetimes";
|
||||
import { fetchToday } from "@/lib/scrapers/sixflags";
|
||||
import { QUEUE_TIMES_IDS } from "@/lib/queue-times-map";
|
||||
import { readParkMeta, getCoasterSet } from "@/lib/park-meta";
|
||||
import { ParkMonthCalendar } from "@/components/ParkMonthCalendar";
|
||||
import { LiveRidePanel } from "@/components/LiveRidePanel";
|
||||
import type { RideStatus, RidesFetchResult } from "@/lib/scrapers/sixflags";
|
||||
import type { LiveRidesResult } from "@/lib/scrapers/queuetimes"; // used as prop type below
|
||||
import { getTodayLocal, isWithinOperatingWindow } from "@/lib/env";
|
||||
import type { LiveRidesResult } from "@/lib/scrapers/queuetimes";
|
||||
import { getTodayLocal } from "@/lib/env";
|
||||
|
||||
const BACKEND_URL = process.env.BACKEND_URL ?? "http://localhost:3001";
|
||||
|
||||
interface PageProps {
|
||||
params: Promise<{ id: string }>;
|
||||
searchParams: Promise<{ month?: string }>;
|
||||
}
|
||||
|
||||
function parseMonthParam(param: string | undefined): { year: number; month: number } {
|
||||
function parseMonthParam(param: string | undefined): string {
|
||||
if (param && /^\d{4}-\d{2}$/.test(param)) {
|
||||
const [y, m] = param.split("-").map(Number);
|
||||
if (y >= 2020 && y <= 2030 && m >= 1 && m <= 12) {
|
||||
return { year: y, month: m };
|
||||
return param;
|
||||
}
|
||||
}
|
||||
const [y, m] = getTodayLocal().split("-").map(Number);
|
||||
return { year: y, month: m };
|
||||
return getTodayLocal().slice(0, 7);
|
||||
}
|
||||
|
||||
export default async function ParkPage({ params, searchParams }: PageProps) {
|
||||
@@ -37,60 +31,30 @@ export default async function ParkPage({ params, searchParams }: PageProps) {
|
||||
const park = PARK_MAP.get(id);
|
||||
if (!park) notFound();
|
||||
|
||||
const today = getTodayLocal();
|
||||
const { year, month } = parseMonthParam(monthParam);
|
||||
const monthStr = parseMonthParam(monthParam);
|
||||
const [year, month] = monthStr.split("-").map(Number);
|
||||
|
||||
const db = openDb();
|
||||
const monthData = getParkMonthData(db, id, year, month);
|
||||
const apiId = getApiId(db, id);
|
||||
db.close();
|
||||
const [calendarData, ridesData] = await Promise.all([
|
||||
fetch(`${BACKEND_URL}/api/calendar/${id}/month?month=${monthStr}`, {
|
||||
next: { revalidate: 300 },
|
||||
}).then((r) => r.json()),
|
||||
fetch(`${BACKEND_URL}/api/parks/${id}/rides`, {
|
||||
next: { revalidate: 60 },
|
||||
}).then((r) => r.json()),
|
||||
]);
|
||||
|
||||
// Prefer live today data from the Six Flags API (5-min ISR cache) so that
|
||||
// weather delays and hour changes surface immediately rather than showing
|
||||
// stale DB values. Fall back to DB if the API call fails.
|
||||
const liveToday = apiId !== null ? await fetchToday(apiId, 300).catch(() => null) : null;
|
||||
const todayData = liveToday
|
||||
? { isOpen: liveToday.isOpen, hoursLabel: liveToday.hoursLabel ?? null, specialType: liveToday.specialType ?? null }
|
||||
: monthData[today];
|
||||
const parkOpenToday = todayData?.isOpen && todayData?.hoursLabel;
|
||||
|
||||
// ── Ride data: try live Queue-Times first, fall back to schedule ──────────
|
||||
const queueTimesId = QUEUE_TIMES_IDS[id];
|
||||
const parkMeta = readParkMeta();
|
||||
const coasterSet = getCoasterSet(id, parkMeta);
|
||||
|
||||
let liveRides: LiveRidesResult | null = null;
|
||||
let ridesResult: RidesFetchResult | null = null;
|
||||
|
||||
// Determine if we're within the 1h-before-open to 1h-after-close window.
|
||||
const withinWindow = todayData?.hoursLabel
|
||||
? isWithinOperatingWindow(todayData.hoursLabel, park.timezone)
|
||||
: false;
|
||||
|
||||
if (queueTimesId) {
|
||||
const raw = await fetchLiveRides(queueTimesId, coasterSet);
|
||||
if (raw) {
|
||||
// Outside the window: show the ride list but force all rides closed
|
||||
liveRides = withinWindow
|
||||
? raw
|
||||
: {
|
||||
...raw,
|
||||
rides: raw.rides.map((r) => ({ ...r, isOpen: false, waitMinutes: 0 })),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Weather delay: park is within operating hours but queue-times shows 0 open rides
|
||||
const isWeatherDelay =
|
||||
withinWindow &&
|
||||
liveRides !== null &&
|
||||
liveRides.rides.length > 0 &&
|
||||
liveRides.rides.every((r) => !r.isOpen);
|
||||
|
||||
// Only hit the schedule API as a fallback when Queue-Times live data is unavailable.
|
||||
if (!liveRides && apiId !== null) {
|
||||
ridesResult = await scrapeRidesForDay(apiId, today);
|
||||
}
|
||||
const { monthData, today } = calendarData;
|
||||
const {
|
||||
parkOpenToday,
|
||||
isWeatherDelay,
|
||||
liveRides,
|
||||
scheduleFallback: ridesResult,
|
||||
}: {
|
||||
parkOpenToday: boolean;
|
||||
isWeatherDelay: boolean;
|
||||
liveRides: LiveRidesResult | null;
|
||||
scheduleFallback: RidesFetchResult | null;
|
||||
} = ridesData;
|
||||
|
||||
return (
|
||||
<div style={{ minHeight: "100vh", background: "var(--color-bg)" }}>
|
||||
@@ -168,14 +132,13 @@ export default async function ParkPage({ params, searchParams }: PageProps) {
|
||||
{liveRides ? (
|
||||
<LiveRidePanel
|
||||
liveRides={liveRides}
|
||||
parkOpenToday={!!parkOpenToday}
|
||||
parkOpenToday={parkOpenToday}
|
||||
isWeatherDelay={isWeatherDelay}
|
||||
/>
|
||||
) : (
|
||||
<RideList
|
||||
ridesResult={ridesResult}
|
||||
parkOpenToday={!!parkOpenToday}
|
||||
apiIdMissing={apiId === null && !queueTimesId}
|
||||
parkOpenToday={parkOpenToday}
|
||||
/>
|
||||
)}
|
||||
</section>
|
||||
@@ -256,24 +219,10 @@ function LiveBadge() {
|
||||
function RideList({
|
||||
ridesResult,
|
||||
parkOpenToday,
|
||||
apiIdMissing,
|
||||
}: {
|
||||
ridesResult: RidesFetchResult | null;
|
||||
parkOpenToday: boolean;
|
||||
apiIdMissing: boolean;
|
||||
}) {
|
||||
if (apiIdMissing) {
|
||||
return (
|
||||
<Callout>
|
||||
Park API ID not discovered yet. Run{" "}
|
||||
<code style={{ background: "var(--color-surface-2)", padding: "1px 5px", borderRadius: 3, fontSize: "0.8em" }}>
|
||||
npm run discover
|
||||
</code>{" "}
|
||||
to enable ride data.
|
||||
</Callout>
|
||||
);
|
||||
}
|
||||
|
||||
if (!parkOpenToday) {
|
||||
return <Callout>Park is closed today — no ride schedule available.</Callout>;
|
||||
}
|
||||
|
||||
Generated
+1109
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "sixflags-backend",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "tsx watch src/index.ts",
|
||||
"build": "tsc",
|
||||
"start": "node dist/index.js",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^2.0.0",
|
||||
"better-sqlite3": "^12.8.0",
|
||||
"hono": "^4.7.0",
|
||||
"node-cron": "^3.0.3"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^22",
|
||||
"@types/node-cron": "^3.0.11",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import Database from "better-sqlite3";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
|
||||
const DATA_DIR = path.join(process.cwd(), "data");
|
||||
const DB_PATH = path.join(DATA_DIR, "parks.db");
|
||||
|
||||
let _db: Database.Database | null = null;
|
||||
|
||||
export function getDb(): Database.Database {
|
||||
if (_db) return _db;
|
||||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||
_db = new Database(DB_PATH);
|
||||
_db.pragma("journal_mode = WAL");
|
||||
_db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS park_days (
|
||||
park_id TEXT NOT NULL,
|
||||
date TEXT NOT NULL,
|
||||
is_open INTEGER NOT NULL DEFAULT 0,
|
||||
hours_label TEXT,
|
||||
special_type TEXT,
|
||||
scraped_at TEXT NOT NULL,
|
||||
PRIMARY KEY (park_id, date)
|
||||
)
|
||||
`);
|
||||
try {
|
||||
_db.exec(`ALTER TABLE park_days ADD COLUMN special_type TEXT`);
|
||||
} catch {
|
||||
// Column already exists
|
||||
}
|
||||
return _db;
|
||||
}
|
||||
|
||||
export function closeDb(): void {
|
||||
if (_db) {
|
||||
_db.close();
|
||||
_db = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
import type Database from "better-sqlite3";
|
||||
import { getDb } from "./index";
|
||||
|
||||
import type { DayData } from "../../../lib/types";
|
||||
export type { DayData };
|
||||
|
||||
interface DayRow {
|
||||
park_id: string;
|
||||
date: string;
|
||||
is_open: number;
|
||||
hours_label: string | null;
|
||||
special_type: string | null;
|
||||
}
|
||||
|
||||
function rowToDayData(row: DayRow): DayData {
|
||||
return {
|
||||
isOpen: row.is_open === 1,
|
||||
hoursLabel: row.hours_label,
|
||||
specialType: row.special_type,
|
||||
};
|
||||
}
|
||||
|
||||
export function upsertDay(
|
||||
parkId: string,
|
||||
date: string,
|
||||
isOpen: boolean,
|
||||
hoursLabel?: string,
|
||||
specialType?: string,
|
||||
): void {
|
||||
getDb()
|
||||
.prepare(
|
||||
`INSERT INTO park_days (park_id, date, is_open, hours_label, special_type, scraped_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (park_id, date) DO UPDATE SET
|
||||
is_open = excluded.is_open,
|
||||
hours_label = excluded.hours_label,
|
||||
special_type = excluded.special_type,
|
||||
scraped_at = excluded.scraped_at
|
||||
WHERE park_days.date >= date('now')`,
|
||||
)
|
||||
.run(parkId, date, isOpen ? 1 : 0, hoursLabel ?? null, specialType ?? null, new Date().toISOString());
|
||||
}
|
||||
|
||||
export function getDateRange(
|
||||
startDate: string,
|
||||
endDate: string,
|
||||
): Record<string, Record<string, DayData>> {
|
||||
const rows = getDb()
|
||||
.prepare(
|
||||
`SELECT park_id, date, is_open, hours_label, special_type
|
||||
FROM park_days
|
||||
WHERE date >= ? AND date <= ?`,
|
||||
)
|
||||
.all(startDate, endDate) as DayRow[];
|
||||
|
||||
const result: Record<string, Record<string, DayData>> = {};
|
||||
for (const row of rows) {
|
||||
if (!result[row.park_id]) result[row.park_id] = {};
|
||||
result[row.park_id][row.date] = rowToDayData(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getParkMonthData(
|
||||
parkId: string,
|
||||
year: number,
|
||||
month: number,
|
||||
): Record<string, DayData> {
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
const rows = getDb()
|
||||
.prepare(
|
||||
`SELECT park_id, date, is_open, hours_label, special_type
|
||||
FROM park_days
|
||||
WHERE park_id = ? AND date LIKE ? || '-%'
|
||||
ORDER BY date`,
|
||||
)
|
||||
.all(parkId, prefix) as DayRow[];
|
||||
|
||||
const result: Record<string, DayData> = {};
|
||||
for (const row of rows) {
|
||||
result[row.date] = rowToDayData(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getMonthCalendar(
|
||||
year: number,
|
||||
month: number,
|
||||
): Record<string, boolean[]> {
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
const rows = getDb()
|
||||
.prepare(
|
||||
`SELECT park_id, date, is_open
|
||||
FROM park_days
|
||||
WHERE date LIKE ? || '-%'
|
||||
ORDER BY date`,
|
||||
)
|
||||
.all(prefix) as { park_id: string; date: string; is_open: number }[];
|
||||
|
||||
const result: Record<string, boolean[]> = {};
|
||||
for (const row of rows) {
|
||||
if (!result[row.park_id]) result[row.park_id] = [];
|
||||
const day = parseInt(row.date.slice(8), 10);
|
||||
result[row.park_id][day - 1] = row.is_open === 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getDayData(parkId: string, date: string): DayData | null {
|
||||
const row = getDb()
|
||||
.prepare(
|
||||
`SELECT park_id, date, is_open, hours_label, special_type
|
||||
FROM park_days
|
||||
WHERE park_id = ? AND date = ?`,
|
||||
)
|
||||
.get(parkId, date) as DayRow | undefined;
|
||||
|
||||
return row ? rowToDayData(row) : null;
|
||||
}
|
||||
|
||||
export function isMonthScraped(
|
||||
parkId: string,
|
||||
year: number,
|
||||
month: number,
|
||||
staleAfterMs: number,
|
||||
): boolean {
|
||||
const daysInMonth = new Date(year, month, 0).getDate();
|
||||
const lastDay = `${year}-${String(month).padStart(2, "0")}-${String(daysInMonth).padStart(2, "0")}`;
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
|
||||
if (lastDay < today) return true;
|
||||
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
const row = getDb()
|
||||
.prepare(
|
||||
`SELECT MAX(scraped_at) AS last_scraped
|
||||
FROM park_days
|
||||
WHERE park_id = ? AND date LIKE ? || '-%'`,
|
||||
)
|
||||
.get(parkId, prefix) as { last_scraped: string | null };
|
||||
|
||||
if (!row.last_scraped) return false;
|
||||
return Date.now() - new Date(row.last_scraped).getTime() < staleAfterMs;
|
||||
}
|
||||
|
||||
export function getLastScrapeTime(): string | null {
|
||||
const row = getDb()
|
||||
.prepare(`SELECT MAX(scraped_at) AS last_scraped FROM park_days`)
|
||||
.get() as { last_scraped: string | null };
|
||||
return row.last_scraped;
|
||||
}
|
||||
|
||||
export function getParkDayCount(): number {
|
||||
const row = getDb()
|
||||
.prepare(`SELECT COUNT(*) AS count FROM park_days`)
|
||||
.get() as { count: number };
|
||||
return row.count;
|
||||
}
|
||||
|
||||
export function transact(fn: () => void): void {
|
||||
getDb().transaction(fn)();
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Hono } from "hono";
|
||||
import { serve } from "@hono/node-server";
|
||||
import { cors } from "hono/cors";
|
||||
import { logger } from "hono/logger";
|
||||
|
||||
import { getDb } from "./db/index";
|
||||
import { startScheduler } from "./services/scheduler";
|
||||
|
||||
import calendarRoutes from "./routes/calendar";
|
||||
import parksRoutes from "./routes/parks";
|
||||
import ridesRoutes from "./routes/rides";
|
||||
import statusRoutes from "./routes/status";
|
||||
import scrapeRoutes from "./routes/scrape";
|
||||
|
||||
const PORT = parseInt(process.env.PORT ?? "3001", 10);
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.use("*", logger());
|
||||
app.use("*", cors());
|
||||
|
||||
app.route("/api/calendar", calendarRoutes);
|
||||
app.route("/api/parks", parksRoutes);
|
||||
app.route("/api/parks", ridesRoutes);
|
||||
app.route("/api/status", statusRoutes);
|
||||
app.route("/api/scrape", scrapeRoutes);
|
||||
|
||||
// Initialize database on startup
|
||||
getDb();
|
||||
console.log("[backend] database initialized");
|
||||
|
||||
// Start cron scheduler
|
||||
startScheduler();
|
||||
|
||||
// Start HTTP server
|
||||
serve({ fetch: app.fetch, port: PORT }, (info) => {
|
||||
console.log(`[backend] listening on http://localhost:${info.port}`);
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
import { Hono } from "hono";
|
||||
import { PARKS } from "../../../lib/parks";
|
||||
import { QUEUE_TIMES_IDS } from "../../../lib/queue-times-map";
|
||||
import { getCoasterSet } from "../../../lib/coaster-data";
|
||||
import { getTodayLocal, isWithinOperatingWindow, getOperatingStatus } from "../../../lib/env";
|
||||
import { fetchToday } from "../../../lib/scrapers/sixflags";
|
||||
import { fetchLiveRides } from "../../../lib/scrapers/queuetimes";
|
||||
import { getDateRange, getParkMonthData, type DayData } from "../db/queries";
|
||||
import { TtlCache } from "../services/cache";
|
||||
|
||||
const todayCache = new TtlCache<{ date: string; isOpen: boolean; hoursLabel?: string; specialType?: string } | null>(5 * 60 * 1000);
|
||||
const ridesCache = new TtlCache<{ openRides: number; openCoasters: number } | null>(5 * 60 * 1000);
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.get("/week", async (c) => {
|
||||
const startParam = c.req.query("start");
|
||||
if (!startParam || !/^\d{4}-\d{2}-\d{2}$/.test(startParam)) {
|
||||
return c.json({ error: "Missing or invalid ?start=YYYY-MM-DD" }, 400);
|
||||
}
|
||||
|
||||
const weekDates = Array.from({ length: 7 }, (_, i) => {
|
||||
const d = new Date(startParam + "T00:00:00");
|
||||
d.setDate(d.getDate() + i);
|
||||
return d.toISOString().slice(0, 10);
|
||||
});
|
||||
const endDate = weekDates[6];
|
||||
const today = getTodayLocal();
|
||||
|
||||
const data = getDateRange(startParam, endDate);
|
||||
|
||||
// Merge live today data
|
||||
if (weekDates.includes(today)) {
|
||||
await Promise.all(
|
||||
PARKS.map(async (p) => {
|
||||
let live = todayCache.get(p.id);
|
||||
if (live === null && !todayCache.get(p.id + "_checked")) {
|
||||
live = await fetchToday(p.apiId).catch(() => null);
|
||||
todayCache.set(p.id, live);
|
||||
todayCache.set(p.id + "_checked", true as any);
|
||||
}
|
||||
if (!live) return;
|
||||
if (!data[p.id]) data[p.id] = {};
|
||||
data[p.id][today] = {
|
||||
isOpen: live.isOpen,
|
||||
hoursLabel: live.hoursLabel ?? null,
|
||||
specialType: live.specialType ?? null,
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const currentWeekStart = (() => {
|
||||
const d = new Date(today + "T00:00:00");
|
||||
d.setDate(d.getDate() - d.getDay());
|
||||
return d.toISOString().slice(0, 10);
|
||||
})();
|
||||
const isCurrentWeek = startParam === currentWeekStart;
|
||||
|
||||
// Live status for today
|
||||
let rideCounts: Record<string, number> = {};
|
||||
let coasterCounts: Record<string, number> = {};
|
||||
let openParkIds: string[] = [];
|
||||
let closingParkIds: string[] = [];
|
||||
let weatherDelayParkIds: string[] = [];
|
||||
|
||||
if (weekDates.includes(today)) {
|
||||
const openTodayParks = PARKS.filter((p) => {
|
||||
const dayData = data[p.id]?.[today];
|
||||
if (!dayData?.isOpen || !dayData.hoursLabel) return false;
|
||||
return isWithinOperatingWindow(dayData.hoursLabel, p.timezone);
|
||||
});
|
||||
openParkIds = openTodayParks.map((p) => p.id);
|
||||
closingParkIds = openTodayParks
|
||||
.filter((p) => {
|
||||
const dayData = data[p.id]?.[today];
|
||||
return dayData?.hoursLabel ? getOperatingStatus(dayData.hoursLabel, p.timezone) === "closing" : false;
|
||||
})
|
||||
.map((p) => p.id);
|
||||
|
||||
const trackedParks = openTodayParks.filter((p) => QUEUE_TIMES_IDS[p.id]);
|
||||
const results = await Promise.all(
|
||||
trackedParks.map(async (p) => {
|
||||
let cached = ridesCache.get(p.id);
|
||||
if (cached === null) {
|
||||
const coasterSet = getCoasterSet(p.id);
|
||||
const result = await fetchLiveRides(QUEUE_TIMES_IDS[p.id], coasterSet).catch(() => null);
|
||||
cached = result
|
||||
? {
|
||||
openRides: result.rides.filter((r) => r.isOpen).length,
|
||||
openCoasters: result.rides.filter((r) => r.isOpen && r.isCoaster).length,
|
||||
}
|
||||
: null;
|
||||
ridesCache.set(p.id, cached);
|
||||
}
|
||||
return { id: p.id, ...(cached ?? { openRides: 0, openCoasters: 0 }) };
|
||||
}),
|
||||
);
|
||||
|
||||
weatherDelayParkIds = results.filter(({ openRides }) => openRides === 0).map(({ id }) => id);
|
||||
rideCounts = Object.fromEntries(results.filter(({ openRides }) => openRides > 0).map(({ id, openRides }) => [id, openRides]));
|
||||
coasterCounts = Object.fromEntries(results.filter(({ openCoasters }) => openCoasters > 0).map(({ id, openCoasters }) => [id, openCoasters]));
|
||||
}
|
||||
|
||||
const scrapedCount = Object.values(data).reduce((sum, parkData) => sum + Object.keys(parkData).length, 0);
|
||||
|
||||
c.header("Cache-Control", "public, max-age=120, stale-while-revalidate=300");
|
||||
return c.json({
|
||||
weekStart: startParam,
|
||||
weekDates,
|
||||
today,
|
||||
isCurrentWeek,
|
||||
data,
|
||||
rideCounts,
|
||||
coasterCounts,
|
||||
openParkIds,
|
||||
closingParkIds,
|
||||
weatherDelayParkIds,
|
||||
hasCoasterData: true,
|
||||
scrapedCount,
|
||||
});
|
||||
});
|
||||
|
||||
app.get("/:parkId/month", async (c) => {
|
||||
const parkId = c.req.param("parkId");
|
||||
const monthParam = c.req.query("month");
|
||||
|
||||
if (!monthParam || !/^\d{4}-\d{2}$/.test(monthParam)) {
|
||||
return c.json({ error: "Missing or invalid ?month=YYYY-MM" }, 400);
|
||||
}
|
||||
|
||||
const [yearStr, monthStr] = monthParam.split("-");
|
||||
const year = parseInt(yearStr);
|
||||
const month = parseInt(monthStr);
|
||||
|
||||
if (month < 1 || month > 12) {
|
||||
return c.json({ error: "Month must be 1-12" }, 400);
|
||||
}
|
||||
|
||||
const monthData = getParkMonthData(parkId, year, month);
|
||||
const today = getTodayLocal();
|
||||
|
||||
// Merge live today if viewing current month
|
||||
const park = PARKS.find((p) => p.id === parkId);
|
||||
if (park) {
|
||||
const liveToday = await fetchToday(park.apiId).catch(() => null);
|
||||
if (liveToday) {
|
||||
monthData[today] = {
|
||||
isOpen: liveToday.isOpen,
|
||||
hoursLabel: liveToday.hoursLabel ?? null,
|
||||
specialType: liveToday.specialType ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
c.header("Cache-Control", "public, max-age=300, stale-while-revalidate=600");
|
||||
return c.json({ parkId, year, month, monthData, today });
|
||||
});
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,19 @@
|
||||
import { Hono } from "hono";
|
||||
import { PARKS, PARK_MAP } from "../../../lib/parks";
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.get("/", (c) => {
|
||||
c.header("Cache-Control", "public, max-age=3600");
|
||||
return c.json({ parks: PARKS });
|
||||
});
|
||||
|
||||
app.get("/:id", (c) => {
|
||||
const park = PARK_MAP.get(c.req.param("id"));
|
||||
if (!park) return c.json({ error: "Park not found" }, 404);
|
||||
|
||||
c.header("Cache-Control", "public, max-age=3600");
|
||||
return c.json(park);
|
||||
});
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,66 @@
|
||||
import { Hono } from "hono";
|
||||
import { PARK_MAP } from "../../../lib/parks";
|
||||
import { QUEUE_TIMES_IDS } from "../../../lib/queue-times-map";
|
||||
import { getCoasterSet } from "../../../lib/coaster-data";
|
||||
import { getTodayLocal, isWithinOperatingWindow } from "../../../lib/env";
|
||||
import { fetchLiveRides } from "../../../lib/scrapers/queuetimes";
|
||||
import { scrapeRidesForDay } from "../../../lib/scrapers/sixflags";
|
||||
import { getDayData } from "../db/queries";
|
||||
import { TtlCache } from "../services/cache";
|
||||
import type { LiveRidesResult } from "../../../lib/scrapers/queuetimes";
|
||||
|
||||
const liveRidesCache = new TtlCache<LiveRidesResult | null>(5 * 60 * 1000);
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.get("/:id/rides", async (c) => {
|
||||
const id = c.req.param("id");
|
||||
const park = PARK_MAP.get(id);
|
||||
if (!park) return c.json({ error: "Park not found" }, 404);
|
||||
|
||||
const today = getTodayLocal();
|
||||
const todayData = getDayData(id, today);
|
||||
const withinWindow = todayData?.hoursLabel
|
||||
? isWithinOperatingWindow(todayData.hoursLabel, park.timezone)
|
||||
: false;
|
||||
|
||||
const queueTimesId = QUEUE_TIMES_IDS[id];
|
||||
let liveRides: LiveRidesResult | null = null;
|
||||
|
||||
if (queueTimesId) {
|
||||
liveRides = liveRidesCache.get(id);
|
||||
if (liveRides === null) {
|
||||
const coasterSet = getCoasterSet(id);
|
||||
liveRides = await fetchLiveRides(queueTimesId, coasterSet).catch(() => null);
|
||||
if (liveRides) liveRidesCache.set(id, liveRides);
|
||||
}
|
||||
|
||||
if (liveRides && !withinWindow) {
|
||||
liveRides = {
|
||||
...liveRides,
|
||||
rides: liveRides.rides.map((r) => ({ ...r, isOpen: false, waitMinutes: 0 })),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const isWeatherDelay =
|
||||
withinWindow && liveRides !== null && liveRides.rides.length > 0 && liveRides.rides.every((r) => !r.isOpen);
|
||||
|
||||
let scheduleFallback = null;
|
||||
if (!liveRides) {
|
||||
scheduleFallback = await scrapeRidesForDay(park.apiId, today).catch(() => null);
|
||||
}
|
||||
|
||||
c.header("Cache-Control", "public, max-age=60, stale-while-revalidate=120");
|
||||
return c.json({
|
||||
parkId: id,
|
||||
today,
|
||||
parkOpenToday: !!(todayData?.isOpen && todayData?.hoursLabel),
|
||||
withinWindow,
|
||||
isWeatherDelay,
|
||||
liveRides,
|
||||
scheduleFallback,
|
||||
});
|
||||
});
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,33 @@
|
||||
import { Hono } from "hono";
|
||||
import { scrapeToday, scrapeCurrentMonth, scrapeUpcomingMonths, scrapeFullYear } from "../services/scraper";
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.post("/trigger", async (c) => {
|
||||
const scope = c.req.query("scope") ?? "today";
|
||||
|
||||
let result;
|
||||
switch (scope) {
|
||||
case "today":
|
||||
result = await scrapeToday();
|
||||
break;
|
||||
case "month":
|
||||
result = await scrapeCurrentMonth();
|
||||
break;
|
||||
case "upcoming":
|
||||
result = await scrapeUpcomingMonths();
|
||||
break;
|
||||
case "full":
|
||||
result = await scrapeFullYear();
|
||||
break;
|
||||
case "force":
|
||||
result = await scrapeFullYear(true);
|
||||
break;
|
||||
default:
|
||||
return c.json({ error: "Invalid scope. Use: today, month, upcoming, full, force" }, 400);
|
||||
}
|
||||
|
||||
return c.json(result);
|
||||
});
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,21 @@
|
||||
import { Hono } from "hono";
|
||||
import { PARKS } from "../../../lib/parks";
|
||||
import { getLastScrapeTime, getParkDayCount } from "../db/queries";
|
||||
import { getLastScrapeResult } from "../services/scraper";
|
||||
|
||||
const app = new Hono();
|
||||
|
||||
app.get("/", (c) => {
|
||||
return c.json({
|
||||
status: "ok",
|
||||
uptime: Math.floor(process.uptime()),
|
||||
parks: PARKS.length,
|
||||
database: {
|
||||
totalDays: getParkDayCount(),
|
||||
lastScrape: getLastScrapeTime(),
|
||||
},
|
||||
lastScrapeResult: getLastScrapeResult(),
|
||||
});
|
||||
});
|
||||
|
||||
export default app;
|
||||
@@ -0,0 +1,35 @@
|
||||
interface CacheEntry<T> {
|
||||
data: T;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
export class TtlCache<T> {
|
||||
private store = new Map<string, CacheEntry<T>>();
|
||||
|
||||
constructor(private defaultTtlMs: number) {}
|
||||
|
||||
get(key: string): T | null {
|
||||
const entry = this.store.get(key);
|
||||
if (!entry) return null;
|
||||
if (Date.now() > entry.expiresAt) {
|
||||
this.store.delete(key);
|
||||
return null;
|
||||
}
|
||||
return entry.data;
|
||||
}
|
||||
|
||||
set(key: string, data: T, ttlMs?: number): void {
|
||||
this.store.set(key, {
|
||||
data,
|
||||
expiresAt: Date.now() + (ttlMs ?? this.defaultTtlMs),
|
||||
});
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.store.clear();
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.store.size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import cron from "node-cron";
|
||||
import { scrapeToday, scrapeCurrentMonth, scrapeUpcomingMonths, scrapeFullYear } from "./scraper";
|
||||
|
||||
let initialized = false;
|
||||
|
||||
export function startScheduler(): void {
|
||||
if (initialized) return;
|
||||
initialized = true;
|
||||
|
||||
// Tier 1: Today — every hour during operating season (Mar-Dec)
|
||||
cron.schedule("0 * * 3-12 *", async () => {
|
||||
console.log(`[scheduler] tier-1: scraping today @ ${new Date().toISOString()}`);
|
||||
await scrapeToday().catch((err) => console.error("[scheduler] tier-1 error:", err));
|
||||
});
|
||||
|
||||
// Tier 2: This week — every 6 hours, current month for all parks
|
||||
cron.schedule("0 */6 * * *", async () => {
|
||||
console.log(`[scheduler] tier-2: scraping current month @ ${new Date().toISOString()}`);
|
||||
await scrapeCurrentMonth().catch((err) => console.error("[scheduler] tier-2 error:", err));
|
||||
});
|
||||
|
||||
// Tier 3: Upcoming — twice daily (3 AM, 3 PM), current + next month
|
||||
cron.schedule("0 3,15 * * *", async () => {
|
||||
console.log(`[scheduler] tier-3: scraping upcoming months @ ${new Date().toISOString()}`);
|
||||
await scrapeUpcomingMonths().catch((err) => console.error("[scheduler] tier-3 error:", err));
|
||||
});
|
||||
|
||||
// Tier 4: Full season — once daily at 3 AM
|
||||
cron.schedule("0 3 * * *", async () => {
|
||||
console.log(`[scheduler] tier-4: scraping full year @ ${new Date().toISOString()}`);
|
||||
await scrapeFullYear().catch((err) => console.error("[scheduler] tier-4 error:", err));
|
||||
});
|
||||
|
||||
console.log("[scheduler] cron jobs registered");
|
||||
console.log(" tier-1: today — hourly (Mar-Dec)");
|
||||
console.log(" tier-2: current month — every 6h");
|
||||
console.log(" tier-3: upcoming — 3 AM + 3 PM");
|
||||
console.log(" tier-4: full year — 3 AM daily");
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { PARKS } from "../../../lib/parks";
|
||||
import { scrapeMonth, fetchToday, RateLimitError } from "../../../lib/scrapers/sixflags";
|
||||
import { upsertDay, isMonthScraped, getDayData, transact } from "../db/queries";
|
||||
import { parseStalenessHours } from "../../../lib/env";
|
||||
|
||||
const DELAY_MS = 1000;
|
||||
const STALE_AFTER_MS = parseStalenessHours(process.env.PARK_HOURS_STALENESS_HOURS, 72) * 60 * 60 * 1000;
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise<void>((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
export interface ScrapeResult {
|
||||
scope: string;
|
||||
fetched: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
updated: number;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
}
|
||||
|
||||
let lastScrapeResult: ScrapeResult | null = null;
|
||||
|
||||
export function getLastScrapeResult(): ScrapeResult | null {
|
||||
return lastScrapeResult;
|
||||
}
|
||||
|
||||
export async function scrapeToday(): Promise<ScrapeResult> {
|
||||
const startedAt = new Date().toISOString();
|
||||
let fetched = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let updated = 0;
|
||||
|
||||
for (const park of PARKS) {
|
||||
try {
|
||||
const live = await fetchToday(park.apiId);
|
||||
if (!live) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
fetched++;
|
||||
|
||||
const existing = getDayData(park.id, live.date);
|
||||
if (
|
||||
existing &&
|
||||
existing.isOpen === live.isOpen &&
|
||||
existing.hoursLabel === (live.hoursLabel ?? null) &&
|
||||
existing.specialType === (live.specialType ?? null)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
upsertDay(park.id, live.date, live.isOpen, live.hoursLabel, live.specialType);
|
||||
updated++;
|
||||
console.log(`[today] ${park.shortName}: updated (${live.isOpen ? "open" : "closed"}${live.hoursLabel ? " " + live.hoursLabel : ""})`);
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
|
||||
const result: ScrapeResult = {
|
||||
scope: "today",
|
||||
fetched,
|
||||
skipped,
|
||||
errors,
|
||||
updated,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
};
|
||||
lastScrapeResult = result;
|
||||
console.log(`[today] done: ${fetched} fetched, ${updated} updated, ${skipped} skipped, ${errors} errors`);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function scrapeMonths(monthList: { year: number; month: number }[], force = false): Promise<ScrapeResult> {
|
||||
const startedAt = new Date().toISOString();
|
||||
let fetched = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
for (const park of PARKS) {
|
||||
for (const { year, month } of monthList) {
|
||||
if (!force && isMonthScraped(park.id, year, month, STALE_AFTER_MS)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const days = await scrapeMonth(park.apiId, year, month);
|
||||
transact(() => {
|
||||
for (const d of days) {
|
||||
upsertDay(park.id, d.date, d.isOpen, d.hoursLabel, d.specialType);
|
||||
}
|
||||
});
|
||||
fetched++;
|
||||
console.log(`[month] ${park.shortName} ${year}-${String(month).padStart(2, "0")}: ${days.filter((d) => d.isOpen).length} open days`);
|
||||
} catch (err) {
|
||||
if (err instanceof RateLimitError) {
|
||||
console.log(`[month] ${park.shortName}: rate limited`);
|
||||
} else {
|
||||
console.error(`[month] ${park.shortName}: error — ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
errors++;
|
||||
}
|
||||
await sleep(DELAY_MS);
|
||||
}
|
||||
}
|
||||
|
||||
const result: ScrapeResult = {
|
||||
scope: `months(${monthList.map((m) => `${m.year}-${String(m.month).padStart(2, "0")}`).join(",")})`,
|
||||
fetched,
|
||||
skipped,
|
||||
errors,
|
||||
updated: fetched,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
};
|
||||
lastScrapeResult = result;
|
||||
console.log(`[month] done: ${fetched} fetched, ${skipped} skipped, ${errors} errors`);
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function scrapeCurrentMonth(): Promise<ScrapeResult> {
|
||||
const now = new Date();
|
||||
return scrapeMonths([{ year: now.getFullYear(), month: now.getMonth() + 1 }]);
|
||||
}
|
||||
|
||||
export async function scrapeUpcomingMonths(): Promise<ScrapeResult> {
|
||||
const now = new Date();
|
||||
const current = { year: now.getFullYear(), month: now.getMonth() + 1 };
|
||||
const next = new Date(now.getFullYear(), now.getMonth() + 1, 1);
|
||||
const nextMonth = { year: next.getFullYear(), month: next.getMonth() + 1 };
|
||||
return scrapeMonths([current, nextMonth]);
|
||||
}
|
||||
|
||||
export async function scrapeFullYear(force = false): Promise<ScrapeResult> {
|
||||
const year = new Date().getFullYear();
|
||||
const months = Array.from({ length: 12 }, (_, i) => ({ year, month: i + 1 }));
|
||||
return scrapeMonths(months, force);
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "CommonJS",
|
||||
"moduleResolution": "node",
|
||||
"esModuleInterop": true,
|
||||
"strict": true,
|
||||
"outDir": "dist",
|
||||
"rootDir": "..",
|
||||
"baseUrl": "..",
|
||||
"paths": {
|
||||
"@lib/*": ["lib/*"]
|
||||
},
|
||||
"skipLibCheck": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": false,
|
||||
"sourceMap": true
|
||||
},
|
||||
"include": ["src/**/*.ts", "../lib/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import { WeekNav } from "./WeekNav";
|
||||
import { Legend } from "./Legend";
|
||||
import { EmptyState } from "./EmptyState";
|
||||
import { PARKS, groupByRegion } from "@/lib/parks";
|
||||
import type { DayData } from "@/lib/db";
|
||||
import type { DayData } from "@/lib/types";
|
||||
|
||||
const REFRESH_INTERVAL_MS = 2 * 60 * 1000; // 2 minutes
|
||||
const OPEN_REFRESH_BUFFER_MS = 30_000; // 30s after opening time before hitting the API
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Park } from "@/lib/scrapers/types";
|
||||
import type { DayData } from "@/lib/db";
|
||||
import type { DayData } from "@/lib/types";
|
||||
import type { Region } from "@/lib/parks";
|
||||
import { ParkCard } from "./ParkCard";
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Link from "next/link";
|
||||
import type { Park } from "@/lib/scrapers/types";
|
||||
import type { DayData } from "@/lib/db";
|
||||
import type { DayData } from "@/lib/types";
|
||||
import { getTimezoneAbbr } from "@/lib/env";
|
||||
|
||||
interface ParkCardProps {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import Link from "next/link";
|
||||
import type { DayData } from "@/lib/db";
|
||||
import type { DayData } from "@/lib/types";
|
||||
import { getTimezoneAbbr } from "@/lib/env";
|
||||
|
||||
interface ParkMonthCalendarProps {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Fragment } from "react";
|
||||
import Link from "next/link";
|
||||
import type { Park } from "@/lib/scrapers/types";
|
||||
import type { DayData } from "@/lib/db";
|
||||
import type { DayData } from "@/lib/types";
|
||||
import type { Region } from "@/lib/parks";
|
||||
import { getTodayLocal, getTimezoneAbbr } from "@/lib/env";
|
||||
|
||||
|
||||
@@ -1,416 +0,0 @@
|
||||
{
|
||||
"greatadventure": {
|
||||
"rcdb_id": 4534,
|
||||
"coasters": [
|
||||
"Superman - Ultimate Flight",
|
||||
"El Toro",
|
||||
"Dark Knight",
|
||||
"Joker",
|
||||
"Jersey Devil Coaster",
|
||||
"Lil' Devil Coaster",
|
||||
"Flash: Vertical Velocity",
|
||||
"Batman The Ride",
|
||||
"Skull Mountain",
|
||||
"Runaway Mine Train",
|
||||
"Medusa",
|
||||
"Harley Quinn Crazy Train",
|
||||
"Nitro"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:40:09.731Z"
|
||||
},
|
||||
"magicmountain": {
|
||||
"rcdb_id": 4532,
|
||||
"coasters": [
|
||||
"Ninja",
|
||||
"New Revolution",
|
||||
"Batman The Ride",
|
||||
"Viper",
|
||||
"Gold Rusher",
|
||||
"Riddler's Revenge",
|
||||
"Canyon Blaster",
|
||||
"Goliath",
|
||||
"X2",
|
||||
"Scream!",
|
||||
"Tatsu",
|
||||
"Apocalypse the Ride",
|
||||
"Road Runner Express",
|
||||
"Speedy Gonzales Hot Rod Racers",
|
||||
"Full Throttle",
|
||||
"Twisted Colossus",
|
||||
"West Coast Racers",
|
||||
"Wonder Woman Flight of Courage"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:43.666Z"
|
||||
},
|
||||
"greatamerica": {
|
||||
"rcdb_id": 4530,
|
||||
"coasters": [
|
||||
"Demon",
|
||||
"Batman The Ride",
|
||||
"American Eagle",
|
||||
"Viper",
|
||||
"Whizzer",
|
||||
"Sprocket Rockets",
|
||||
"Raging Bull",
|
||||
"Flash: Vertical Velocity",
|
||||
"Superman - Ultimate Flight",
|
||||
"Dark Knight",
|
||||
"Little Dipper",
|
||||
"Goliath",
|
||||
"X-Flight",
|
||||
"Joker",
|
||||
"Maxx Force",
|
||||
"Wrath of Rakshasa"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:29:24.092Z"
|
||||
},
|
||||
"overgeorgia": {
|
||||
"rcdb_id": 4535,
|
||||
"coasters": [
|
||||
"Blue Hawk",
|
||||
"Great American Scream Machine",
|
||||
"Dahlonega Mine Train",
|
||||
"Batman The Ride",
|
||||
"Georgia Scorcher",
|
||||
"Superman - Ultimate Flight",
|
||||
"Joker Funhouse Coaster",
|
||||
"Goliath",
|
||||
"Dare Devil Dive",
|
||||
"Twisted Cyclone",
|
||||
"Riddler Mindbender",
|
||||
"Georgia Gold Rusher"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:29:26.121Z"
|
||||
},
|
||||
"overtexas": {
|
||||
"rcdb_id": 4531,
|
||||
"coasters": [
|
||||
"Pandemonium",
|
||||
"New Texas Giant",
|
||||
"Joker",
|
||||
"Aquaman: Power Wave",
|
||||
"Shock Wave",
|
||||
"Judge Roy Scream",
|
||||
"Runaway Mine Train",
|
||||
"Runaway Mountain",
|
||||
"Mini Mine Train",
|
||||
"Mr. Freeze",
|
||||
"Batman The Ride",
|
||||
"Titan",
|
||||
"Wile E. Coyote's Grand Canyon Blaster"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:45.715Z"
|
||||
},
|
||||
"stlouis": {
|
||||
"rcdb_id": 4536,
|
||||
"coasters": [
|
||||
"Ninja",
|
||||
"River King Mine Train",
|
||||
"Mr. Freeze Reverse Blast",
|
||||
"Batman The Ride",
|
||||
"Screamin' Eagle",
|
||||
"Boss",
|
||||
"Pandemonium",
|
||||
"American Thunder",
|
||||
"Boomerang",
|
||||
"Rookie Racer"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:47.770Z"
|
||||
},
|
||||
"fiestatexas": {
|
||||
"rcdb_id": 4538,
|
||||
"coasters": [
|
||||
"Batgirl Coaster Chase",
|
||||
"Road Runner Express",
|
||||
"Poltergeist",
|
||||
"Boomerang Coast to Coaster",
|
||||
"Superman Krypton Coaster",
|
||||
"Pandemonium",
|
||||
"Chupacabra",
|
||||
"Iron Rattler",
|
||||
"Batman The Ride",
|
||||
"Wonder Woman Golden Lasso Coaster",
|
||||
"Dr. Diabolical's Cliffhanger"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:49.819Z"
|
||||
},
|
||||
"newengland": {
|
||||
"rcdb_id": 4565,
|
||||
"coasters": [
|
||||
"Joker",
|
||||
"Thunderbolt",
|
||||
"Great Chase",
|
||||
"Riddler Revenge",
|
||||
"Superman the Ride",
|
||||
"Flashback",
|
||||
"Catwoman's Whip",
|
||||
"Pandemonium",
|
||||
"Batman - The Dark Knight",
|
||||
"Wicked Cyclone",
|
||||
"Gotham City Gauntlet Escape from Arkham Asylum"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:51.866Z"
|
||||
},
|
||||
"discoverykingdom": {
|
||||
"rcdb_id": 4711,
|
||||
"coasters": [
|
||||
"Roadrunner Express",
|
||||
"Medusa",
|
||||
"Cobra",
|
||||
"Flash: Vertical Velocity",
|
||||
"Kong",
|
||||
"Boomerang",
|
||||
"Superman Ultimate Flight",
|
||||
"Joker",
|
||||
"Batman The Ride",
|
||||
"Sidewinder Safari"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:53.909Z"
|
||||
},
|
||||
"mexico": {
|
||||
"rcdb_id": 4629,
|
||||
"coasters": [
|
||||
"Tsunami",
|
||||
"Superman Krypton Coaster",
|
||||
"Batgirl Batarang",
|
||||
"Batman The Ride",
|
||||
"Superman el Último Escape",
|
||||
"Dark Knight",
|
||||
"Joker",
|
||||
"Medusa Steel Coaster",
|
||||
"Wonder Woman",
|
||||
"Speedway Stunt Coaster"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:55.963Z"
|
||||
},
|
||||
"greatescape": {
|
||||
"rcdb_id": 4596,
|
||||
"coasters": [
|
||||
"Comet",
|
||||
"Steamin' Demon",
|
||||
"Flashback",
|
||||
"Canyon Blaster",
|
||||
"Frankie's Mine Train",
|
||||
"Bobcat"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:45:58.013Z"
|
||||
},
|
||||
"darienlake": {
|
||||
"rcdb_id": 4581,
|
||||
"coasters": [
|
||||
"Predator",
|
||||
"Viper",
|
||||
"Mind Eraser",
|
||||
"Boomerang",
|
||||
"Ride of Steel",
|
||||
"Hoot N Holler",
|
||||
"Moto Coaster",
|
||||
"Tantrum"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:00.042Z"
|
||||
},
|
||||
"cedarpoint": {
|
||||
"rcdb_id": 4529,
|
||||
"coasters": [
|
||||
"Raptor",
|
||||
"Rougarou",
|
||||
"Magnum XL-200",
|
||||
"Blue Streak",
|
||||
"Corkscrew",
|
||||
"Gemini",
|
||||
"Wilderness Run",
|
||||
"Woodstock Express",
|
||||
"Millennium Force",
|
||||
"Iron Dragon",
|
||||
"Cedar Creek Mine Ride",
|
||||
"Maverick",
|
||||
"GateKeeper",
|
||||
"Valravn",
|
||||
"Steel Vengeance",
|
||||
"Top Thrill 2",
|
||||
"Wild Mouse",
|
||||
"Siren’s Curse"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:02.082Z"
|
||||
},
|
||||
"knotts": {
|
||||
"rcdb_id": 4546,
|
||||
"coasters": [
|
||||
"Jaguar!",
|
||||
"GhostRider",
|
||||
"Xcelerator",
|
||||
"Silver Bullet",
|
||||
"Sierra Sidewinder",
|
||||
"Pony Express",
|
||||
"Coast Rider",
|
||||
"HangTime",
|
||||
"Snoopy’s Tenderpaw Twister Coaster"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:04.120Z"
|
||||
},
|
||||
"canadaswonderland": {
|
||||
"rcdb_id": 4539,
|
||||
"coasters": [
|
||||
"Flight Deck",
|
||||
"Dragon Fyre",
|
||||
"Mighty Canadian Minebuster",
|
||||
"Wilde Beast",
|
||||
"Ghoster Coaster",
|
||||
"Thunder Run",
|
||||
"Bat",
|
||||
"Vortex",
|
||||
"Taxi Jam",
|
||||
"Fly",
|
||||
"Silver Streak",
|
||||
"Backlot Stunt Coaster",
|
||||
"Behemoth",
|
||||
"Leviathan",
|
||||
"Wonder Mountain's Guardian",
|
||||
"Yukon Striker",
|
||||
"Snoopy's Racing Railway",
|
||||
"AlpenFury"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:06.152Z"
|
||||
},
|
||||
"carowinds": {
|
||||
"rcdb_id": 4542,
|
||||
"coasters": [
|
||||
"Carolina Cyclone",
|
||||
"Woodstock Express",
|
||||
"Carolina Goldrusher",
|
||||
"Hurler",
|
||||
"Vortex",
|
||||
"Wilderness Run",
|
||||
"Afterburn",
|
||||
"Flying Cobras",
|
||||
"Thunder Striker",
|
||||
"Fury 325",
|
||||
"Copperhead Strike",
|
||||
"Snoopy’s Racing Railway",
|
||||
"Ricochet",
|
||||
"Kiddy Hawk"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:08.185Z"
|
||||
},
|
||||
"kingsdominion": {
|
||||
"rcdb_id": 4544,
|
||||
"coasters": [
|
||||
"Racer 75",
|
||||
"Woodstock Express",
|
||||
"Grizzly",
|
||||
"Flight of Fear",
|
||||
"Reptilian",
|
||||
"Great Pumpkin Coaster",
|
||||
"Apple Zapple",
|
||||
"Backlot Stunt Coaster",
|
||||
"Dominator",
|
||||
"Pantherian",
|
||||
"Twisted Timbers",
|
||||
"Tumbili",
|
||||
"Rapterra"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:10.223Z"
|
||||
},
|
||||
"kingsisland": {
|
||||
"rcdb_id": 4540,
|
||||
"coasters": [
|
||||
"Flight of Fear",
|
||||
"Beast",
|
||||
"Racer",
|
||||
"Adventure Express",
|
||||
"Woodstock Express",
|
||||
"Bat",
|
||||
"Great Pumpkin Coaster",
|
||||
"Invertigo",
|
||||
"Diamondback",
|
||||
"Banshee",
|
||||
"Orion",
|
||||
"Mystic Timbers",
|
||||
"Snoopy's Soap Box Racers",
|
||||
"Woodstock’s Air Rail",
|
||||
"Queen City Stunt Coaster"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:12.251Z"
|
||||
},
|
||||
"valleyfair": {
|
||||
"rcdb_id": 4552,
|
||||
"coasters": [
|
||||
"High Roller",
|
||||
"Corkscrew",
|
||||
"Excalibur",
|
||||
"Wild Thing",
|
||||
"Mad Mouse",
|
||||
"Steel Venom",
|
||||
"Renegade",
|
||||
"Cosmic Coaster"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:14.298Z"
|
||||
},
|
||||
"worldsoffun": {
|
||||
"rcdb_id": 4533,
|
||||
"coasters": [
|
||||
"Timber Wolf",
|
||||
"Cosmic Coaster",
|
||||
"Mamba",
|
||||
"Spinning Dragons",
|
||||
"Patriot",
|
||||
"Prowler",
|
||||
"Zambezi Zinger",
|
||||
"Boomerang"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:16.328Z"
|
||||
},
|
||||
"miadventure": {
|
||||
"rcdb_id": 4578,
|
||||
"coasters": [
|
||||
"Corkscrew",
|
||||
"Wolverine Wildcat",
|
||||
"Zach's Zoomer",
|
||||
"Shivering Timbers",
|
||||
"Mad Mouse",
|
||||
"Thunderhawk",
|
||||
"Woodstock Express"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:18.370Z"
|
||||
},
|
||||
"dorneypark": {
|
||||
"rcdb_id": 4588,
|
||||
"coasters": [
|
||||
"Thunderhawk",
|
||||
"Steel Force",
|
||||
"Wild Mouse",
|
||||
"Woodstock Express",
|
||||
"Talon",
|
||||
"Hydra the Revenge",
|
||||
"Possessed",
|
||||
"Iron Menace"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:20.413Z"
|
||||
},
|
||||
"cagreatamerica": {
|
||||
"rcdb_id": 4541,
|
||||
"coasters": [
|
||||
"Demon",
|
||||
"Grizzly",
|
||||
"Woodstock Express",
|
||||
"Patriot",
|
||||
"Flight Deck",
|
||||
"Lucy's Crabbie Cabbies",
|
||||
"Psycho Mouse",
|
||||
"Gold Striker",
|
||||
"RailBlazer"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:22.465Z"
|
||||
},
|
||||
"frontiercity": {
|
||||
"rcdb_id": 4559,
|
||||
"coasters": [
|
||||
"Silver Bullet",
|
||||
"Wildcat",
|
||||
"Diamondback",
|
||||
"Steel Lasso",
|
||||
"Frankie's Mine Train"
|
||||
],
|
||||
"coasters_scraped_at": "2026-04-04T17:46:24.519Z"
|
||||
}
|
||||
}
|
||||
+5
-5
@@ -3,21 +3,21 @@ services:
|
||||
image: gitea.thewrightserver.net/josh/sixflagssupercalendar:web
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- park_data:/app/data
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- BACKEND_URL=http://backend:3001
|
||||
restart: unless-stopped
|
||||
|
||||
scraper:
|
||||
image: gitea.thewrightserver.net/josh/sixflagssupercalendar:scraper
|
||||
backend:
|
||||
image: gitea.thewrightserver.net/josh/sixflagssupercalendar:backend
|
||||
ports:
|
||||
- "3001:3001"
|
||||
volumes:
|
||||
- park_data:/app/data
|
||||
environment:
|
||||
- NODE_ENV=production
|
||||
- TZ=America/New_York
|
||||
- PARK_HOURS_STALENESS_HOURS=72
|
||||
- COASTER_STALENESS_HOURS=720
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
import { normalizeForMatch } from "./coaster-match";
|
||||
|
||||
export const COASTER_LISTS: Record<string, string[]> = {
|
||||
greatadventure: [
|
||||
"Superman - Ultimate Flight",
|
||||
"El Toro",
|
||||
"Dark Knight",
|
||||
"Joker",
|
||||
"Jersey Devil Coaster",
|
||||
"Lil' Devil Coaster",
|
||||
"Flash: Vertical Velocity",
|
||||
"Batman The Ride",
|
||||
"Skull Mountain",
|
||||
"Runaway Mine Train",
|
||||
"Medusa",
|
||||
"Harley Quinn Crazy Train",
|
||||
"Nitro",
|
||||
],
|
||||
magicmountain: [
|
||||
"Ninja",
|
||||
"New Revolution",
|
||||
"Batman The Ride",
|
||||
"Viper",
|
||||
"Gold Rusher",
|
||||
"Riddler's Revenge",
|
||||
"Canyon Blaster",
|
||||
"Goliath",
|
||||
"X2",
|
||||
"Scream!",
|
||||
"Tatsu",
|
||||
"Apocalypse the Ride",
|
||||
"Road Runner Express",
|
||||
"Speedy Gonzales Hot Rod Racers",
|
||||
"Full Throttle",
|
||||
"Twisted Colossus",
|
||||
"West Coast Racers",
|
||||
"Wonder Woman Flight of Courage",
|
||||
],
|
||||
greatamerica: [
|
||||
"Demon",
|
||||
"Batman The Ride",
|
||||
"American Eagle",
|
||||
"Viper",
|
||||
"Whizzer",
|
||||
"Sprocket Rockets",
|
||||
"Raging Bull",
|
||||
"Flash: Vertical Velocity",
|
||||
"Superman - Ultimate Flight",
|
||||
"Dark Knight",
|
||||
"Little Dipper",
|
||||
"Goliath",
|
||||
"X-Flight",
|
||||
"Joker",
|
||||
"Maxx Force",
|
||||
"Wrath of Rakshasa",
|
||||
],
|
||||
overgeorgia: [
|
||||
"Blue Hawk",
|
||||
"Great American Scream Machine",
|
||||
"Dahlonega Mine Train",
|
||||
"Batman The Ride",
|
||||
"Georgia Scorcher",
|
||||
"Superman - Ultimate Flight",
|
||||
"Joker Funhouse Coaster",
|
||||
"Goliath",
|
||||
"Dare Devil Dive",
|
||||
"Twisted Cyclone",
|
||||
"Riddler Mindbender",
|
||||
"Georgia Gold Rusher",
|
||||
],
|
||||
overtexas: [
|
||||
"Pandemonium",
|
||||
"New Texas Giant",
|
||||
"Joker",
|
||||
"Aquaman: Power Wave",
|
||||
"Shock Wave",
|
||||
"Judge Roy Scream",
|
||||
"Runaway Mine Train",
|
||||
"Runaway Mountain",
|
||||
"Mini Mine Train",
|
||||
"Mr. Freeze",
|
||||
"Batman The Ride",
|
||||
"Titan",
|
||||
"Wile E. Coyote's Grand Canyon Blaster",
|
||||
],
|
||||
stlouis: [
|
||||
"Ninja",
|
||||
"River King Mine Train",
|
||||
"Mr. Freeze Reverse Blast",
|
||||
"Batman The Ride",
|
||||
"Screamin' Eagle",
|
||||
"Boss",
|
||||
"Pandemonium",
|
||||
"American Thunder",
|
||||
"Boomerang",
|
||||
"Rookie Racer",
|
||||
],
|
||||
fiestatexas: [
|
||||
"Batgirl Coaster Chase",
|
||||
"Road Runner Express",
|
||||
"Poltergeist",
|
||||
"Boomerang Coast to Coaster",
|
||||
"Superman Krypton Coaster",
|
||||
"Pandemonium",
|
||||
"Chupacabra",
|
||||
"Iron Rattler",
|
||||
"Batman The Ride",
|
||||
"Wonder Woman Golden Lasso Coaster",
|
||||
"Dr. Diabolical's Cliffhanger",
|
||||
],
|
||||
newengland: [
|
||||
"Joker",
|
||||
"Thunderbolt",
|
||||
"Great Chase",
|
||||
"Riddler Revenge",
|
||||
"Superman the Ride",
|
||||
"Flashback",
|
||||
"Catwoman's Whip",
|
||||
"Pandemonium",
|
||||
"Batman - The Dark Knight",
|
||||
"Wicked Cyclone",
|
||||
"Gotham City Gauntlet Escape from Arkham Asylum",
|
||||
],
|
||||
discoverykingdom: [
|
||||
"Roadrunner Express",
|
||||
"Medusa",
|
||||
"Cobra",
|
||||
"Flash: Vertical Velocity",
|
||||
"Kong",
|
||||
"Boomerang",
|
||||
"Superman Ultimate Flight",
|
||||
"Joker",
|
||||
"Batman The Ride",
|
||||
"Sidewinder Safari",
|
||||
],
|
||||
mexico: [
|
||||
"Tsunami",
|
||||
"Superman Krypton Coaster",
|
||||
"Batgirl Batarang",
|
||||
"Batman The Ride",
|
||||
"Superman el Último Escape",
|
||||
"Dark Knight",
|
||||
"Joker",
|
||||
"Medusa Steel Coaster",
|
||||
"Wonder Woman",
|
||||
"Speedway Stunt Coaster",
|
||||
],
|
||||
greatescape: [
|
||||
"Comet",
|
||||
"Steamin' Demon",
|
||||
"Flashback",
|
||||
"Canyon Blaster",
|
||||
"Frankie's Mine Train",
|
||||
"Bobcat",
|
||||
],
|
||||
darienlake: [
|
||||
"Predator",
|
||||
"Viper",
|
||||
"Mind Eraser",
|
||||
"Boomerang",
|
||||
"Ride of Steel",
|
||||
"Hoot N Holler",
|
||||
"Moto Coaster",
|
||||
"Tantrum",
|
||||
],
|
||||
cedarpoint: [
|
||||
"Raptor",
|
||||
"Rougarou",
|
||||
"Magnum XL-200",
|
||||
"Blue Streak",
|
||||
"Corkscrew",
|
||||
"Gemini",
|
||||
"Wilderness Run",
|
||||
"Woodstock Express",
|
||||
"Millennium Force",
|
||||
"Iron Dragon",
|
||||
"Cedar Creek Mine Ride",
|
||||
"Maverick",
|
||||
"GateKeeper",
|
||||
"Valravn",
|
||||
"Steel Vengeance",
|
||||
"Top Thrill 2",
|
||||
"Wild Mouse",
|
||||
"Siren's Curse",
|
||||
],
|
||||
knotts: [
|
||||
"Jaguar!",
|
||||
"GhostRider",
|
||||
"Xcelerator",
|
||||
"Silver Bullet",
|
||||
"Sierra Sidewinder",
|
||||
"Pony Express",
|
||||
"Coast Rider",
|
||||
"HangTime",
|
||||
"Snoopy's Tenderpaw Twister Coaster",
|
||||
],
|
||||
canadaswonderland: [
|
||||
"Flight Deck",
|
||||
"Dragon Fyre",
|
||||
"Mighty Canadian Minebuster",
|
||||
"Wilde Beast",
|
||||
"Ghoster Coaster",
|
||||
"Thunder Run",
|
||||
"Bat",
|
||||
"Vortex",
|
||||
"Taxi Jam",
|
||||
"Fly",
|
||||
"Silver Streak",
|
||||
"Backlot Stunt Coaster",
|
||||
"Behemoth",
|
||||
"Leviathan",
|
||||
"Wonder Mountain's Guardian",
|
||||
"Yukon Striker",
|
||||
"Snoopy's Racing Railway",
|
||||
"AlpenFury",
|
||||
],
|
||||
carowinds: [
|
||||
"Carolina Cyclone",
|
||||
"Woodstock Express",
|
||||
"Carolina Goldrusher",
|
||||
"Hurler",
|
||||
"Vortex",
|
||||
"Wilderness Run",
|
||||
"Afterburn",
|
||||
"Flying Cobras",
|
||||
"Thunder Striker",
|
||||
"Fury 325",
|
||||
"Copperhead Strike",
|
||||
"Snoopy's Racing Railway",
|
||||
"Ricochet",
|
||||
"Kiddy Hawk",
|
||||
],
|
||||
kingsdominion: [
|
||||
"Racer 75",
|
||||
"Woodstock Express",
|
||||
"Grizzly",
|
||||
"Flight of Fear",
|
||||
"Reptilian",
|
||||
"Great Pumpkin Coaster",
|
||||
"Apple Zapple",
|
||||
"Backlot Stunt Coaster",
|
||||
"Dominator",
|
||||
"Pantherian",
|
||||
"Twisted Timbers",
|
||||
"Tumbili",
|
||||
"Rapterra",
|
||||
],
|
||||
kingsisland: [
|
||||
"Flight of Fear",
|
||||
"Beast",
|
||||
"Racer",
|
||||
"Adventure Express",
|
||||
"Woodstock Express",
|
||||
"Bat",
|
||||
"Great Pumpkin Coaster",
|
||||
"Invertigo",
|
||||
"Diamondback",
|
||||
"Banshee",
|
||||
"Orion",
|
||||
"Mystic Timbers",
|
||||
"Snoopy's Soap Box Racers",
|
||||
"Woodstock's Air Rail",
|
||||
"Queen City Stunt Coaster",
|
||||
],
|
||||
valleyfair: [
|
||||
"High Roller",
|
||||
"Corkscrew",
|
||||
"Excalibur",
|
||||
"Wild Thing",
|
||||
"Mad Mouse",
|
||||
"Steel Venom",
|
||||
"Renegade",
|
||||
"Cosmic Coaster",
|
||||
],
|
||||
worldsoffun: [
|
||||
"Timber Wolf",
|
||||
"Cosmic Coaster",
|
||||
"Mamba",
|
||||
"Spinning Dragons",
|
||||
"Patriot",
|
||||
"Prowler",
|
||||
"Zambezi Zinger",
|
||||
"Boomerang",
|
||||
],
|
||||
miadventure: [
|
||||
"Corkscrew",
|
||||
"Wolverine Wildcat",
|
||||
"Zach's Zoomer",
|
||||
"Shivering Timbers",
|
||||
"Mad Mouse",
|
||||
"Thunderhawk",
|
||||
"Woodstock Express",
|
||||
],
|
||||
dorneypark: [
|
||||
"Thunderhawk",
|
||||
"Steel Force",
|
||||
"Wild Mouse",
|
||||
"Woodstock Express",
|
||||
"Talon",
|
||||
"Hydra the Revenge",
|
||||
"Possessed",
|
||||
"Iron Menace",
|
||||
],
|
||||
cagreatamerica: [
|
||||
"Demon",
|
||||
"Grizzly",
|
||||
"Woodstock Express",
|
||||
"Patriot",
|
||||
"Flight Deck",
|
||||
"Lucy's Crabbie Cabbies",
|
||||
"Psycho Mouse",
|
||||
"Gold Striker",
|
||||
"RailBlazer",
|
||||
],
|
||||
frontiercity: [
|
||||
"Silver Bullet",
|
||||
"Wildcat",
|
||||
"Diamondback",
|
||||
"Steel Lasso",
|
||||
"Frankie's Mine Train",
|
||||
],
|
||||
};
|
||||
|
||||
export function getCoasterSet(parkId: string): Set<string> | null {
|
||||
const coasters = COASTER_LISTS[parkId];
|
||||
if (!coasters || coasters.length === 0) return null;
|
||||
return new Set(coasters.map(normalizeForMatch));
|
||||
}
|
||||
|
||||
export function hasCoasterData(): boolean {
|
||||
return Object.values(COASTER_LISTS).some((list) => list.length > 0);
|
||||
}
|
||||
@@ -1,288 +0,0 @@
|
||||
import Database from "better-sqlite3";
|
||||
import path from "path";
|
||||
import fs from "fs";
|
||||
|
||||
const DATA_DIR = path.join(process.cwd(), "data");
|
||||
const DB_PATH = path.join(DATA_DIR, "parks.db");
|
||||
|
||||
export type DbInstance = Database.Database;
|
||||
|
||||
export function openDb(): Database.Database {
|
||||
fs.mkdirSync(DATA_DIR, { recursive: true });
|
||||
const db = new Database(DB_PATH);
|
||||
db.pragma("journal_mode = WAL");
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS park_days (
|
||||
park_id TEXT NOT NULL,
|
||||
date TEXT NOT NULL, -- YYYY-MM-DD
|
||||
is_open INTEGER NOT NULL DEFAULT 0,
|
||||
hours_label TEXT,
|
||||
special_type TEXT, -- 'passholder_preview' | null
|
||||
scraped_at TEXT NOT NULL,
|
||||
PRIMARY KEY (park_id, date)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS park_api_ids (
|
||||
park_id TEXT PRIMARY KEY,
|
||||
api_id INTEGER NOT NULL,
|
||||
api_abbreviation TEXT,
|
||||
api_name TEXT,
|
||||
discovered_at TEXT NOT NULL
|
||||
)
|
||||
`);
|
||||
// Migrate existing databases that predate the special_type column
|
||||
try {
|
||||
db.exec(`ALTER TABLE park_days ADD COLUMN special_type TEXT`);
|
||||
} catch {
|
||||
// Column already exists — safe to ignore
|
||||
}
|
||||
return db;
|
||||
}
|
||||
|
||||
export function upsertDay(
|
||||
db: Database.Database,
|
||||
parkId: string,
|
||||
date: string,
|
||||
isOpen: boolean,
|
||||
hoursLabel?: string,
|
||||
specialType?: string
|
||||
) {
|
||||
// Today and future dates: full upsert — hours can change (e.g. weather delays,
|
||||
// early closures) and the dateless API endpoint now returns today's live data.
|
||||
//
|
||||
// Past dates: INSERT-only — never overwrite once the day has passed.
|
||||
db.prepare(`
|
||||
INSERT INTO park_days (park_id, date, is_open, hours_label, special_type, scraped_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (park_id, date) DO UPDATE SET
|
||||
is_open = excluded.is_open,
|
||||
hours_label = excluded.hours_label,
|
||||
special_type = excluded.special_type,
|
||||
scraped_at = excluded.scraped_at
|
||||
WHERE park_days.date >= date('now')
|
||||
`).run(parkId, date, isOpen ? 1 : 0, hoursLabel ?? null, specialType ?? null, new Date().toISOString());
|
||||
}
|
||||
|
||||
export interface DayData {
|
||||
isOpen: boolean;
|
||||
hoursLabel: string | null;
|
||||
specialType: string | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns scraped data for all parks across a date range.
|
||||
* Shape: { parkId: { 'YYYY-MM-DD': DayData } }
|
||||
* Missing dates mean that date hasn't been scraped yet (not necessarily closed).
|
||||
*/
|
||||
export function getDateRange(
|
||||
db: Database.Database,
|
||||
startDate: string,
|
||||
endDate: string
|
||||
): Record<string, Record<string, DayData>> {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT park_id, date, is_open, hours_label, special_type
|
||||
FROM park_days
|
||||
WHERE date >= ? AND date <= ?`
|
||||
)
|
||||
.all(startDate, endDate) as {
|
||||
park_id: string;
|
||||
date: string;
|
||||
is_open: number;
|
||||
hours_label: string | null;
|
||||
special_type: string | null;
|
||||
}[];
|
||||
|
||||
const result: Record<string, Record<string, DayData>> = {};
|
||||
for (const row of rows) {
|
||||
if (!result[row.park_id]) result[row.park_id] = {};
|
||||
result[row.park_id][row.date] = {
|
||||
isOpen: row.is_open === 1,
|
||||
hoursLabel: row.hours_label,
|
||||
specialType: row.special_type,
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns scraped DayData for a single park for an entire month.
|
||||
* Shape: { 'YYYY-MM-DD': DayData }
|
||||
*/
|
||||
export function getParkMonthData(
|
||||
db: Database.Database,
|
||||
parkId: string,
|
||||
year: number,
|
||||
month: number,
|
||||
): Record<string, DayData> {
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT date, is_open, hours_label, special_type
|
||||
FROM park_days
|
||||
WHERE park_id = ? AND date LIKE ? || '-%'
|
||||
ORDER BY date`
|
||||
)
|
||||
.all(parkId, prefix) as {
|
||||
date: string;
|
||||
is_open: number;
|
||||
hours_label: string | null;
|
||||
special_type: string | null;
|
||||
}[];
|
||||
|
||||
const result: Record<string, DayData> = {};
|
||||
for (const row of rows) {
|
||||
result[row.date] = {
|
||||
isOpen: row.is_open === 1,
|
||||
hoursLabel: row.hours_label,
|
||||
specialType: row.special_type,
|
||||
};
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Returns a map of parkId → boolean[] (index 0 = day 1) for a given month. */
|
||||
export function getMonthCalendar(
|
||||
db: Database.Database,
|
||||
year: number,
|
||||
month: number
|
||||
): Record<string, boolean[]> {
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT park_id, date, is_open
|
||||
FROM park_days
|
||||
WHERE date LIKE ? || '-%'
|
||||
ORDER BY date`
|
||||
)
|
||||
.all(prefix) as { park_id: string; date: string; is_open: number }[];
|
||||
|
||||
const result: Record<string, boolean[]> = {};
|
||||
for (const row of rows) {
|
||||
if (!result[row.park_id]) result[row.park_id] = [];
|
||||
const day = parseInt(row.date.slice(8), 10);
|
||||
result[row.park_id][day - 1] = row.is_open === 1;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
import { parseStalenessHours } from "./env";
|
||||
const STALE_AFTER_MS = parseStalenessHours(process.env.PARK_HOURS_STALENESS_HOURS, 72) * 60 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Returns true when the scraper should skip this park+month.
|
||||
*
|
||||
* Two reasons to skip:
|
||||
* 1. The month is entirely in the past — the API will never return data for
|
||||
* those dates again, so re-scraping wastes a call and risks nothing but
|
||||
* wasted time. Historical records are preserved forever by upsertDay.
|
||||
* 2. The month was scraped within the last 7 days — data is still fresh.
|
||||
*/
|
||||
export function isMonthScraped(
|
||||
db: Database.Database,
|
||||
parkId: string,
|
||||
year: number,
|
||||
month: number
|
||||
): boolean {
|
||||
// Compute the last calendar day of this month (avoids timezone issues).
|
||||
const daysInMonth = new Date(year, month, 0).getDate();
|
||||
const lastDay = `${year}-${String(month).padStart(2, "0")}-${String(daysInMonth).padStart(2, "0")}`;
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
|
||||
// Past month — history is locked in, no API data available, always skip.
|
||||
if (lastDay < today) return true;
|
||||
|
||||
// Current/future month — skip only if recently scraped.
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT MAX(scraped_at) AS last_scraped
|
||||
FROM park_days
|
||||
WHERE park_id = ? AND date LIKE ? || '-%'`
|
||||
)
|
||||
.get(parkId, prefix) as { last_scraped: string | null };
|
||||
|
||||
if (!row.last_scraped) return false;
|
||||
const ageMs = Date.now() - new Date(row.last_scraped).getTime();
|
||||
return ageMs < STALE_AFTER_MS;
|
||||
}
|
||||
|
||||
export function getApiId(db: Database.Database, parkId: string): number | null {
|
||||
const row = db
|
||||
.prepare("SELECT api_id FROM park_api_ids WHERE park_id = ?")
|
||||
.get(parkId) as { api_id: number } | undefined;
|
||||
return row?.api_id ?? null;
|
||||
}
|
||||
|
||||
export function setApiId(
|
||||
db: Database.Database,
|
||||
parkId: string,
|
||||
apiId: number,
|
||||
apiAbbreviation?: string,
|
||||
apiName?: string
|
||||
) {
|
||||
db.prepare(`
|
||||
INSERT INTO park_api_ids (park_id, api_id, api_abbreviation, api_name, discovered_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT (park_id) DO UPDATE SET
|
||||
api_id = excluded.api_id,
|
||||
api_abbreviation = excluded.api_abbreviation,
|
||||
api_name = excluded.api_name,
|
||||
discovered_at = excluded.discovered_at
|
||||
`).run(
|
||||
parkId,
|
||||
apiId,
|
||||
apiAbbreviation ?? null,
|
||||
apiName ?? null,
|
||||
new Date().toISOString()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the next park+month to scrape.
|
||||
* Priority: never-scraped first, then oldest scraped_at.
|
||||
* Considers current month through monthsAhead months into the future.
|
||||
*/
|
||||
export function getNextScrapeTarget(
|
||||
db: Database.Database,
|
||||
parkIds: string[],
|
||||
monthsAhead = 12
|
||||
): { parkId: string; year: number; month: number } | null {
|
||||
const now = new Date();
|
||||
|
||||
const candidates: {
|
||||
parkId: string;
|
||||
year: number;
|
||||
month: number;
|
||||
lastScraped: string | null;
|
||||
}[] = [];
|
||||
|
||||
for (const parkId of parkIds) {
|
||||
for (let i = 0; i < monthsAhead; i++) {
|
||||
const d = new Date(now.getFullYear(), now.getMonth() + i, 1);
|
||||
const year = d.getFullYear();
|
||||
const month = d.getMonth() + 1;
|
||||
const prefix = `${year}-${String(month).padStart(2, "0")}`;
|
||||
|
||||
const row = db
|
||||
.prepare(
|
||||
`SELECT MAX(scraped_at) AS last_scraped
|
||||
FROM park_days
|
||||
WHERE park_id = ? AND date LIKE ? || '-%'`
|
||||
)
|
||||
.get(parkId, prefix) as { last_scraped: string | null };
|
||||
|
||||
candidates.push({ parkId, year, month, lastScraped: row.last_scraped });
|
||||
}
|
||||
}
|
||||
|
||||
// Never-scraped (null) first, then oldest scraped_at
|
||||
candidates.sort((a, b) => {
|
||||
if (!a.lastScraped && !b.lastScraped) return 0;
|
||||
if (!a.lastScraped) return -1;
|
||||
if (!b.lastScraped) return 1;
|
||||
return a.lastScraped.localeCompare(b.lastScraped);
|
||||
});
|
||||
|
||||
const top = candidates[0];
|
||||
return top ? { parkId: top.parkId, year: top.year, month: top.month } : null;
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
/**
|
||||
* park-meta.json — persisted alongside the SQLite DB in data/
|
||||
*
|
||||
* This file stores per-park metadata that doesn't belong in the schedule DB:
|
||||
* - rcdb_id: user-supplied RCDB park ID (fills into https://rcdb.com/{id}.htm)
|
||||
* - coasters: list of operating roller coaster names scraped from RCDB
|
||||
* - coasters_scraped_at: ISO timestamp of last RCDB scrape
|
||||
*
|
||||
* discover.ts: ensures every park has a skeleton entry (rcdb_id null by default)
|
||||
* scrape.ts: populates coasters[] for parks with a known rcdb_id (30-day staleness)
|
||||
*/
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
const META_PATH = path.join(process.cwd(), "data", "park-meta.json");
|
||||
|
||||
export interface ParkMeta {
|
||||
/** RCDB park page ID — user fills this in manually after discover creates the skeleton */
|
||||
rcdb_id: number | null;
|
||||
/** Operating roller coaster names scraped from RCDB */
|
||||
coasters: string[];
|
||||
/** ISO timestamp of when coasters was last scraped from RCDB */
|
||||
coasters_scraped_at: string | null;
|
||||
}
|
||||
|
||||
export type ParkMetaMap = Record<string, ParkMeta>;
|
||||
|
||||
export function readParkMeta(): ParkMetaMap {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(META_PATH, "utf8")) as ParkMetaMap;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export function writeParkMeta(meta: ParkMetaMap): void {
|
||||
fs.mkdirSync(path.dirname(META_PATH), { recursive: true });
|
||||
fs.writeFileSync(META_PATH, JSON.stringify(meta, null, 2) + "\n");
|
||||
}
|
||||
|
||||
/** Default skeleton entry for a park that has never been configured. */
|
||||
export function defaultParkMeta(): ParkMeta {
|
||||
return { rcdb_id: null, coasters: [], coasters_scraped_at: null };
|
||||
}
|
||||
|
||||
const COASTER_STALE_MS = parseStalenessHours(process.env.COASTER_STALENESS_HOURS, 720) * 60 * 60 * 1000;
|
||||
|
||||
/** Returns true when the coaster list needs to be re-scraped from RCDB. */
|
||||
export function areCoastersStale(entry: ParkMeta): boolean {
|
||||
if (!entry.coasters_scraped_at) return true;
|
||||
return Date.now() - new Date(entry.coasters_scraped_at).getTime() > COASTER_STALE_MS;
|
||||
}
|
||||
|
||||
import { normalizeForMatch } from "./coaster-match";
|
||||
export { normalizeForMatch as normalizeRideName } from "./coaster-match";
|
||||
import { parseStalenessHours } from "./env";
|
||||
|
||||
/**
|
||||
* Returns a Set of normalized coaster names for fast membership checks.
|
||||
* Returns null when no coaster data exists for the park.
|
||||
*/
|
||||
export function getCoasterSet(parkId: string, meta: ParkMetaMap): Set<string> | null {
|
||||
const entry = meta[parkId];
|
||||
if (!entry || entry.coasters.length === 0) return null;
|
||||
return new Set(entry.coasters.map(normalizeForMatch));
|
||||
}
|
||||
@@ -11,6 +11,7 @@ export const PARKS: Park[] = [
|
||||
// ── Six Flags branded parks ──────────────────────────────────────────────
|
||||
{
|
||||
id: "greatadventure",
|
||||
apiId: 905,
|
||||
name: "Six Flags Great Adventure",
|
||||
shortName: "Great Adventure",
|
||||
chain: "sixflags",
|
||||
@@ -22,6 +23,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "magicmountain",
|
||||
apiId: 906,
|
||||
name: "Six Flags Magic Mountain",
|
||||
shortName: "Magic Mountain",
|
||||
chain: "sixflags",
|
||||
@@ -33,6 +35,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "greatamerica",
|
||||
apiId: 910,
|
||||
name: "Six Flags Great America",
|
||||
shortName: "Great America",
|
||||
chain: "sixflags",
|
||||
@@ -44,6 +47,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "overgeorgia",
|
||||
apiId: 902,
|
||||
name: "Six Flags Over Georgia",
|
||||
shortName: "Over Georgia",
|
||||
chain: "sixflags",
|
||||
@@ -55,6 +59,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "overtexas",
|
||||
apiId: 901,
|
||||
name: "Six Flags Over Texas",
|
||||
shortName: "Over Texas",
|
||||
chain: "sixflags",
|
||||
@@ -66,6 +71,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "stlouis",
|
||||
apiId: 903,
|
||||
name: "Six Flags St. Louis",
|
||||
shortName: "St. Louis",
|
||||
chain: "sixflags",
|
||||
@@ -77,6 +83,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "fiestatexas",
|
||||
apiId: 914,
|
||||
name: "Six Flags Fiesta Texas",
|
||||
shortName: "Fiesta Texas",
|
||||
chain: "sixflags",
|
||||
@@ -88,6 +95,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "newengland",
|
||||
apiId: 935,
|
||||
name: "Six Flags New England",
|
||||
shortName: "New England",
|
||||
chain: "sixflags",
|
||||
@@ -99,6 +107,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "discoverykingdom",
|
||||
apiId: 936,
|
||||
name: "Six Flags Discovery Kingdom",
|
||||
shortName: "Discovery Kingdom",
|
||||
chain: "sixflags",
|
||||
@@ -110,6 +119,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "mexico",
|
||||
apiId: 960,
|
||||
name: "Six Flags Mexico",
|
||||
shortName: "Mexico",
|
||||
chain: "sixflags",
|
||||
@@ -121,6 +131,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "greatescape",
|
||||
apiId: 924,
|
||||
name: "Six Flags Great Escape",
|
||||
shortName: "Great Escape",
|
||||
chain: "sixflags",
|
||||
@@ -132,6 +143,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "darienlake",
|
||||
apiId: 945,
|
||||
name: "Six Flags Darien Lake",
|
||||
shortName: "Darien Lake",
|
||||
chain: "sixflags",
|
||||
@@ -144,6 +156,7 @@ export const PARKS: Park[] = [
|
||||
// ── Former Cedar Fair theme parks ─────────────────────────────────────────
|
||||
{
|
||||
id: "cedarpoint",
|
||||
apiId: 1,
|
||||
name: "Cedar Point",
|
||||
shortName: "Cedar Point",
|
||||
chain: "sixflags",
|
||||
@@ -155,6 +168,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "knotts",
|
||||
apiId: 4,
|
||||
name: "Knott's Berry Farm",
|
||||
shortName: "Knott's",
|
||||
chain: "sixflags",
|
||||
@@ -166,6 +180,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "canadaswonderland",
|
||||
apiId: 40,
|
||||
name: "Canada's Wonderland",
|
||||
shortName: "Canada's Wonderland",
|
||||
chain: "sixflags",
|
||||
@@ -177,6 +192,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "carowinds",
|
||||
apiId: 30,
|
||||
name: "Carowinds",
|
||||
shortName: "Carowinds",
|
||||
chain: "sixflags",
|
||||
@@ -188,6 +204,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "kingsdominion",
|
||||
apiId: 25,
|
||||
name: "Kings Dominion",
|
||||
shortName: "Kings Dominion",
|
||||
chain: "sixflags",
|
||||
@@ -199,6 +216,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "kingsisland",
|
||||
apiId: 20,
|
||||
name: "Kings Island",
|
||||
shortName: "Kings Island",
|
||||
chain: "sixflags",
|
||||
@@ -210,6 +228,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "valleyfair",
|
||||
apiId: 14,
|
||||
name: "Valleyfair",
|
||||
shortName: "Valleyfair",
|
||||
chain: "sixflags",
|
||||
@@ -221,6 +240,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "worldsoffun",
|
||||
apiId: 6,
|
||||
name: "Worlds of Fun",
|
||||
shortName: "Worlds of Fun",
|
||||
chain: "sixflags",
|
||||
@@ -232,6 +252,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "miadventure",
|
||||
apiId: 12,
|
||||
name: "Michigan's Adventure",
|
||||
shortName: "Michigan's Adventure",
|
||||
chain: "sixflags",
|
||||
@@ -243,6 +264,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "dorneypark",
|
||||
apiId: 8,
|
||||
name: "Dorney Park",
|
||||
shortName: "Dorney Park",
|
||||
chain: "sixflags",
|
||||
@@ -254,6 +276,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "cagreatamerica",
|
||||
apiId: 35,
|
||||
name: "California's Great America",
|
||||
shortName: "CA Great America",
|
||||
chain: "sixflags",
|
||||
@@ -265,6 +288,7 @@ export const PARKS: Park[] = [
|
||||
},
|
||||
{
|
||||
id: "frontiercity",
|
||||
apiId: 943,
|
||||
name: "Frontier City",
|
||||
shortName: "Frontier City",
|
||||
chain: "sixflags",
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
/**
|
||||
* RCDB (Roller Coaster DataBase) scraper.
|
||||
*
|
||||
* Fetches a park's RCDB page (https://rcdb.com/{id}.htm) and extracts the
|
||||
* names of operating roller coasters from the "Operating Roller Coasters"
|
||||
* section.
|
||||
*
|
||||
* RCDB has no public API. This scraper reads the static HTML page.
|
||||
* Please scrape infrequently (30-day staleness window) to be respectful.
|
||||
*/
|
||||
|
||||
const BASE = "https://rcdb.com";
|
||||
|
||||
const HEADERS = {
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
Accept: "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
};
|
||||
|
||||
/**
|
||||
* Scrape operating roller coaster names for a park.
|
||||
*
|
||||
* Returns an array of coaster names on success, or null when the page
|
||||
* cannot be fetched or contains no operating coasters.
|
||||
*/
|
||||
export async function scrapeRcdbCoasters(rcdbId: number): Promise<string[] | null> {
|
||||
const url = `${BASE}/${rcdbId}.htm`;
|
||||
try {
|
||||
const res = await fetch(url, { headers: HEADERS, signal: AbortSignal.timeout(15_000) });
|
||||
if (!res.ok) {
|
||||
console.error(` RCDB ${rcdbId}: HTTP ${res.status}`);
|
||||
return null;
|
||||
}
|
||||
const html = await res.text();
|
||||
return parseOperatingCoasters(html);
|
||||
} catch (err) {
|
||||
console.error(` RCDB ${rcdbId}: ${err}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse operating roller coaster names from RCDB park page HTML.
|
||||
*
|
||||
* RCDB park pages list coasters in sections bounded by <section> tags.
|
||||
* The operating section heading looks like:
|
||||
* <h4>Operating Roller Coasters: <a href="...">16</a></h4>
|
||||
*
|
||||
* Each coaster is an <a> link to its detail page with an unquoted href:
|
||||
* <td data-sort="Batman The Ride"><a href=/5.htm>Batman The Ride</a>
|
||||
*
|
||||
* We extract only those links (href=/DIGITS.htm) from within the
|
||||
* operating section, stopping at the next <section> tag.
|
||||
*/
|
||||
function parseOperatingCoasters(html: string): string[] {
|
||||
// Find the "Operating Roller Coasters" section heading.
|
||||
const opIdx = html.search(/Operating\s+Roller\s+Coasters/i);
|
||||
if (opIdx === -1) return [];
|
||||
|
||||
// The section ends at the next <section> tag (e.g. "Defunct Roller Coasters").
|
||||
const after = html.slice(opIdx);
|
||||
const nextSection = after.search(/<section\b/i);
|
||||
const sectionHtml = nextSection > 0 ? after.slice(0, nextSection) : after;
|
||||
|
||||
// Extract coaster names from links to RCDB detail pages.
|
||||
// RCDB uses unquoted href attributes: href=/1234.htm
|
||||
// General links (/g.htm, /r.htm, /location.htm, etc.) won't match \d+\.htm.
|
||||
const names: string[] = [];
|
||||
const linkPattern = /<a\s[^>]*href=["']?\/(\d+)\.htm["']?[^>]*>([^<]+)<\/a>/gi;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = linkPattern.exec(sectionHtml)) !== null) {
|
||||
const name = decodeHtmlEntities(match[2].trim());
|
||||
if (name) names.push(name);
|
||||
}
|
||||
|
||||
// Deduplicate while preserving order
|
||||
return [...new Set(names)];
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(text: string): string {
|
||||
return text
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
|
||||
.replace(/&[a-z]+;/gi, "");
|
||||
}
|
||||
@@ -1,11 +1,8 @@
|
||||
/**
|
||||
* Six Flags scraper — calls the internal CloudFront operating-hours API directly.
|
||||
* Six Flags API client — calls the internal CloudFront operating-hours API.
|
||||
*
|
||||
* API: https://d18car1k0ff81h.cloudfront.net/operating-hours/park/{apiId}?date=YYYYMM
|
||||
* Returns full month data in one request — no browser needed.
|
||||
*
|
||||
* Each park has a numeric API ID that must be discovered first (see scripts/discover.ts).
|
||||
* Once stored in the DB, this scraper never touches a browser again.
|
||||
* Returns full month data in one request.
|
||||
*
|
||||
* Rate limiting: on 429/503, exponential backoff (30s → 60s → 120s), MAX_RETRIES attempts.
|
||||
*/
|
||||
@@ -309,7 +306,6 @@ export async function scrapeRidesForDay(
|
||||
|
||||
/**
|
||||
* Fetch operating hours for an entire month in a single API call.
|
||||
* apiId must be pre-discovered via scripts/discover.ts.
|
||||
*/
|
||||
export async function scrapeMonth(
|
||||
apiId: number,
|
||||
@@ -325,8 +321,7 @@ export async function scrapeMonth(
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch park info for a given API ID (used during discovery to identify park type).
|
||||
* Uses the current month so there's always some data.
|
||||
* Fetch park info for a given API ID. Uses the current month so there's always some data.
|
||||
*/
|
||||
export async function fetchParkInfo(
|
||||
apiId: number
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export interface Park {
|
||||
id: string;
|
||||
apiId: number;
|
||||
name: string;
|
||||
shortName: string;
|
||||
chain: "sixflags" | string;
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
export interface DayData {
|
||||
isOpen: boolean;
|
||||
hoursLabel: string | null;
|
||||
specialType: string | null;
|
||||
}
|
||||
@@ -11,8 +11,6 @@ const CSP = [
|
||||
].join("; ");
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
// better-sqlite3 is a native module — must not be bundled by webpack
|
||||
serverExternalPackages: ["better-sqlite3"],
|
||||
output: "standalone",
|
||||
|
||||
async headers() {
|
||||
|
||||
Generated
+3
-777
File diff suppressed because it is too large
Load Diff
@@ -7,27 +7,21 @@
|
||||
"build": "next build",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
"scrape": "tsx scripts/scrape.ts",
|
||||
"scrape:force": "tsx scripts/scrape.ts --rescrape",
|
||||
"discover": "tsx scripts/discover.ts",
|
||||
"debug": "tsx scripts/debug.ts",
|
||||
"test": "tsx --test tests/*.test.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^12.8.0",
|
||||
"next": "^15.3.0",
|
||||
"react": "^19.0.0",
|
||||
"react-dom": "^19.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/node": "^22",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"eslint": "^9",
|
||||
"eslint-config-next": "^15.3.0",
|
||||
"playwright": "^1.59.1",
|
||||
"tailwindcss": "^4",
|
||||
"tsx": "^4.21.0",
|
||||
"typescript": "^5"
|
||||
|
||||
+2
-13
@@ -9,7 +9,6 @@
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
import { openDb, getApiId } from "../lib/db";
|
||||
import { PARKS } from "../lib/parks";
|
||||
import { scrapeMonthRaw } from "../lib/scrapers/sixflags";
|
||||
|
||||
@@ -52,16 +51,6 @@ async function main() {
|
||||
const month = parseInt(monthStr);
|
||||
const day = parseInt(dayStr);
|
||||
|
||||
const db = openDb();
|
||||
const apiId = getApiId(db, park.id);
|
||||
db.close();
|
||||
|
||||
if (apiId === null) {
|
||||
console.error(`No API ID found for ${park.name} — run: npm run discover`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Collect all output so we can write it to a file as well
|
||||
const lines: string[] = [];
|
||||
const out = (...args: string[]) => {
|
||||
const line = args.join(" ");
|
||||
@@ -70,13 +59,13 @@ async function main() {
|
||||
};
|
||||
|
||||
out(`Park : ${park.name} (${park.id})`);
|
||||
out(`API ID : ${apiId}`);
|
||||
out(`API ID : ${park.apiId}`);
|
||||
out(`Date : ${dateStr}`);
|
||||
out(`Fetched : ${new Date().toISOString()}`);
|
||||
out("");
|
||||
out(`Fetching ${year}-${String(month).padStart(2, "0")} from API...`);
|
||||
|
||||
const raw = await scrapeMonthRaw(apiId, year, month);
|
||||
const raw = await scrapeMonthRaw(park.apiId, year, month);
|
||||
|
||||
const targetDate = `${String(month).padStart(2, "0")}/${String(day).padStart(2, "0")}/${year}`;
|
||||
const dayData = raw.dates.find((d) => d.date === targetDate);
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
/**
|
||||
* One-time discovery script — finds the CloudFront API ID for each park.
|
||||
*
|
||||
* Run this once before using scrape.ts:
|
||||
* npx tsx scripts/discover.ts
|
||||
*
|
||||
* For each park in the registry it:
|
||||
* 1. Opens the park's hours page in a headless browser
|
||||
* 2. Intercepts all calls to the operating-hours CloudFront API
|
||||
* 3. Identifies the main theme park ID (filters out water parks, safari, etc.)
|
||||
* 4. Stores the ID in the database
|
||||
*
|
||||
* Re-running is safe — already-discovered parks are skipped.
|
||||
*/
|
||||
|
||||
import { chromium } from "playwright";
|
||||
import { openDb, getApiId, setApiId, type DbInstance } from "../lib/db";
|
||||
import { PARKS } from "../lib/parks";
|
||||
import { fetchParkInfo, isMainThemePark } from "../lib/scrapers/sixflags";
|
||||
import { readParkMeta, writeParkMeta, defaultParkMeta } from "../lib/park-meta";
|
||||
|
||||
const CLOUDFRONT_PATTERN = /operating-hours\/park\/(\d+)/;
|
||||
|
||||
async function discoverParkId(slug: string): Promise<number | null> {
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
try {
|
||||
const context = await browser.newContext({
|
||||
userAgent:
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
locale: "en-US",
|
||||
});
|
||||
const page = await context.newPage();
|
||||
|
||||
const capturedIds = new Set<number>();
|
||||
page.on("request", (req) => {
|
||||
const match = req.url().match(CLOUDFRONT_PATTERN);
|
||||
if (match) capturedIds.add(parseInt(match[1]));
|
||||
});
|
||||
|
||||
await page
|
||||
.goto(`https://www.sixflags.com/${slug}/park-hours?date=2026-05-01`, {
|
||||
waitUntil: "networkidle",
|
||||
timeout: 30_000,
|
||||
})
|
||||
.catch(() => null);
|
||||
|
||||
await context.close();
|
||||
|
||||
if (capturedIds.size === 0) return null;
|
||||
|
||||
// Check each captured ID — pick the main theme park (not water park / safari)
|
||||
for (const id of capturedIds) {
|
||||
const info = await fetchParkInfo(id);
|
||||
if (info && isMainThemePark(info.parkName)) {
|
||||
console.log(
|
||||
` → ID ${id} | ${info.parkAbbreviation} | ${info.parkName}`
|
||||
);
|
||||
return id;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: return the lowest ID (usually the main park)
|
||||
const fallback = Math.min(...capturedIds);
|
||||
console.log(` → fallback to lowest ID: ${fallback}`);
|
||||
return fallback;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
function purgeRemovedParks(db: DbInstance) {
|
||||
const knownIds = new Set(PARKS.map((p) => p.id));
|
||||
|
||||
const staleParkIds = (
|
||||
db.prepare("SELECT DISTINCT park_id FROM park_api_ids").all() as { park_id: string }[]
|
||||
)
|
||||
.map((r) => r.park_id)
|
||||
.filter((id) => !knownIds.has(id));
|
||||
|
||||
if (staleParkIds.length === 0) return;
|
||||
|
||||
console.log(`\nRemoving ${staleParkIds.length} park(s) no longer in registry:`);
|
||||
for (const parkId of staleParkIds) {
|
||||
const days = (
|
||||
db.prepare("SELECT COUNT(*) AS n FROM park_days WHERE park_id = ?").get(parkId) as { n: number }
|
||||
).n;
|
||||
db.prepare("DELETE FROM park_days WHERE park_id = ?").run(parkId);
|
||||
db.prepare("DELETE FROM park_api_ids WHERE park_id = ?").run(parkId);
|
||||
console.log(` removed ${parkId} (${days} day rows deleted)`);
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const db = openDb();
|
||||
|
||||
purgeRemovedParks(db);
|
||||
|
||||
for (const park of PARKS) {
|
||||
const existing = getApiId(db, park.id);
|
||||
if (existing !== null) {
|
||||
console.log(`${park.name}: already known (API ID ${existing}) — skip`);
|
||||
continue;
|
||||
}
|
||||
|
||||
process.stdout.write(`${park.name} (${park.slug})... `);
|
||||
|
||||
try {
|
||||
const apiId = await discoverParkId(park.slug);
|
||||
if (apiId === null) {
|
||||
console.log("FAILED — no API IDs captured");
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fetch full info to store name/abbreviation
|
||||
const info = await fetchParkInfo(apiId);
|
||||
setApiId(db, park.id, apiId, info?.parkAbbreviation, info?.parkName);
|
||||
} catch (err) {
|
||||
console.log(`ERROR: ${err}`);
|
||||
}
|
||||
|
||||
// Small delay between parks to be polite
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
|
||||
// ── Ensure park-meta.json has a skeleton entry for every park ────────────
|
||||
// Users fill in rcdb_id manually; scrape.ts populates coasters[] from RCDB.
|
||||
const meta = readParkMeta();
|
||||
let metaChanged = false;
|
||||
|
||||
for (const park of PARKS) {
|
||||
if (!meta[park.id]) {
|
||||
meta[park.id] = defaultParkMeta();
|
||||
metaChanged = true;
|
||||
}
|
||||
}
|
||||
// Remove entries for parks no longer in the registry
|
||||
for (const id of Object.keys(meta)) {
|
||||
if (!PARKS.find((p) => p.id === id)) {
|
||||
delete meta[id];
|
||||
metaChanged = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (metaChanged) {
|
||||
writeParkMeta(meta);
|
||||
console.log("\nUpdated data/park-meta.json");
|
||||
console.log(" → Set rcdb_id for each park to enable the coaster filter.");
|
||||
console.log(" Find a park's RCDB ID from: https://rcdb.com (the number in the URL).");
|
||||
}
|
||||
|
||||
// Print summary
|
||||
console.log("\n── Discovered IDs ──");
|
||||
for (const park of PARKS) {
|
||||
const id = getApiId(db, park.id);
|
||||
const rcdbId = meta[park.id]?.rcdb_id;
|
||||
const rcdbStr = rcdbId ? `rcdb:${rcdbId}` : "rcdb:?";
|
||||
console.log(` ${park.id.padEnd(30)} api:${String(id ?? "?").padEnd(8)} ${rcdbStr}`);
|
||||
}
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Nightly scraper scheduler — runs inside the Docker scraper service.
|
||||
#
|
||||
# Behaviour:
|
||||
# 1. Runs an initial scrape immediately on container start.
|
||||
# 2. Sleeps until 3:00 AM (container timezone, set via TZ env var).
|
||||
# 3. Runs the scraper, then sleeps until the next 3:00 AM, forever.
|
||||
#
|
||||
# Timezone: set TZ in the scraper service environment to control when
|
||||
# "3am" is (e.g. TZ=America/New_York). Defaults to UTC if unset.
|
||||
|
||||
log() {
|
||||
echo "[scheduler] $(date '+%Y-%m-%d %H:%M %Z') — $*"
|
||||
}
|
||||
|
||||
run_scrape() {
|
||||
log "Starting scrape"
|
||||
if npm run scrape; then
|
||||
log "Scrape completed"
|
||||
else
|
||||
log "Scrape failed — will retry at next scheduled time"
|
||||
fi
|
||||
}
|
||||
|
||||
seconds_until_3am() {
|
||||
now=$(date +%s)
|
||||
# Try today's 3am first; if already past, use tomorrow's.
|
||||
target=$(date -d "today 03:00" +%s)
|
||||
if [ "$now" -ge "$target" ]; then
|
||||
target=$(date -d "tomorrow 03:00" +%s)
|
||||
fi
|
||||
echo $((target - now))
|
||||
}
|
||||
|
||||
# ── Run immediately on startup ────────────────────────────────────────────────
|
||||
run_scrape
|
||||
|
||||
# ── Nightly loop ──────────────────────────────────────────────────────────────
|
||||
while true; do
|
||||
wait=$(seconds_until_3am)
|
||||
next=$(date -d "now + ${wait} seconds" '+%Y-%m-%d %H:%M %Z')
|
||||
log "Next scrape in $((wait / 3600))h $((( wait % 3600) / 60))m (${next})"
|
||||
sleep "$wait"
|
||||
run_scrape
|
||||
done
|
||||
@@ -1,164 +0,0 @@
|
||||
/**
|
||||
* Scrape job — fetches 2026 operating hours for all parks from the Six Flags API.
|
||||
*
|
||||
* Prerequisite: run `npm run discover` first to populate API IDs.
|
||||
*
|
||||
* npm run scrape — skips months scraped within the last 7 days
|
||||
* npm run scrape:force — re-scrapes everything
|
||||
*/
|
||||
|
||||
import { openDb, upsertDay, getApiId, isMonthScraped } from "../lib/db";
|
||||
import { PARKS } from "../lib/parks";
|
||||
import { scrapeMonth, fetchToday, RateLimitError } from "../lib/scrapers/sixflags";
|
||||
import { readParkMeta, writeParkMeta, areCoastersStale } from "../lib/park-meta";
|
||||
import { scrapeRcdbCoasters } from "../lib/scrapers/rcdb";
|
||||
|
||||
const YEAR = 2026;
|
||||
const MONTHS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
|
||||
const DELAY_MS = 1000;
|
||||
const FORCE = process.argv.includes("--rescrape");
|
||||
|
||||
async function sleep(ms: number) {
|
||||
return new Promise<void>((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const db = openDb();
|
||||
|
||||
const ready = PARKS.filter((p) => getApiId(db, p.id) !== null);
|
||||
const needsDiscovery = PARKS.filter((p) => getApiId(db, p.id) === null);
|
||||
|
||||
if (needsDiscovery.length > 0) {
|
||||
console.log(
|
||||
`⚠ ${needsDiscovery.length} park(s) need discovery first: ${needsDiscovery.map((p) => p.id).join(", ")}\n`
|
||||
);
|
||||
}
|
||||
|
||||
if (ready.length === 0) {
|
||||
console.log("No parks ready — run: npm run discover");
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Scraping ${YEAR} — ${ready.length} parks\n`);
|
||||
|
||||
let totalFetched = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
for (const park of ready) {
|
||||
const apiId = getApiId(db, park.id)!;
|
||||
const label = park.shortName.padEnd(22);
|
||||
|
||||
let openDays = 0;
|
||||
let fetched = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
process.stdout.write(` ${label} `);
|
||||
|
||||
for (const month of MONTHS) {
|
||||
if (!FORCE && isMonthScraped(db, park.id, YEAR, month)) {
|
||||
process.stdout.write("·");
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const days = await scrapeMonth(apiId, YEAR, month);
|
||||
db.transaction(() => {
|
||||
for (const d of days) upsertDay(db, park.id, d.date, d.isOpen, d.hoursLabel, d.specialType);
|
||||
})();
|
||||
openDays += days.filter((d) => d.isOpen).length;
|
||||
fetched++;
|
||||
process.stdout.write("█");
|
||||
if (fetched + skipped + errors < MONTHS.length) await sleep(DELAY_MS);
|
||||
} catch (err) {
|
||||
if (err instanceof RateLimitError) {
|
||||
process.stdout.write("✗");
|
||||
} else {
|
||||
process.stdout.write("✗");
|
||||
console.error(`\n error: ${err instanceof Error ? err.message : err}`);
|
||||
}
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
totalFetched += fetched;
|
||||
totalSkipped += skipped;
|
||||
totalErrors += errors;
|
||||
|
||||
if (errors > 0) {
|
||||
console.log(` ${errors} error(s)`);
|
||||
} else if (skipped === MONTHS.length) {
|
||||
console.log(" up to date");
|
||||
} else {
|
||||
console.log(` ${openDays} open days`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`\n ${totalFetched} fetched ${totalSkipped} skipped ${totalErrors} errors`);
|
||||
if (totalErrors > 0) console.log(" Re-run to retry failed months.");
|
||||
|
||||
// ── Today scrape (always fresh — dateless endpoint returns current day) ────
|
||||
console.log("\n── Today's data ──");
|
||||
for (const park of ready) {
|
||||
const apiId = getApiId(db, park.id)!;
|
||||
process.stdout.write(` ${park.shortName.padEnd(22)} `);
|
||||
try {
|
||||
const today = await fetchToday(apiId);
|
||||
if (today) {
|
||||
upsertDay(db, park.id, today.date, today.isOpen, today.hoursLabel, today.specialType);
|
||||
console.log(today.isOpen ? `open ${today.hoursLabel ?? ""}` : "closed");
|
||||
} else {
|
||||
console.log("no data");
|
||||
}
|
||||
} catch {
|
||||
console.log("error");
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
|
||||
db.close();
|
||||
|
||||
// ── RCDB coaster scrape (30-day staleness) ────────────────────────────────
|
||||
const meta = readParkMeta();
|
||||
const rcdbParks = PARKS.filter((p) => {
|
||||
const entry = meta[p.id];
|
||||
return entry?.rcdb_id && (FORCE || areCoastersStale(entry));
|
||||
});
|
||||
|
||||
if (rcdbParks.length === 0) {
|
||||
console.log("\nCoaster data up to date.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`\n── RCDB coaster scrape — ${rcdbParks.length} park(s) ──`);
|
||||
|
||||
for (const park of rcdbParks) {
|
||||
const entry = meta[park.id];
|
||||
const rcdbId = entry.rcdb_id!;
|
||||
process.stdout.write(` ${park.shortName.padEnd(30)} `);
|
||||
|
||||
const coasters = await scrapeRcdbCoasters(rcdbId);
|
||||
if (coasters === null) {
|
||||
console.log("FAILED");
|
||||
continue;
|
||||
}
|
||||
|
||||
entry.coasters = coasters;
|
||||
entry.coasters_scraped_at = new Date().toISOString();
|
||||
console.log(`${coasters.length} coasters`);
|
||||
|
||||
// Polite delay between RCDB requests
|
||||
await new Promise((r) => setTimeout(r, 2000));
|
||||
}
|
||||
|
||||
writeParkMeta(meta);
|
||||
console.log(" Saved to data/park-meta.json");
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error("Fatal:", err);
|
||||
process.exit(1);
|
||||
});
|
||||
+1
-1
@@ -23,5 +23,5 @@
|
||||
}
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
"exclude": ["node_modules"]
|
||||
"exclude": ["node_modules", "backend"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user