feat: automated nightly scraper + housekeeping
All checks were successful
Build and Deploy / Build & Push (push) Successful in 3m11s
All checks were successful
Build and Deploy / Build & Push (push) Successful in 3m11s
Scraper automation (docker-compose): - Add scraper service to docker-compose.yml using the same image and shared park_data volume; overrides CMD to run scrape-schedule.sh - scripts/scrape-schedule.sh: runs an initial scrape on container start, then sleeps until 3:00 AM (respects TZ env var) and repeats nightly; logs timestamps and next-run countdown; non-fatal on scrape errors Staleness window: 7 days → 72 hours in lib/db.ts so data refreshes more frequently with the automated schedule in place Remove favicon: delete app/icon.tsx and public/logo.svg Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
45
scripts/scrape-schedule.sh
Normal file
45
scripts/scrape-schedule.sh
Normal file
@@ -0,0 +1,45 @@
|
||||
#!/bin/sh
|
||||
# Nightly scraper scheduler — runs inside the Docker scraper service.
|
||||
#
|
||||
# Behaviour:
|
||||
# 1. Runs an initial scrape immediately on container start.
|
||||
# 2. Sleeps until 3:00 AM (container timezone, set via TZ env var).
|
||||
# 3. Runs the scraper, then sleeps until the next 3:00 AM, forever.
|
||||
#
|
||||
# Timezone: set TZ in the scraper service environment to control when
|
||||
# "3am" is (e.g. TZ=America/New_York). Defaults to UTC if unset.
|
||||
|
||||
log() {
|
||||
echo "[scheduler] $(date '+%Y-%m-%d %H:%M %Z') — $*"
|
||||
}
|
||||
|
||||
run_scrape() {
|
||||
log "Starting scrape"
|
||||
if npm run scrape; then
|
||||
log "Scrape completed"
|
||||
else
|
||||
log "Scrape failed — will retry at next scheduled time"
|
||||
fi
|
||||
}
|
||||
|
||||
seconds_until_3am() {
|
||||
now=$(date +%s)
|
||||
# Try today's 3am first; if already past, use tomorrow's.
|
||||
target=$(date -d "today 03:00" +%s)
|
||||
if [ "$now" -ge "$target" ]; then
|
||||
target=$(date -d "tomorrow 03:00" +%s)
|
||||
fi
|
||||
echo $((target - now))
|
||||
}
|
||||
|
||||
# ── Run immediately on startup ────────────────────────────────────────────────
|
||||
run_scrape
|
||||
|
||||
# ── Nightly loop ──────────────────────────────────────────────────────────────
|
||||
while true; do
|
||||
wait=$(seconds_until_3am)
|
||||
next=$(date -d "now + ${wait} seconds" '+%Y-%m-%d %H:%M %Z')
|
||||
log "Next scrape in $((wait / 3600))h $((( wait % 3600) / 60))m (${next})"
|
||||
sleep "$wait"
|
||||
run_scrape
|
||||
done
|
||||
Reference in New Issue
Block a user