diff --git a/docker-compose.yml b/docker-compose.yml index 96fb7e7..6c0bde9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: build: . image: kuh-no-bowls:latest container_name: kuh-no-bowls-scraper - restart: unless-stopped + profiles: ["oneshot"] env_file: .env environment: LOG_PATH: /var/log/kuh-no-bowls/kuh-no-bowls.log @@ -13,6 +13,24 @@ services: networks: - services_net + # Hourly Open-Meteo weather -> knoebels."LZ_open_meteo_hourly". + # Oneshot, run from cron. Default mode does the recent sync; pass --backfill + # once to load history: docker compose --profile oneshot run --rm weather --backfill + weather: + build: . + image: kuh-no-bowls:latest + container_name: kuh-no-bowls-weather + profiles: ["oneshot"] + entrypoint: ["python", "-u", "weather_logger.py"] + env_file: .env + environment: + LOG_PATH: /var/log/kuh-no-bowls/weather.log + TZ: America/New_York + volumes: + - knb_logs:/var/log/kuh-no-bowls + networks: + - services_net + dashboard: build: . image: kuh-no-bowls:latest diff --git a/queries/dm_knb_queue_weather.sql b/queries/dm_knb_queue_weather.sql new file mode 100644 index 0000000..d5bdbb3 --- /dev/null +++ b/queries/dm_knb_queue_weather.sql @@ -0,0 +1,61 @@ +-- Full queue-time time series joined to hourly Open-Meteo weather. +-- Queue readings (sub-hour, every ~5 min) are bucketed to the hour and matched +-- to knoebels."LZ_open_meteo_hourly". Both sides are naive America/New_York, so +-- the hour buckets line up directly with no timezone math. LEFT JOIN keeps every +-- queue row even if an hour's weather is missing (e.g. the once-a-year DST hour). +SELECT a.time_stamp, + a._id AS ride_id, + TRIM(r.name) AS ride_name, + TRIM(c."name") AS category, + ROUND((r.minimum_height_requirement * 39.37)::numeric, 0) AS minimum_height_requirement_inches, + ROUND((r.minimum_unaccompanied_height_requirement * 39.37)::numeric, 0) AS minimum_unaccompanied_height_requirement_inches, + ROUND((r.maximum_height_requirement * 39.37)::numeric, 0) AS maximum_height_requirement_inches, + r."location" AS coordinates, + a.queue_time AS queue_time_sec, + w.temperature_2m AS temperature_f, + w.apparent_temperature AS feels_like_f, + w.relative_humidity_2m AS humidity_pct, + w.precipitation AS precipitation_in, + w.rain AS rain_in, + w.weather_code AS wmo_weather_code, + CASE w.weather_code + WHEN 0 THEN 'Clear sky' + WHEN 1 THEN 'Mainly clear' + WHEN 2 THEN 'Partly cloudy' + WHEN 3 THEN 'Overcast' + WHEN 45 THEN 'Fog' + WHEN 48 THEN 'Depositing rime fog' + WHEN 51 THEN 'Light drizzle' + WHEN 53 THEN 'Moderate drizzle' + WHEN 55 THEN 'Dense drizzle' + WHEN 56 THEN 'Light freezing drizzle' + WHEN 57 THEN 'Dense freezing drizzle' + WHEN 61 THEN 'Slight rain' + WHEN 63 THEN 'Moderate rain' + WHEN 65 THEN 'Heavy rain' + WHEN 66 THEN 'Light freezing rain' + WHEN 67 THEN 'Heavy freezing rain' + WHEN 71 THEN 'Slight snowfall' + WHEN 73 THEN 'Moderate snowfall' + WHEN 75 THEN 'Heavy snowfall' + WHEN 77 THEN 'Snow grains' + WHEN 80 THEN 'Slight rain showers' + WHEN 81 THEN 'Moderate rain showers' + WHEN 82 THEN 'Violent rain showers' + WHEN 85 THEN 'Slight snow showers' + WHEN 86 THEN 'Heavy snow showers' + WHEN 95 THEN 'Thunderstorm' + WHEN 96 THEN 'Thunderstorm with slight hail' + WHEN 99 THEN 'Thunderstorm with heavy hail' + END AS weather_description, + w.cloud_cover AS cloud_cover_pct, + w.wind_speed_10m AS wind_mph, + w.wind_gusts_10m AS wind_gust_mph + FROM knoebels."LZ_attractions_io" a + JOIN knoebels."LZ_attractions_io_poi" r + ON r._id = a._id + JOIN knoebels."LZ_attractions_io_categories" c + ON r.category = c._id + LEFT JOIN knoebels."LZ_open_meteo_hourly" w + ON w."time" = date_trunc('hour', a.time_stamp) + WHERE a.queue_time IS NOT NULL; diff --git a/systemd/knb-weather.service b/systemd/knb-weather.service new file mode 100644 index 0000000..eaf30e5 --- /dev/null +++ b/systemd/knb-weather.service @@ -0,0 +1,14 @@ +[Unit] +Description=Knoebels Open-Meteo hourly weather sync (one-shot) +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/home/nox/docker/kuh-no-bowls +# `compose run` activates the service regardless of its compose profile. +# Default (no args) = recent sync (Forecast API, past_days=7, self-healing). +# One-time history load was done manually with: ... run --rm weather --backfill +ExecStart=/usr/bin/docker compose run --rm weather +StandardOutput=append:/home/nox/docker/kuh-no-bowls/cron-logs/weather-cron.log +StandardError=append:/home/nox/docker/kuh-no-bowls/cron-logs/weather-cron.log diff --git a/systemd/knb-weather.timer b/systemd/knb-weather.timer new file mode 100644 index 0000000..685c7d5 --- /dev/null +++ b/systemd/knb-weather.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run the Knoebels weather sync every 6 hours + +[Timer] +# Every 6 hours (00/06/12/18 ET). past_days=7 means each run refreshes the +# last week, so a missed run self-heals and provisional values get refined. +OnCalendar=*-*-* 00/6:00:00 +Persistent=true +RandomizedDelaySec=300 + +[Install] +WantedBy=timers.target diff --git a/weather_logger.py b/weather_logger.py new file mode 100644 index 0000000..5a87224 --- /dev/null +++ b/weather_logger.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Fetch hourly weather for Knoebels (Elysburg, PA) from Open-Meteo and upsert +into knoebels."LZ_open_meteo_hourly". + +Two modes: + (default) recent -- Forecast API with past_days; keeps the table current and + self-heals recent gaps. This is what the daily cron runs. + --backfill -- Archive (ERA5 reanalysis) API back to --start + (default 2024-05-15, when collection began). + +Timestamps are stored as naive America/New_York to match how the rest of the +knoebels schema records time. Upserts are idempotent via ON CONFLICT (time), +so rerunning overwrites provisional values with better data as it arrives. +""" + +import argparse +import logging +import os +import sys +from datetime import date, datetime + +import psycopg2 +import requests +from psycopg2.extras import execute_values + +# Knoebels Amusement Resort, Elysburg PA +LATITUDE = 40.7906 +LONGITUDE = -76.4847 +COLLECTION_START = "2024-05-15" # first day of queue-time collection +TIMEZONE = "America/New_York" + +ARCHIVE_ENDPOINT = "https://archive-api.open-meteo.com/v1/archive" +FORECAST_ENDPOINT = "https://api.open-meteo.com/v1/forecast" + +# Hourly variables available identically in both the archive and forecast APIs. +HOURLY_VARS = [ + "temperature_2m", + "relative_humidity_2m", + "apparent_temperature", + "precipitation", + "rain", + "weather_code", + "cloud_cover", + "wind_speed_10m", + "wind_gusts_10m", +] + +COMMON_PARAMS = { + "latitude": LATITUDE, + "longitude": LONGITUDE, + "hourly": ",".join(HOURLY_VARS), + "temperature_unit": "fahrenheit", + "precipitation_unit": "inch", + "wind_speed_unit": "mph", + "timezone": TIMEZONE, +} + +LOG_PATH = os.environ.get("LOG_PATH") # unset -> log to stderr + +DB_PASSWORD = os.environ.get("DB_PASSWORD") +if not DB_PASSWORD: + sys.stderr.write("ERROR: DB_PASSWORD env var is required\n") + sys.exit(1) + +db_params = { + "dbname": os.environ.get("DB_NAME", "gp0"), + "user": os.environ.get("DB_USER", "kuhnobowls"), + "password": DB_PASSWORD, + "host": os.environ.get("DB_HOST", "192.168.88.9"), + "port": os.environ.get("DB_PORT", "5432"), + "options": "-c search_path=knoebels", +} + +CREATE_TABLE = """ +CREATE TABLE IF NOT EXISTS knoebels."LZ_open_meteo_hourly" ( + "time" timestamp without time zone PRIMARY KEY, + temperature_2m real, + relative_humidity_2m smallint, + apparent_temperature real, + precipitation real, + rain real, + weather_code smallint, + cloud_cover smallint, + wind_speed_10m real, + wind_gusts_10m real, + inserted_at timestamp without time zone DEFAULT now() +) +""" + +UPSERT = """ +INSERT INTO knoebels."LZ_open_meteo_hourly" + ("time", temperature_2m, relative_humidity_2m, apparent_temperature, + precipitation, rain, weather_code, cloud_cover, wind_speed_10m, wind_gusts_10m) +VALUES %s +ON CONFLICT ("time") DO UPDATE SET + temperature_2m = EXCLUDED.temperature_2m, + relative_humidity_2m = EXCLUDED.relative_humidity_2m, + apparent_temperature = EXCLUDED.apparent_temperature, + precipitation = EXCLUDED.precipitation, + rain = EXCLUDED.rain, + weather_code = EXCLUDED.weather_code, + cloud_cover = EXCLUDED.cloud_cover, + wind_speed_10m = EXCLUDED.wind_speed_10m, + wind_gusts_10m = EXCLUDED.wind_gusts_10m, + inserted_at = now() +""" + + +def fetch(endpoint, extra): + params = dict(COMMON_PARAMS, **extra) + resp = requests.get(endpoint, params=params, timeout=60) + resp.raise_for_status() + return resp.json() + + +def rows_from_payload(payload): + """Turn an Open-Meteo payload into row tuples, dropping all-null hours + (the archive API pads the last few days with nulls until ERA5 lands).""" + hourly = payload["hourly"] + times = hourly["time"] + cols = [hourly[var] for var in HOURLY_VARS] + rows = [] + for i, t in enumerate(times): + values = [col[i] for col in cols] + if all(v is None for v in values): + continue + rows.append((datetime.fromisoformat(t), *values)) + return rows + + +def upsert(conn, rows): + if not rows: + return 0 + with conn.cursor() as cur: + execute_values(cur, UPSERT, rows, page_size=1000) + conn.commit() + return len(rows) + + +def run_recent(conn, past_days, forecast_days): + payload = fetch(FORECAST_ENDPOINT, + {"past_days": past_days, "forecast_days": forecast_days}) + rows = rows_from_payload(payload) + n = upsert(conn, rows) + span = f"{rows[0][0]} .. {rows[-1][0]}" if rows else "(none)" + logging.info("recent: upserted %d hours [%s]", n, span) + return n + + +def run_backfill(conn, start): + end = date.today().isoformat() + payload = fetch(ARCHIVE_ENDPOINT, {"start_date": start, "end_date": end}) + rows = rows_from_payload(payload) + n = upsert(conn, rows) + span = f"{rows[0][0]} .. {rows[-1][0]}" if rows else "(none)" + logging.info("backfill: upserted %d hours [%s]", n, span) + return n + + +def main(): + parser = argparse.ArgumentParser(description="Open-Meteo -> knoebels weather logger") + parser.add_argument("--backfill", action="store_true", + help="pull the full ERA5 archive history before the recent sync") + parser.add_argument("--start", default=COLLECTION_START, + help="backfill start date (YYYY-MM-DD)") + parser.add_argument("--past-days", type=int, default=7, + help="forecast-API lookback for the recent sync (max 92)") + parser.add_argument("--forecast-days", type=int, default=1, + help="forecast-API lookahead for the recent sync (0-16)") + args = parser.parse_args() + + logging.basicConfig( + format="%(asctime)s %(levelname)s %(message)s", + level=logging.INFO, + filename=LOG_PATH, + filemode="a" if LOG_PATH else None, + ) + logging.info("-" * 80) + + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cur: + cur.execute(CREATE_TABLE) + conn.commit() + + if args.backfill: + run_backfill(conn, args.start) + # Always finish with a recent sync so the ERA5 gap (last ~5 days) is + # filled and the table is current. + run_recent(conn, args.past_days, args.forecast_days) + + +if __name__ == "__main__": + main()