From 10bdee8e1f84bf611662336095f590ca186d41b7 Mon Sep 17 00:00:00 2001 From: Wesley Ray Date: Wed, 6 May 2026 23:20:12 -0400 Subject: [PATCH] Containerize scraper and externalize credentials Move DB host/user/password, Discord webhook, and log path to env vars so the script can be deployed via Docker compose. Discord notifications are now skipped silently when DISCORD_WEBHOOK is unset (the old hardcoded webhook is removed pending rotation). Adds Dockerfile, docker-compose.yml, requirements.txt, .env.example, and .gitignore for the cyrion deployment. --- .env.example | 16 ++ .gitignore | 7 + Dockerfile | 10 ++ attractions_api_logger.py | 369 +++++++++++++++++++------------------- docker-compose.yml | 14 ++ requirements.txt | 2 + 6 files changed, 237 insertions(+), 181 deletions(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b0010fc --- /dev/null +++ b/.env.example @@ -0,0 +1,16 @@ +# Knoebels scraper environment configuration. +# Copy to .env on the deploy host and fill in real values. +# Never commit the real .env. + +# PostgreSQL connection (phlegethon by default) +DB_HOST=192.168.88.9 +DB_PORT=5432 +DB_USER=kuhnobowls +DB_PASSWORD= + +# Optional Discord webhook for notifications. +# Leave empty/unset to skip Discord calls entirely. +DISCORD_WEBHOOK= + +# Log path inside the container. Mapped to the knb_logs named volume by compose. +LOG_PATH=/var/log/kuh-no-bowls/kuh-no-bowls.log diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fe35ae3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.env +__pycache__/ +*.pyc +.venv/ +venv/ +historical_data/ +*.log diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ddc22b3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.13-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY attractions_api_logger.py . + +ENTRYPOINT ["python", "-u", "attractions_api_logger.py"] diff --git a/attractions_api_logger.py b/attractions_api_logger.py index 9331c1b..c716f83 100755 --- a/attractions_api_logger.py +++ b/attractions_api_logger.py @@ -1,181 +1,188 @@ -#!/usr/bin/env python3 - -from datetime import datetime, date, timedelta -import json, requests, psycopg2, logging, sys -from psycopg2 import sql - -ATTRACTIONS_IO_ENDPOINT = "https://live-data.attractions.io/72f0ea9e-d196-508a-bee8-cce62c3228c7.json" -DISCORD_WEBHOOK = 'https://discord.com/api/webhooks/878651005015842827/o_KXXiPgor-DhZQ9vgL-MNOYb-RDmQCAbuYmgUpsoEOqu3XCfBdUM6rDmF1cMrZg12b0' - -db_params = { - 'dbname': 'gp0', - 'user': 'kuhnobowls', - 'password': 'saddog095', - 'host': '192.168.88.5', - 'port': '5432', - 'options': '-c search_path=knoebels' -} - -park_hours_query = """ -SELECT * -FROM knoebels."LZ_attractions_io_resort" -ORDER BY datestamp DESC -LIMIT 1 -""" - -def send_to_discord(message): - # The payload to send to the webhook - data = { - "content": message - } - - try: - # Send the POST request to the webhook URL - response = requests.post(DISCORD_WEBHOOK, json=data) - - # Check if the request was successful - if response.status_code == 204: - print("Message sent successfully.") - else: - print(f"Failed to send message. Status code: {response.status_code}, Response: {response.text}") - except requests.exceptions.RequestException as e: - print(f"Error sending message: {e}") - -def log_to_database(table_name, data): - with psycopg2.connect(**db_params) as conn: - with conn.cursor() as cursor: - columns = len(data[0]) - query = sql.SQL("INSERT INTO {table} VALUES ({values})").format( - table=sql.Identifier(table_name), - values=sql.SQL(', ').join(sql.Placeholder() * columns) - ) - - cursor.executemany(query, data) - conn.commit() - -def generic_query(q): - ret = [] - with psycopg2.connect(**db_params) as conn: - with conn.cursor() as cursor: - cursor.execute(q) - rows = cursor.fetchall() - col_names = [desc[0] for desc in cursor.description] - - for row in rows: - ret.append(row) - - return ret - -def fetch_webpage(url): - HEADERS = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'DNT': '1', - 'Sec-GPC': '1', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - } - try: - response = requests.get(url, headers=HEADERS) - - if response.status_code == 200: - return response.json() - else: - return f"Error: {response.status_code} - Unable to fetch webpage" - except Exception as e: - return f"Error: {str(e)} - Unable to fetch webpage" - -def pull_data_and_log(log_park_hours): - json_data = fetch_webpage(ATTRACTIONS_IO_ENDPOINT) - - logging.debug('Parsing park hours...') - try: - park_opens = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['start'] - park_closes = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['end'] - except KeyError as e: - logging.error('Unable to pull park hours. Is the park open today?') - return - - logging.debug('Checking that park hours from API are actually occuring today...') - if not datetime.today().date() == datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S").date(): - logging.error('API not returning park hours for today yet. Exiting...') - return - - resort_data = [[datetime.now(), json_data['entities']['Resort']['records'][0]['_id'], park_opens, park_closes]] - if log_park_hours: - logging.info('Inserting park hours into LZ_attractions_io_resort...') - log_to_database('LZ_attractions_io_resort', resort_data) - if not (datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S") - timedelta(minutes=5)) <= datetime.now() <= (datetime.strptime(park_closes, "%Y-%m-%d %H:%M:%S") + timedelta(minutes=5)): - logging.info('Park not open now. Exiting...') - return - - attraction_data = [] - - for attraction in json_data['entities']['Item']['records']: - try: - isOperational = attraction['IsOperational'] - except: - isOperational = None - try: - QueueTime = attraction['QueueTime'] - except: - QueueTime = None - try: - QueueStatusMessage = attraction['QueueStatusMessage'] - except: - QueueStatusMessage = None - try: - IsOpen = attraction['IsOpen'] - except: - IsOpen = None - try: - OpeningTimes = json.loads(attraction['OpeningTimes']) - start = OpeningTimes['start'] - end = OpeningTimes['end'] - except: - start = None - end = None - - row = [attraction['_id'], isOperational, QueueTime, QueueStatusMessage, IsOpen, start, end, datetime.now()] - attraction_data.append(row) - - queueline_data = [] - - for queue in json_data['entities']['QueueLine']['records']: - row = [datetime.now(), queue['_id'], queue['QueueTime']] - queueline_data.append(row) - - logging.debug('Inserting rows into LZ_attractions_io...') - log_to_database('LZ_attractions_io', attraction_data) - - logging.debug('Inserting rows into LZ_attractions_io_queuetimes...') - log_to_database('LZ_attractions_io_queuetimes', queueline_data) - - -logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, filename="/home/hoid/git_repos/kuh-no-bowls/knb.log",filemode="a") - -logging.info('--------------------------------------------------------------------------------') -logging.info('Fetching most recent park hours entry from DB...') - -park_hours_pull = generic_query(park_hours_query)[0] -logging.debug('%s - %s', park_hours_pull[2], park_hours_pull[3]) - -if park_hours_pull[2].date() == date.today(): - logging.info('Park hours for today already in DB') - park_opens = park_hours_pull[2] - park_closes = park_hours_pull[3] - if (park_opens - timedelta(minutes=5)) <= datetime.now() <= (park_closes + timedelta(minutes=5)): - logging.info('Park is open! Pulling data from API...') - pull_data_and_log(False) - else: - logging.info('Park not currently open. Exiting...') -else: - logging.info('Park hours for today not in DB') - pull_data_and_log(True) - - +#!/usr/bin/env python3 + +import os +from datetime import datetime, date, timedelta +import json, requests, psycopg2, logging, sys +from psycopg2 import sql + +ATTRACTIONS_IO_ENDPOINT = "https://live-data.attractions.io/72f0ea9e-d196-508a-bee8-cce62c3228c7.json" +DISCORD_WEBHOOK = os.environ.get('DISCORD_WEBHOOK', '').strip() + +DB_PASSWORD = os.environ.get('DB_PASSWORD') +if not DB_PASSWORD: + sys.stderr.write("ERROR: DB_PASSWORD env var is required\n") + sys.exit(1) + +db_params = { + 'dbname': 'gp0', + 'user': os.environ.get('DB_USER', 'kuhnobowls'), + 'password': DB_PASSWORD, + 'host': os.environ.get('DB_HOST', '192.168.88.9'), + 'port': os.environ.get('DB_PORT', '5432'), + 'options': '-c search_path=knoebels' +} + +LOG_PATH = os.environ.get('LOG_PATH', '/var/log/kuh-no-bowls.log') + +park_hours_query = """ +SELECT * +FROM knoebels."LZ_attractions_io_resort" +ORDER BY datestamp DESC +LIMIT 1 +""" + +def send_to_discord(message): + if not DISCORD_WEBHOOK: + # Discord notifications disabled — skip silently. + return + + data = { + "content": message + } + + try: + response = requests.post(DISCORD_WEBHOOK, json=data) + + if response.status_code == 204: + print("Message sent successfully.") + else: + print(f"Failed to send message. Status code: {response.status_code}, Response: {response.text}") + except requests.exceptions.RequestException as e: + print(f"Error sending message: {e}") + +def log_to_database(table_name, data): + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + columns = len(data[0]) + query = sql.SQL("INSERT INTO {table} VALUES ({values})").format( + table=sql.Identifier(table_name), + values=sql.SQL(', ').join(sql.Placeholder() * columns) + ) + + cursor.executemany(query, data) + conn.commit() + +def generic_query(q): + ret = [] + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + cursor.execute(q) + rows = cursor.fetchall() + col_names = [desc[0] for desc in cursor.description] + + for row in rows: + ret.append(row) + + return ret + +def fetch_webpage(url): + HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'DNT': '1', + 'Sec-GPC': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + } + try: + response = requests.get(url, headers=HEADERS) + + if response.status_code == 200: + return response.json() + else: + return f"Error: {response.status_code} - Unable to fetch webpage" + except Exception as e: + return f"Error: {str(e)} - Unable to fetch webpage" + +def pull_data_and_log(log_park_hours): + json_data = fetch_webpage(ATTRACTIONS_IO_ENDPOINT) + + logging.debug('Parsing park hours...') + try: + park_opens = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['start'] + park_closes = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['end'] + except KeyError as e: + logging.error('Unable to pull park hours. Is the park open today?') + return + + logging.debug('Checking that park hours from API are actually occuring today...') + if not datetime.today().date() == datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S").date(): + logging.error('API not returning park hours for today yet. Exiting...') + return + + resort_data = [[datetime.now(), json_data['entities']['Resort']['records'][0]['_id'], park_opens, park_closes]] + if log_park_hours: + logging.info('Inserting park hours into LZ_attractions_io_resort...') + log_to_database('LZ_attractions_io_resort', resort_data) + if not (datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S") - timedelta(minutes=5)) <= datetime.now() <= (datetime.strptime(park_closes, "%Y-%m-%d %H:%M:%S") + timedelta(minutes=5)): + logging.info('Park not open now. Exiting...') + return + + attraction_data = [] + + for attraction in json_data['entities']['Item']['records']: + try: + isOperational = attraction['IsOperational'] + except: + isOperational = None + try: + QueueTime = attraction['QueueTime'] + except: + QueueTime = None + try: + QueueStatusMessage = attraction['QueueStatusMessage'] + except: + QueueStatusMessage = None + try: + IsOpen = attraction['IsOpen'] + except: + IsOpen = None + try: + OpeningTimes = json.loads(attraction['OpeningTimes']) + start = OpeningTimes['start'] + end = OpeningTimes['end'] + except: + start = None + end = None + + row = [attraction['_id'], isOperational, QueueTime, QueueStatusMessage, IsOpen, start, end, datetime.now()] + attraction_data.append(row) + + queueline_data = [] + + for queue in json_data['entities']['QueueLine']['records']: + row = [datetime.now(), queue['_id'], queue['QueueTime']] + queueline_data.append(row) + + logging.debug('Inserting rows into LZ_attractions_io...') + log_to_database('LZ_attractions_io', attraction_data) + + logging.debug('Inserting rows into LZ_attractions_io_queuetimes...') + log_to_database('LZ_attractions_io_queuetimes', queueline_data) + + +logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, filename=LOG_PATH, filemode="a") + +logging.info('--------------------------------------------------------------------------------') +logging.info('Fetching most recent park hours entry from DB...') + +park_hours_pull = generic_query(park_hours_query)[0] +logging.debug('%s - %s', park_hours_pull[2], park_hours_pull[3]) + +if park_hours_pull[2].date() == date.today(): + logging.info('Park hours for today already in DB') + park_opens = park_hours_pull[2] + park_closes = park_hours_pull[3] + if (park_opens - timedelta(minutes=5)) <= datetime.now() <= (park_closes + timedelta(minutes=5)): + logging.info('Park is open! Pulling data from API...') + pull_data_and_log(False) + else: + logging.info('Park not currently open. Exiting...') +else: + logging.info('Park hours for today not in DB') + pull_data_and_log(True) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..43a2980 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +services: + scraper: + build: . + image: kuh-no-bowls:latest + container_name: kuh-no-bowls-scraper + env_file: .env + environment: + LOG_PATH: /var/log/kuh-no-bowls/kuh-no-bowls.log + TZ: America/New_York + volumes: + - knb_logs:/var/log/kuh-no-bowls + +volumes: + knb_logs: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..74060fa --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +psycopg2-binary