wes/kuh-no-bowls
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Containerize scraper and externalize credentials

Browse Source 
Move DB host/user/password, Discord webhook, and log path to env vars
so the script can be deployed via Docker compose. Discord notifications
are now skipped silently when DISCORD_WEBHOOK is unset (the old
hardcoded webhook is removed pending rotation). Adds Dockerfile,
docker-compose.yml, requirements.txt, .env.example, and .gitignore for
the cyrion deployment.
This commit is contained in:
wes
2026-05-06 23:20:12 -04:00
parent b9864b36b4
commit 10bdee8e1f
6 changed files with 237 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+16
View File
@@ -0,0 +1,16 @@
# Knoebels scraper environment configuration.
# Copy to .env on the deploy host and fill in real values.
# Never commit the real .env.
# PostgreSQL connection (phlegethon by default)
DB_HOST=192.168.88.9
DB_PORT=5432
DB_USER=kuhnobowls
DB_PASSWORD=
# Optional Discord webhook for notifications.
# Leave empty/unset to skip Discord calls entirely.
DISCORD_WEBHOOK=
# Log path inside the container. Mapped to the knb_logs named volume by compose.
LOG_PATH=/var/log/kuh-no-bowls/kuh-no-bowls.log
.gitignore
+7
View File
@@ -0,0 +1,7 @@
.env
__pycache__/
*.pyc
.venv/
venv/
historical_data/
*.log
Dockerfile
+10
View File
@@ -0,0 +1,10 @@
FROM python:3.13-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY attractions_api_logger.py .
ENTRYPOINT ["python", "-u", "attractions_api_logger.py"]
attractions_api_logger.py
+188 -181
View File
@@ -1,181 +1,188 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from datetime import datetime, date, timedelta import os
import json, requests, psycopg2, logging, sys from datetime import datetime, date, timedelta
from psycopg2 import sql import json, requests, psycopg2, logging, sys
from psycopg2 import sql
ATTRACTIONS_IO_ENDPOINT = "https://live-data.attractions.io/72f0ea9e-d196-508a-bee8-cce62c3228c7.json"
DISCORD_WEBHOOK = 'https://discord.com/api/webhooks/878651005015842827/o_KXXiPgor-DhZQ9vgL-MNOYb-RDmQCAbuYmgUpsoEOqu3XCfBdUM6rDmF1cMrZg12b0' ATTRACTIONS_IO_ENDPOINT = "https://live-data.attractions.io/72f0ea9e-d196-508a-bee8-cce62c3228c7.json"
DISCORD_WEBHOOK = os.environ.get('DISCORD_WEBHOOK', '').strip()
db_params = {
'dbname': 'gp0', DB_PASSWORD = os.environ.get('DB_PASSWORD')
'user': 'kuhnobowls', if not DB_PASSWORD:
'password': 'saddog095', sys.stderr.write("ERROR: DB_PASSWORD env var is required\n")
'host': '192.168.88.5', sys.exit(1)
'port': '5432',
'options': '-c search_path=knoebels' db_params = {
} 'dbname': 'gp0',
'user': os.environ.get('DB_USER', 'kuhnobowls'),
park_hours_query = """ 'password': DB_PASSWORD,
SELECT * 'host': os.environ.get('DB_HOST', '192.168.88.9'),
FROM knoebels."LZ_attractions_io_resort" 'port': os.environ.get('DB_PORT', '5432'),
ORDER BY datestamp DESC 'options': '-c search_path=knoebels'
LIMIT 1 }
"""
LOG_PATH = os.environ.get('LOG_PATH', '/var/log/kuh-no-bowls.log')
def send_to_discord(message):
# The payload to send to the webhook park_hours_query = """
data = { SELECT *
"content": message FROM knoebels."LZ_attractions_io_resort"
} ORDER BY datestamp DESC
LIMIT 1
try: """
# Send the POST request to the webhook URL
response = requests.post(DISCORD_WEBHOOK, json=data) def send_to_discord(message):
if not DISCORD_WEBHOOK:
# Check if the request was successful # Discord notifications disabled — skip silently.
if response.status_code == 204: return
print("Message sent successfully.")
else: data = {
print(f"Failed to send message. Status code: {response.status_code}, Response: {response.text}") "content": message
except requests.exceptions.RequestException as e: }
print(f"Error sending message: {e}")
try:
def log_to_database(table_name, data): response = requests.post(DISCORD_WEBHOOK, json=data)
with psycopg2.connect(**db_params) as conn:
with conn.cursor() as cursor: if response.status_code == 204:
columns = len(data[0]) print("Message sent successfully.")
query = sql.SQL("INSERT INTO {table} VALUES ({values})").format( else:
table=sql.Identifier(table_name), print(f"Failed to send message. Status code: {response.status_code}, Response: {response.text}")
values=sql.SQL(', ').join(sql.Placeholder() * columns) except requests.exceptions.RequestException as e:
) print(f"Error sending message: {e}")
cursor.executemany(query, data) def log_to_database(table_name, data):
conn.commit() with psycopg2.connect(**db_params) as conn:
with conn.cursor() as cursor:
def generic_query(q): columns = len(data[0])
ret = [] query = sql.SQL("INSERT INTO {table} VALUES ({values})").format(
with psycopg2.connect(**db_params) as conn: table=sql.Identifier(table_name),
with conn.cursor() as cursor: values=sql.SQL(', ').join(sql.Placeholder() * columns)
cursor.execute(q) )
rows = cursor.fetchall()
col_names = [desc[0] for desc in cursor.description] cursor.executemany(query, data)
conn.commit()
for row in rows:
ret.append(row) def generic_query(q):
ret = []
return ret with psycopg2.connect(**db_params) as conn:
with conn.cursor() as cursor:
def fetch_webpage(url): cursor.execute(q)
HEADERS = { rows = cursor.fetchall()
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0', col_names = [desc[0] for desc in cursor.description]
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5', for row in rows:
'DNT': '1', ret.append(row)
'Sec-GPC': '1',
'Connection': 'keep-alive', return ret
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document', def fetch_webpage(url):
'Sec-Fetch-Mode': 'navigate', HEADERS = {
'Sec-Fetch-Site': 'none', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0',
'Sec-Fetch-User': '?1', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
} 'Accept-Language': 'en-US,en;q=0.5',
try: 'DNT': '1',
response = requests.get(url, headers=HEADERS) 'Sec-GPC': '1',
'Connection': 'keep-alive',
if response.status_code == 200: 'Upgrade-Insecure-Requests': '1',
return response.json() 'Sec-Fetch-Dest': 'document',
else: 'Sec-Fetch-Mode': 'navigate',
return f"Error: {response.status_code} - Unable to fetch webpage" 'Sec-Fetch-Site': 'none',
except Exception as e: 'Sec-Fetch-User': '?1',
return f"Error: {str(e)} - Unable to fetch webpage" }
try:
def pull_data_and_log(log_park_hours): response = requests.get(url, headers=HEADERS)
json_data = fetch_webpage(ATTRACTIONS_IO_ENDPOINT)
if response.status_code == 200:
logging.debug('Parsing park hours...') return response.json()
try: else:
park_opens = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['start'] return f"Error: {response.status_code} - Unable to fetch webpage"
park_closes = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['end'] except Exception as e:
except KeyError as e: return f"Error: {str(e)} - Unable to fetch webpage"
logging.error('Unable to pull park hours. Is the park open today?')
return def pull_data_and_log(log_park_hours):
json_data = fetch_webpage(ATTRACTIONS_IO_ENDPOINT)
logging.debug('Checking that park hours from API are actually occuring today...')
if not datetime.today().date() == datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S").date(): logging.debug('Parsing park hours...')
logging.error('API not returning park hours for today yet. Exiting...') try:
return park_opens = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['start']
park_closes = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['end']
resort_data = [[datetime.now(), json_data['entities']['Resort']['records'][0]['_id'], park_opens, park_closes]] except KeyError as e:
if log_park_hours: logging.error('Unable to pull park hours. Is the park open today?')
logging.info('Inserting park hours into LZ_attractions_io_resort...') return
log_to_database('LZ_attractions_io_resort', resort_data)
if not (datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S") - timedelta(minutes=5)) <= datetime.now() <= (datetime.strptime(park_closes, "%Y-%m-%d %H:%M:%S") + timedelta(minutes=5)): logging.debug('Checking that park hours from API are actually occuring today...')
logging.info('Park not open now. Exiting...') if not datetime.today().date() == datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S").date():
return logging.error('API not returning park hours for today yet. Exiting...')
return
attraction_data = []
resort_data = [[datetime.now(), json_data['entities']['Resort']['records'][0]['_id'], park_opens, park_closes]]
for attraction in json_data['entities']['Item']['records']: if log_park_hours:
try: logging.info('Inserting park hours into LZ_attractions_io_resort...')
isOperational = attraction['IsOperational'] log_to_database('LZ_attractions_io_resort', resort_data)
except: if not (datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S") - timedelta(minutes=5)) <= datetime.now() <= (datetime.strptime(park_closes, "%Y-%m-%d %H:%M:%S") + timedelta(minutes=5)):
isOperational = None logging.info('Park not open now. Exiting...')
try: return
QueueTime = attraction['QueueTime']
except: attraction_data = []
QueueTime = None
try: for attraction in json_data['entities']['Item']['records']:
QueueStatusMessage = attraction['QueueStatusMessage'] try:
except: isOperational = attraction['IsOperational']
QueueStatusMessage = None except:
try: isOperational = None
IsOpen = attraction['IsOpen'] try:
except: QueueTime = attraction['QueueTime']
IsOpen = None except:
try: QueueTime = None
OpeningTimes = json.loads(attraction['OpeningTimes']) try:
start = OpeningTimes['start'] QueueStatusMessage = attraction['QueueStatusMessage']
end = OpeningTimes['end'] except:
except: QueueStatusMessage = None
start = None try:
end = None IsOpen = attraction['IsOpen']
except:
row = [attraction['_id'], isOperational, QueueTime, QueueStatusMessage, IsOpen, start, end, datetime.now()] IsOpen = None
attraction_data.append(row) try:
OpeningTimes = json.loads(attraction['OpeningTimes'])
queueline_data = [] start = OpeningTimes['start']
end = OpeningTimes['end']
for queue in json_data['entities']['QueueLine']['records']: except:
row = [datetime.now(), queue['_id'], queue['QueueTime']] start = None
queueline_data.append(row) end = None
logging.debug('Inserting rows into LZ_attractions_io...') row = [attraction['_id'], isOperational, QueueTime, QueueStatusMessage, IsOpen, start, end, datetime.now()]
log_to_database('LZ_attractions_io', attraction_data) attraction_data.append(row)
logging.debug('Inserting rows into LZ_attractions_io_queuetimes...') queueline_data = []
log_to_database('LZ_attractions_io_queuetimes', queueline_data)
for queue in json_data['entities']['QueueLine']['records']:
row = [datetime.now(), queue['_id'], queue['QueueTime']]
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, filename="/home/hoid/git_repos/kuh-no-bowls/knb.log",filemode="a") queueline_data.append(row)
logging.info('--------------------------------------------------------------------------------') logging.debug('Inserting rows into LZ_attractions_io...')
logging.info('Fetching most recent park hours entry from DB...') log_to_database('LZ_attractions_io', attraction_data)
park_hours_pull = generic_query(park_hours_query)[0] logging.debug('Inserting rows into LZ_attractions_io_queuetimes...')
logging.debug('%s - %s', park_hours_pull[2], park_hours_pull[3]) log_to_database('LZ_attractions_io_queuetimes', queueline_data)
if park_hours_pull[2].date() == date.today():
logging.info('Park hours for today already in DB') logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, filename=LOG_PATH, filemode="a")
park_opens = park_hours_pull[2]
park_closes = park_hours_pull[3] logging.info('--------------------------------------------------------------------------------')
if (park_opens - timedelta(minutes=5)) <= datetime.now() <= (park_closes + timedelta(minutes=5)): logging.info('Fetching most recent park hours entry from DB...')
logging.info('Park is open! Pulling data from API...')
pull_data_and_log(False) park_hours_pull = generic_query(park_hours_query)[0]
else: logging.debug('%s - %s', park_hours_pull[2], park_hours_pull[3])
logging.info('Park not currently open. Exiting...')
else: if park_hours_pull[2].date() == date.today():
logging.info('Park hours for today not in DB') logging.info('Park hours for today already in DB')
pull_data_and_log(True) park_opens = park_hours_pull[2]
park_closes = park_hours_pull[3]
if (park_opens - timedelta(minutes=5)) <= datetime.now() <= (park_closes + timedelta(minutes=5)):
logging.info('Park is open! Pulling data from API...')
pull_data_and_log(False)
else:
logging.info('Park not currently open. Exiting...')
else:
logging.info('Park hours for today not in DB')
pull_data_and_log(True)
docker-compose.yml
+14
View File
@@ -0,0 +1,14 @@
services:
scraper:
build: .
image: kuh-no-bowls:latest
container_name: kuh-no-bowls-scraper
env_file: .env
environment:
LOG_PATH: /var/log/kuh-no-bowls/kuh-no-bowls.log
TZ: America/New_York
volumes:
- knb_logs:/var/log/kuh-no-bowls
volumes:
knb_logs:
requirements.txt
+2
View File
@@ -0,0 +1,2 @@
requests
psycopg2-binary