commit 0ce13cf8a4a32941ba82dbc84a0f38c14a60a081 Author: Wesley Ray Date: Mon May 20 12:20:38 2024 -0400 first commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/attractions_api_logger.py b/attractions_api_logger.py new file mode 100755 index 0000000..060f4b6 --- /dev/null +++ b/attractions_api_logger.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 + +from datetime import datetime, date +import json, requests, psycopg2, logging +from psycopg2 import sql + +ATTRACTIONS_IO_ENDPOINT = "https://live-data.attractions.io/72f0ea9e-d196-508a-bee8-cce62c3228c7.json" +DISCORD_WEBHOOK = 'https://discord.com/api/webhooks/878651005015842827/o_KXXiPgor-DhZQ9vgL-MNOYb-RDmQCAbuYmgUpsoEOqu3XCfBdUM6rDmF1cMrZg12b0' + +db_params = { + 'dbname': 'gp0', + 'user': 'kuhnobowls', + 'password': 'saddog095', + 'host': '192.168.88.5', + 'port': '5432', + 'options': '-c search_path=knoebels' +} + +park_hours_query = """ +SELECT * +FROM knoebels."LZ_attractions_io_resort" +ORDER BY datestamp DESC +LIMIT 1 +""" + +def send_to_discord(message): + # The payload to send to the webhook + data = { + "content": message + } + + try: + # Send the POST request to the webhook URL + response = requests.post(DISCORD_WEBHOOK, json=data) + + # Check if the request was successful + if response.status_code == 204: + print("Message sent successfully.") + else: + print(f"Failed to send message. Status code: {response.status_code}, Response: {response.text}") + except requests.exceptions.RequestException as e: + print(f"Error sending message: {e}") + +def log_to_database(table_name, data): + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + columns = len(data[0]) + query = sql.SQL("INSERT INTO {table} VALUES ({values})").format( + table=sql.Identifier(table_name), + values=sql.SQL(', ').join(sql.Placeholder() * columns) + ) + + cursor.executemany(query, data) + conn.commit() + +def generic_query(q): + ret = [] + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + cursor.execute(q) + rows = cursor.fetchall() + col_names = [desc[0] for desc in cursor.description] + + for row in rows: + ret.append(row) + + return ret + +def fetch_webpage(url): + HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'DNT': '1', + 'Sec-GPC': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + } + try: + response = requests.get(url, headers=HEADERS) + + if response.status_code == 200: + return response.json() + else: + return f"Error: {response.status_code} - Unable to fetch webpage" + except Exception as e: + return f"Error: {str(e)} - Unable to fetch webpage" + +def pull_data_and_log(log_park_hours): + json_data = fetch_webpage(ATTRACTIONS_IO_ENDPOINT) + + try: + park_opens = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['start'] + park_closes = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['end'] + except KeyError as e: + logging.error('Unable to pull park hours. Is the park open today?') + return + + resort_data = [[datetime.now(), json_data['entities']['Resort']['records'][0]['_id'], park_opens, park_closes]] + if log_park_hours: + log_to_database('LZ_attractions_io_resort', resort_data) + + attraction_data = [] + + for attraction in json_data['entities']['Item']['records']: + try: + isOperational = attraction['IsOperational'] + except: + isOperational = None + try: + QueueTime = attraction['QueueTime'] + except: + QueueTime = None + try: + QueueStatusMessage = attraction['QueueStatusMessage'] + except: + QueueStatusMessage = None + try: + IsOpen = attraction['IsOpen'] + except: + IsOpen = None + try: + OpeningTimes = json.loads(attraction['OpeningTimes']) + start = OpeningTimes['start'] + end = OpeningTimes['end'] + except: + start = None + end = None + + row = [attraction['_id'], isOperational, QueueTime, QueueStatusMessage, IsOpen, start, end, datetime.now()] + attraction_data.append(row) + + queueline_data = [] + + for queue in json_data['entities']['QueueLine']['records']: + row = [datetime.now(), queue['_id'], queue['QueueTime']] + queueline_data.append(row) + + log_to_database('LZ_attractions_io', attraction_data) + log_to_database('LZ_attractions_io_queuetimes', queueline_data)# LZ_attractions_io_queuetimes + + +logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, filename="/home/hoid/git_repos/kuh-no-bowls/knb.log",filemode="a") + +logging.info('--------------------------------------------------------------------------------') +logging.info('Fetching most recent park hours entry from DB...') + +park_hours_pull = generic_query(park_hours_query)[0] +logging.debug('%s - %s', park_hours_pull[2], park_hours_pull[3]) + +if park_hours_pull[2].date() == date.today(): + logging.info('Park hours for today already in DB') + park_opens = park_hours_pull[2] + park_closes = park_hours_pull[3] + if park_opens <= datetime.now() <= park_closes: + pull_data_and_log(False) + else: + logging.info('Park not currently open. Exiting...') +else: + logging.info('Park hours for today not in DB') + pull_data_and_log(True) + + diff --git a/import_poi_data.py b/import_poi_data.py new file mode 100755 index 0000000..e619cff --- /dev/null +++ b/import_poi_data.py @@ -0,0 +1,118 @@ +''' +Importing POI data from attractions.io api +''' +#!/usr/bin/env python3 + +from datetime import datetime, date +import json, requests, psycopg2 +from psycopg2 import sql + +db_params = { + 'dbname': 'gp0', + 'user': 'kuhnobowls', + 'password': 'saddog095', + 'host': '192.168.88.5', + 'port': '5432', + 'options': '-c search_path=knoebels' +} + +def log_to_database(table_name, data): + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + columns = len(data[0]) + query = sql.SQL("INSERT INTO {table} VALUES ({values})").format( + table=sql.Identifier(table_name), + values=sql.SQL(', ').join(sql.Placeholder() * columns) + ) + + cursor.executemany(query, data) + conn.commit() + +def generic_query(q): + ret = [] + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + cursor.execute(q) + rows = cursor.fetchall() + col_names = [desc[0] for desc in cursor.description] + + for row in rows: + ret.append(row) + + return ret + +data = None + +with open('records.json', 'r', encoding='utf-8') as file: + data = json.load(file) + +rows = [] + +for poi in data['Item']: + aoi_id = poi['_id'] + Name = poi['Name'] + AbbreviatedName = poi['AbbreviatedName'] + Summary = poi['Summary'] + Keywords = poi['Keywords'] + DefaultImage = poi['DefaultImage'] + Location = poi['Location'] + Featured = poi['Featured'] + WayfindingEnabled = poi['WayfindingEnabled'] + VisibleOnMap = poi['VisibleOnMap'] + Category = poi['Category'] + Parent = poi['Parent'] + + try: + MenuURL = poi['MenuURL'] + except: + MenuURL = None + + try: + MinimumHeightRequirement = poi['MinimumHeightRequirement'] + except: + MinimumHeightRequirement = None + + try: + MinimumUnaccompaniedHeightRequirement = poi['MinimumUnaccompaniedHeightRequirement'] + except: + MinimumUnaccompaniedHeightRequirement = None + + try: + MaximumHeightRequirement = poi['MaximumHeightRequirement'] + except: + MaximumHeightRequirement = None + + try: + MinimumAgeRequirement = poi['MinimumAgeRequirement'] + except: + MinimumAgeRequirement = None + + try: + MinimumUnaccompaniedAgeRequirement = poi['MinimumUnaccompaniedAgeRequirement'] + except: + MinimumUnaccompaniedAgeRequirement = None + + try: + MaximumAgeRequirement = poi['MaximumAgeRequirement'] + except: + MaximumAgeRequirement = None + + try: + RestrictionSummary = poi['RestrictionSummary'] + except: + RestrictionSummary = None + + row = [aoi_id, Name, AbbreviatedName, Summary, Keywords, DefaultImage, Location, Featured, WayfindingEnabled, VisibleOnMap, Category, Parent, MenuURL, MinimumHeightRequirement, MinimumUnaccompaniedHeightRequirement, MaximumHeightRequirement, MinimumAgeRequirement, MinimumUnaccompaniedAgeRequirement, MaximumAgeRequirement, RestrictionSummary] + rows.append(row) + +cats = [] + +for cat in data['Category']: + aoi_id = cat['_id'] + Name = cat['Name'] + Parent = cat['Parent'] + row = [aoi_id, Name, Parent] + cats.append(row) + +log_to_database('LZ_attractions_io_categories', cats) +#log_to_database('LZ_attractions_io_poi', rows) diff --git a/queries/dm_knb_live_data.sql b/queries/dm_knb_live_data.sql new file mode 100755 index 0000000..c61acaf --- /dev/null +++ b/queries/dm_knb_live_data.sql @@ -0,0 +1,49 @@ +--Using CTE to remove rows where nothing changed +WITH livedata as ( +SELECT _id, + is_operational, + queue_time, + queue_status_message, + is_open, + open_time, + close_time, + MIN(time_stamp) as time_stamp +FROM knoebels."LZ_attractions_io" as livedata +GROUP BY _id, is_operational, queue_time, queue_status_message, is_open, open_time, close_time +) +SELECT livedata.time_stamp + ,livedata._id + ,poi.name + ,prices.price + ,is_operational + ,queue_time + ,queue_status_message + ,is_open + ,open_time + ,close_time + ,hand_stamp_included + ,bargain_night_included + ,capacity + ,duration + ,summary + ,keywords + ,default_image + ,location + ,featured + ,wayfinding_enabled + ,visible_on_map + ,category + ,ceil(minimum_height_requirement*39.37) AS minimum_height_requirement + ,ceil(minimum_unaccompanied_height_requirement*39.37) AS minimum_unaccompanied_height_requirement + ,ceil(maximum_height_requirement*39.37) AS maximum_height_requirement + ,minimum_age_requirement + ,minimum_unaccompanied_age_requirement + ,maximum_age_requirement + ,restriction_summary +--FROM knoebels."LZ_attractions_io" AS livedata +FROM livedata +JOIN knoebels."LZ_attractions_io_poi" AS poi ON livedata._id = poi._id +JOIN knoebels.ride_master rm ON livedata._id = rm.attractions_dot_io_id +JOIN (SELECT row_number() OVER (PARTITION BY ride ORDER BY time_stamp DESC) AS row_number, * FROM knoebels.ride_data_in) prices ON prices.ride = rm.ride and prices.row_number = 1 +--ORDER BY name, time_stamp +--where name = 'Sklooosh' diff --git a/ride_price_scrape.py b/ride_price_scrape.py new file mode 100755 index 0000000..839d860 --- /dev/null +++ b/ride_price_scrape.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +from datetime import datetime +import json, re, requests, psycopg2 + +RIDE_PAGE = "https://www.knoebels.com/ride" +PATTERN = r'ride-list-item" data-ride="(?P[\S\s]+?)"[\S\s]+?rounded">\$(?P\d\.\d{2})' + +db_params = { + 'dbname': 'gp0', + 'user': 'kuhnobowls', + 'password': 'saddog095', + 'host': '192.168.88.5', + 'port': '5432' +} + +insert_query = """ + INSERT INTO knoebels.ride_data_in (time_stamp, ride, price) + VALUES (%s, %s, %s) +""" + +def generic_query(q): + ret = {} + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + cursor.execute(q) + rows = cursor.fetchall() + col_names = [desc[0] for desc in cursor.description] + + for row in rows: + ret[row[1]] = row[2] + + return ret + +def log_to_database(ride, price): + data = [datetime.now(), ride, price] + + # Connect to the database using a context manager + with psycopg2.connect(**db_params) as conn: + with conn.cursor() as cursor: + cursor.execute(insert_query, data) + conn.commit() + +def fetch_webpage(url): + HEADERS = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.5', + 'DNT': '1', + 'Sec-GPC': '1', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + } + try: + response = requests.get(url, headers=HEADERS) + + if response.status_code == 200: + return response.text + else: + return f"Error: {response.status_code} - Unable to fetch webpage" + except Exception as e: + return f"Error: {str(e)} - Unable to fetch webpage" + +current_values = generic_query("SELECT * FROM knoebels.ride_data_in") + +html = fetch_webpage(RIDE_PAGE) # fetch page +matches = re.finditer(PATTERN, html) # run regex + +data = [] +for match in matches: + data.append({'ride':match.group('ride').replace(' ', '_').lower(), 'price':match.group('price')}) # regex grabs number of bytes, cast to integer and divide by 1,073,741,824 + +for r in data: + current_price = float(r["price"]) + try: + old_price = float(current_values[r["ride"]]) + except: + old_price = -1.0 + if current_price != old_price: + print("New price for: ", r["ride"], current_price, ", Old Price: ", old_price) + log_to_database(r["ride"], r["price"])