catch-up commit + csv with 2024 data

attractions_api_logger.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-from datetime import datetime, date
+from datetime import datetime, date, timedelta
 import json, requests, psycopg2, logging, sys
 from psycopg2 import sql
 
@@ -93,6 +93,7 @@ def fetch_webpage(url):
 def pull_data_and_log(log_park_hours):
     json_data = fetch_webpage(ATTRACTIONS_IO_ENDPOINT)
 
+    logging.debug('Parsing park hours...')
     try:
         park_opens = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['start']
         park_closes = json.loads(json_data['entities']['Resort']['records'][0]['OpeningTimes'])['end']
@@ -100,9 +101,18 @@ def pull_data_and_log(log_park_hours):
         logging.error('Unable to pull park hours. Is the park open today?')
         return
 
+    logging.debug('Checking that park hours from API are actually occuring today...')
+    if not datetime.today().date() == datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S").date():
+        logging.error('API not returning park hours for today yet. Exiting...')
+        return
+
     resort_data = [[datetime.now(), json_data['entities']['Resort']['records'][0]['_id'], park_opens, park_closes]]
     if log_park_hours:
+        logging.info('Inserting park hours into LZ_attractions_io_resort...')
         log_to_database('LZ_attractions_io_resort', resort_data)
+        if not (datetime.strptime(park_opens, "%Y-%m-%d %H:%M:%S") - timedelta(minutes=5)) <= datetime.now() <= (datetime.strptime(park_closes, "%Y-%m-%d %H:%M:%S") + timedelta(minutes=5)):
+            logging.info('Park not open now. Exiting...')
+            return
 
     attraction_data = []
 
@@ -140,8 +150,11 @@ def pull_data_and_log(log_park_hours):
         row = [datetime.now(), queue['_id'], queue['QueueTime']]
         queueline_data.append(row)
 
+    logging.debug('Inserting rows into LZ_attractions_io...')
     log_to_database('LZ_attractions_io', attraction_data)
-    log_to_database('LZ_attractions_io_queuetimes', queueline_data)# LZ_attractions_io_queuetimes
+    
+    logging.debug('Inserting rows into LZ_attractions_io_queuetimes...')
+    log_to_database('LZ_attractions_io_queuetimes', queueline_data)
 
 
 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.DEBUG, filename="/home/hoid/git_repos/kuh-no-bowls/knb.log",filemode="a")
@@ -152,20 +165,12 @@ logging.info('Fetching most recent park hours entry from DB...')
 park_hours_pull = generic_query(park_hours_query)[0]
 logging.debug('%s - %s', park_hours_pull[2], park_hours_pull[3])
 
-try:
-    closed = park_hours_pull[4]
-except:
-    closed = False
-
-if park_hours_pull[4]:
-    logging.info('Park closed today. Exiting...')
-    sys.exit()
-
 if park_hours_pull[2].date() == date.today():
     logging.info('Park hours for today already in DB')
     park_opens = park_hours_pull[2]
     park_closes = park_hours_pull[3]
-    if park_opens <= datetime.now() <= park_closes:
+    if (park_opens - timedelta(minutes=5)) <= datetime.now() <= (park_closes + timedelta(minutes=5)):
+        logging.info('Park is open! Pulling data from API...')
         pull_data_and_log(False)
     else:
         logging.info('Park not currently open. Exiting...')

queries/dm_knb_latest_ride_info.sql

@@ -28,6 +28,8 @@
     poi.keywords,
     poi.default_image,
     poi.location,
+	poi.location[0] as lattitude,
+	poi.location[1] as longitude,
     poi.featured,
     poi.wayfinding_enabled,
     poi.visible_on_map,

queries/dm_knb_live_data.sql

@@ -1,16 +1,16 @@
 --Using CTE to remove rows where nothing changed
-WITH livedata as (
+--WITH livedata as (
-SELECT  _id,
+--SELECT  _id,
-		is_operational,
+--		is_operational,
-		queue_time,
+--		queue_time,
-		queue_status_message,
+--		queue_status_message,
-		is_open,
+--		is_open,
-		open_time,
+--		open_time,
-		close_time,
+--		close_time,
-		MIN(time_stamp) as time_stamp
+--		MIN(time_stamp) as time_stamp
-FROM knoebels."LZ_attractions_io" as livedata
+--FROM knoebels."LZ_attractions_io" as livedata
-GROUP BY _id, is_operational, queue_time, queue_status_message, is_open, open_time, close_time
+--GROUP BY _id, is_operational, queue_time, queue_status_message, is_open, open_time, close_time
-)
+--)
 SELECT 	livedata.time_stamp
 		,livedata._id
 	 	,poi.name
@@ -29,6 +29,8 @@ SELECT 	livedata.time_stamp
 	 	,keywords
 	 	,default_image
 	 	,location
+		,location[0] as lattitude
+		,location[1] as longitude
 	 	,featured
 	 	,wayfinding_enabled
 	 	,visible_on_map
@@ -40,13 +42,13 @@ SELECT 	livedata.time_stamp
 	 	,minimum_unaccompanied_age_requirement
 	 	,maximum_age_requirement
 	 	,restriction_summary
---FROM knoebels."LZ_attractions_io" AS livedata
+FROM knoebels."LZ_attractions_io" AS livedata
-FROM livedata
+--FROM livedata
 --This POI data comes from a massive JSON file hosted by attractions.io the app downloads it on first launch. Not sure how often it's updated or only as-needed
 JOIN knoebels."LZ_attractions_io_poi" AS poi ON livedata._id = poi._id
 -- This is my own "master" file for ride data, for data points not served by the attractions.io API. Handstamp inclusions/exclusions, capacity, duration, etc.
 JOIN knoebels.ride_master rm ON livedata._id = rm.attractions_dot_io_id
 -- fetch most recent price data, scraped from webpage, this can differ from the POI data used by the app
 LEFT JOIN (SELECT row_number() OVER (PARTITION BY ride ORDER BY time_stamp DESC) AS row_number, * FROM knoebels.ride_data_in) prices ON prices.ride = rm.ride and prices.row_number = 1
---ORDER BY name, time_stamp
 --where name = 'Sklooosh'
+--ORDER BY name, time_stamp