diff --git a/import_poi_data.py b/import_poi_data.py index e619cff..01f4b95 100755 --- a/import_poi_data.py +++ b/import_poi_data.py @@ -1,118 +1,166 @@ -''' -Importing POI data from attractions.io api -''' -#!/usr/bin/env python3 - -from datetime import datetime, date -import json, requests, psycopg2 -from psycopg2 import sql - -db_params = { - 'dbname': 'gp0', - 'user': 'kuhnobowls', - 'password': 'saddog095', - 'host': '192.168.88.5', - 'port': '5432', - 'options': '-c search_path=knoebels' -} - -def log_to_database(table_name, data): - with psycopg2.connect(**db_params) as conn: - with conn.cursor() as cursor: - columns = len(data[0]) - query = sql.SQL("INSERT INTO {table} VALUES ({values})").format( - table=sql.Identifier(table_name), - values=sql.SQL(', ').join(sql.Placeholder() * columns) - ) - - cursor.executemany(query, data) - conn.commit() - -def generic_query(q): - ret = [] - with psycopg2.connect(**db_params) as conn: - with conn.cursor() as cursor: - cursor.execute(q) - rows = cursor.fetchall() - col_names = [desc[0] for desc in cursor.description] - - for row in rows: - ret.append(row) - - return ret - -data = None - -with open('records.json', 'r', encoding='utf-8') as file: - data = json.load(file) - -rows = [] - -for poi in data['Item']: - aoi_id = poi['_id'] - Name = poi['Name'] - AbbreviatedName = poi['AbbreviatedName'] - Summary = poi['Summary'] - Keywords = poi['Keywords'] - DefaultImage = poi['DefaultImage'] - Location = poi['Location'] - Featured = poi['Featured'] - WayfindingEnabled = poi['WayfindingEnabled'] - VisibleOnMap = poi['VisibleOnMap'] - Category = poi['Category'] - Parent = poi['Parent'] - - try: - MenuURL = poi['MenuURL'] - except: - MenuURL = None - - try: - MinimumHeightRequirement = poi['MinimumHeightRequirement'] - except: - MinimumHeightRequirement = None - - try: - MinimumUnaccompaniedHeightRequirement = poi['MinimumUnaccompaniedHeightRequirement'] - except: - MinimumUnaccompaniedHeightRequirement = None - - try: - MaximumHeightRequirement = poi['MaximumHeightRequirement'] - except: - MaximumHeightRequirement = None - - try: - MinimumAgeRequirement = poi['MinimumAgeRequirement'] - except: - MinimumAgeRequirement = None - - try: - MinimumUnaccompaniedAgeRequirement = poi['MinimumUnaccompaniedAgeRequirement'] - except: - MinimumUnaccompaniedAgeRequirement = None - - try: - MaximumAgeRequirement = poi['MaximumAgeRequirement'] - except: - MaximumAgeRequirement = None - - try: - RestrictionSummary = poi['RestrictionSummary'] - except: - RestrictionSummary = None - - row = [aoi_id, Name, AbbreviatedName, Summary, Keywords, DefaultImage, Location, Featured, WayfindingEnabled, VisibleOnMap, Category, Parent, MenuURL, MinimumHeightRequirement, MinimumUnaccompaniedHeightRequirement, MaximumHeightRequirement, MinimumAgeRequirement, MinimumUnaccompaniedAgeRequirement, MaximumAgeRequirement, RestrictionSummary] - rows.append(row) - -cats = [] - -for cat in data['Category']: - aoi_id = cat['_id'] - Name = cat['Name'] - Parent = cat['Parent'] - row = [aoi_id, Name, Parent] - cats.append(row) - -log_to_database('LZ_attractions_io_categories', cats) -#log_to_database('LZ_attractions_io_poi', rows) +#!/usr/bin/env python3 +"""Import POI and category data from the attractions.io records bundle.""" + +import io +import json +import logging +import os +import sys +import uuid +import zipfile +from datetime import datetime, timezone + +import psycopg2 +import requests +from psycopg2.extras import execute_values + +API_BASE = 'https://api.attractions.io/v1' +API_KEY = '72f0ea9e-d196-508a-bee8-cce62c3228c7' # Knoebels app id (BuildConfig.API_KEY) +APP_VERSION = '1.3.2' # bump if a newer xapk reveals drift +APP_BUILD = 70 +NATIVE_VERSION = '1.0.129+6c346f2' + +DB_PASSWORD = os.environ.get('DB_PASSWORD') +if not DB_PASSWORD: + sys.stderr.write("ERROR: DB_PASSWORD env var is required\n") + sys.exit(1) + +db_params = { + 'dbname': 'gp0', + 'user': os.environ.get('DB_USER', 'kuhnobowls'), + 'password': DB_PASSWORD, + 'host': os.environ.get('DB_HOST', '192.168.88.9'), + 'port': os.environ.get('DB_PORT', '5432'), + 'options': '-c search_path=knoebels', +} + +CATEGORY_COLUMNS = ('_id', 'name', 'parent') + +POI_COLUMNS = ( + '_id', 'name', 'abbreviated_name', 'summary', 'keywords', 'default_image', + 'location', 'featured', 'wayfinding_enabled', 'visible_on_map', 'category', + 'parent', 'menu_url', 'minimum_height_requirement', + 'minimum_unaccompanied_height_requirement', 'maximum_height_requirement', + 'minimum_age_requirement', 'minimum_unaccompanied_age_requirement', + 'maximum_age_requirement', 'restriction_summary', +) + + +def _occasio_headers(installation_token=None): + auth = f'Attractions-Io api-key="{API_KEY}"' + if installation_token: + auth += f', installation-token="{installation_token}"' + return { + 'Authorization': auth, + 'X-Idempotency-Key': str(uuid.uuid4()), + 'Date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'Occasio-Platform': 'Android', + 'Occasio-Platform-Version': '14', + 'Occasio-App-Version': APP_VERSION, + 'Occasio-App-Build': str(APP_BUILD), + 'Occasio-Native-Version': NATIVE_VERSION, + } + + +def fetch_records_bundle(): + """Two-step Occasio data fetch: register an installation, then pull the data zip.""" + install_resp = requests.post( + f'{API_BASE}/installation', + headers=_occasio_headers(), + files={ + 'device_identifier': (None, '123'), + 'user_identifier': (None, str(uuid.uuid4())), + 'app_build': (None, str(APP_BUILD)), + 'app_version': (None, APP_VERSION), + }, + ) + install_resp.raise_for_status() + token = install_resp.json()['token'] + + data_resp = requests.get( + f'{API_BASE}/data', + headers=_occasio_headers(installation_token=token), + ) + data_resp.raise_for_status() + + with zipfile.ZipFile(io.BytesIO(data_resp.content)) as zf: + with zf.open('records.json') as f: + return json.load(f) + + +def _location_to_point(loc): + # Location arrives as 'lat,lon'; Postgres point input wants '(x,y)'. + if not loc: + return None + return f'({loc})' + + +def _category_row(cat): + return (cat['_id'], cat['Name'], cat.get('Parent')) + + +def _poi_row(poi): + return ( + poi['_id'], + poi.get('Name'), + poi.get('AbbreviatedName'), + poi.get('Summary'), + poi.get('Keywords'), + poi.get('DefaultImage'), + _location_to_point(poi.get('Location')), + poi.get('Featured'), + poi.get('WayfindingEnabled'), + poi.get('VisibleOnMap'), + poi.get('Category'), + poi.get('Parent'), + poi.get('MenuURL'), + poi.get('MinimumHeightRequirement'), + poi.get('MinimumUnaccompaniedHeightRequirement'), + poi.get('MaximumHeightRequirement'), + poi.get('MinimumAgeRequirement'), + poi.get('MinimumUnaccompaniedAgeRequirement'), + poi.get('MaximumAgeRequirement'), + poi.get('RestrictionSummary'), + ) + + +def upsert_rows(conn, table, columns, rows, *, location_column=None): + if not rows: + return 0 + cols_sql = ', '.join(f'"{c}"' for c in columns) + update_sql = ', '.join(f'"{c}" = EXCLUDED."{c}"' for c in columns if c != '_id') + # Force the 'point' cast on the location column so multi-row VALUES doesn't + # fall back to inferring the type as text. + placeholders = ', '.join( + '%s::point' if c == location_column else '%s' for c in columns + ) + template = f'({placeholders})' + query = ( + f'INSERT INTO knoebels."{table}" ({cols_sql}) VALUES %s ' + f'ON CONFLICT (_id) DO UPDATE SET {update_sql}' + ) + with conn.cursor() as cur: + execute_values(cur, query, rows, template=template) + return len(rows) + + +def main(): + logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) + + logging.info('Fetching records bundle from attractions.io...') + data = fetch_records_bundle() + logging.info('Got %d Items, %d Categories', len(data['Item']), len(data['Category'])) + + cats = [_category_row(c) for c in data['Category']] + pois = [_poi_row(p) for p in data['Item']] + + with psycopg2.connect(**db_params) as conn: + n_cats = upsert_rows(conn, 'LZ_attractions_io_categories', CATEGORY_COLUMNS, cats) + n_pois = upsert_rows(conn, 'LZ_attractions_io_poi', POI_COLUMNS, pois, + location_column='location') + conn.commit() + logging.info('Upserted %d categories, %d POIs', n_cats, n_pois) + + +if __name__ == '__main__': + main()