Fetch POI records from attractions.io API and upsert
Replace the local records.json read with a two-step Occasio fetch (install + data zip), then upsert Items and Categories into the knoebels schema with ON CONFLICT (_id) DO UPDATE so re-runs refresh in place. Also brings DB credentials in line with attractions_api_logger.py (env vars, default host 192.168.88.9), drops dead helpers, and uses a single shared connection. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
+166
-118
@@ -1,118 +1,166 @@
|
|||||||
'''
|
#!/usr/bin/env python3
|
||||||
Importing POI data from attractions.io api
|
"""Import POI and category data from the attractions.io records bundle."""
|
||||||
'''
|
|
||||||
#!/usr/bin/env python3
|
import io
|
||||||
|
import json
|
||||||
from datetime import datetime, date
|
import logging
|
||||||
import json, requests, psycopg2
|
import os
|
||||||
from psycopg2 import sql
|
import sys
|
||||||
|
import uuid
|
||||||
db_params = {
|
import zipfile
|
||||||
'dbname': 'gp0',
|
from datetime import datetime, timezone
|
||||||
'user': 'kuhnobowls',
|
|
||||||
'password': 'saddog095',
|
import psycopg2
|
||||||
'host': '192.168.88.5',
|
import requests
|
||||||
'port': '5432',
|
from psycopg2.extras import execute_values
|
||||||
'options': '-c search_path=knoebels'
|
|
||||||
}
|
API_BASE = 'https://api.attractions.io/v1'
|
||||||
|
API_KEY = '72f0ea9e-d196-508a-bee8-cce62c3228c7' # Knoebels app id (BuildConfig.API_KEY)
|
||||||
def log_to_database(table_name, data):
|
APP_VERSION = '1.3.2' # bump if a newer xapk reveals drift
|
||||||
with psycopg2.connect(**db_params) as conn:
|
APP_BUILD = 70
|
||||||
with conn.cursor() as cursor:
|
NATIVE_VERSION = '1.0.129+6c346f2'
|
||||||
columns = len(data[0])
|
|
||||||
query = sql.SQL("INSERT INTO {table} VALUES ({values})").format(
|
DB_PASSWORD = os.environ.get('DB_PASSWORD')
|
||||||
table=sql.Identifier(table_name),
|
if not DB_PASSWORD:
|
||||||
values=sql.SQL(', ').join(sql.Placeholder() * columns)
|
sys.stderr.write("ERROR: DB_PASSWORD env var is required\n")
|
||||||
)
|
sys.exit(1)
|
||||||
|
|
||||||
cursor.executemany(query, data)
|
db_params = {
|
||||||
conn.commit()
|
'dbname': 'gp0',
|
||||||
|
'user': os.environ.get('DB_USER', 'kuhnobowls'),
|
||||||
def generic_query(q):
|
'password': DB_PASSWORD,
|
||||||
ret = []
|
'host': os.environ.get('DB_HOST', '192.168.88.9'),
|
||||||
with psycopg2.connect(**db_params) as conn:
|
'port': os.environ.get('DB_PORT', '5432'),
|
||||||
with conn.cursor() as cursor:
|
'options': '-c search_path=knoebels',
|
||||||
cursor.execute(q)
|
}
|
||||||
rows = cursor.fetchall()
|
|
||||||
col_names = [desc[0] for desc in cursor.description]
|
CATEGORY_COLUMNS = ('_id', 'name', 'parent')
|
||||||
|
|
||||||
for row in rows:
|
POI_COLUMNS = (
|
||||||
ret.append(row)
|
'_id', 'name', 'abbreviated_name', 'summary', 'keywords', 'default_image',
|
||||||
|
'location', 'featured', 'wayfinding_enabled', 'visible_on_map', 'category',
|
||||||
return ret
|
'parent', 'menu_url', 'minimum_height_requirement',
|
||||||
|
'minimum_unaccompanied_height_requirement', 'maximum_height_requirement',
|
||||||
data = None
|
'minimum_age_requirement', 'minimum_unaccompanied_age_requirement',
|
||||||
|
'maximum_age_requirement', 'restriction_summary',
|
||||||
with open('records.json', 'r', encoding='utf-8') as file:
|
)
|
||||||
data = json.load(file)
|
|
||||||
|
|
||||||
rows = []
|
def _occasio_headers(installation_token=None):
|
||||||
|
auth = f'Attractions-Io api-key="{API_KEY}"'
|
||||||
for poi in data['Item']:
|
if installation_token:
|
||||||
aoi_id = poi['_id']
|
auth += f', installation-token="{installation_token}"'
|
||||||
Name = poi['Name']
|
return {
|
||||||
AbbreviatedName = poi['AbbreviatedName']
|
'Authorization': auth,
|
||||||
Summary = poi['Summary']
|
'X-Idempotency-Key': str(uuid.uuid4()),
|
||||||
Keywords = poi['Keywords']
|
'Date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||||
DefaultImage = poi['DefaultImage']
|
'Occasio-Platform': 'Android',
|
||||||
Location = poi['Location']
|
'Occasio-Platform-Version': '14',
|
||||||
Featured = poi['Featured']
|
'Occasio-App-Version': APP_VERSION,
|
||||||
WayfindingEnabled = poi['WayfindingEnabled']
|
'Occasio-App-Build': str(APP_BUILD),
|
||||||
VisibleOnMap = poi['VisibleOnMap']
|
'Occasio-Native-Version': NATIVE_VERSION,
|
||||||
Category = poi['Category']
|
}
|
||||||
Parent = poi['Parent']
|
|
||||||
|
|
||||||
try:
|
def fetch_records_bundle():
|
||||||
MenuURL = poi['MenuURL']
|
"""Two-step Occasio data fetch: register an installation, then pull the data zip."""
|
||||||
except:
|
install_resp = requests.post(
|
||||||
MenuURL = None
|
f'{API_BASE}/installation',
|
||||||
|
headers=_occasio_headers(),
|
||||||
try:
|
files={
|
||||||
MinimumHeightRequirement = poi['MinimumHeightRequirement']
|
'device_identifier': (None, '123'),
|
||||||
except:
|
'user_identifier': (None, str(uuid.uuid4())),
|
||||||
MinimumHeightRequirement = None
|
'app_build': (None, str(APP_BUILD)),
|
||||||
|
'app_version': (None, APP_VERSION),
|
||||||
try:
|
},
|
||||||
MinimumUnaccompaniedHeightRequirement = poi['MinimumUnaccompaniedHeightRequirement']
|
)
|
||||||
except:
|
install_resp.raise_for_status()
|
||||||
MinimumUnaccompaniedHeightRequirement = None
|
token = install_resp.json()['token']
|
||||||
|
|
||||||
try:
|
data_resp = requests.get(
|
||||||
MaximumHeightRequirement = poi['MaximumHeightRequirement']
|
f'{API_BASE}/data',
|
||||||
except:
|
headers=_occasio_headers(installation_token=token),
|
||||||
MaximumHeightRequirement = None
|
)
|
||||||
|
data_resp.raise_for_status()
|
||||||
try:
|
|
||||||
MinimumAgeRequirement = poi['MinimumAgeRequirement']
|
with zipfile.ZipFile(io.BytesIO(data_resp.content)) as zf:
|
||||||
except:
|
with zf.open('records.json') as f:
|
||||||
MinimumAgeRequirement = None
|
return json.load(f)
|
||||||
|
|
||||||
try:
|
|
||||||
MinimumUnaccompaniedAgeRequirement = poi['MinimumUnaccompaniedAgeRequirement']
|
def _location_to_point(loc):
|
||||||
except:
|
# Location arrives as 'lat,lon'; Postgres point input wants '(x,y)'.
|
||||||
MinimumUnaccompaniedAgeRequirement = None
|
if not loc:
|
||||||
|
return None
|
||||||
try:
|
return f'({loc})'
|
||||||
MaximumAgeRequirement = poi['MaximumAgeRequirement']
|
|
||||||
except:
|
|
||||||
MaximumAgeRequirement = None
|
def _category_row(cat):
|
||||||
|
return (cat['_id'], cat['Name'], cat.get('Parent'))
|
||||||
try:
|
|
||||||
RestrictionSummary = poi['RestrictionSummary']
|
|
||||||
except:
|
def _poi_row(poi):
|
||||||
RestrictionSummary = None
|
return (
|
||||||
|
poi['_id'],
|
||||||
row = [aoi_id, Name, AbbreviatedName, Summary, Keywords, DefaultImage, Location, Featured, WayfindingEnabled, VisibleOnMap, Category, Parent, MenuURL, MinimumHeightRequirement, MinimumUnaccompaniedHeightRequirement, MaximumHeightRequirement, MinimumAgeRequirement, MinimumUnaccompaniedAgeRequirement, MaximumAgeRequirement, RestrictionSummary]
|
poi.get('Name'),
|
||||||
rows.append(row)
|
poi.get('AbbreviatedName'),
|
||||||
|
poi.get('Summary'),
|
||||||
cats = []
|
poi.get('Keywords'),
|
||||||
|
poi.get('DefaultImage'),
|
||||||
for cat in data['Category']:
|
_location_to_point(poi.get('Location')),
|
||||||
aoi_id = cat['_id']
|
poi.get('Featured'),
|
||||||
Name = cat['Name']
|
poi.get('WayfindingEnabled'),
|
||||||
Parent = cat['Parent']
|
poi.get('VisibleOnMap'),
|
||||||
row = [aoi_id, Name, Parent]
|
poi.get('Category'),
|
||||||
cats.append(row)
|
poi.get('Parent'),
|
||||||
|
poi.get('MenuURL'),
|
||||||
log_to_database('LZ_attractions_io_categories', cats)
|
poi.get('MinimumHeightRequirement'),
|
||||||
#log_to_database('LZ_attractions_io_poi', rows)
|
poi.get('MinimumUnaccompaniedHeightRequirement'),
|
||||||
|
poi.get('MaximumHeightRequirement'),
|
||||||
|
poi.get('MinimumAgeRequirement'),
|
||||||
|
poi.get('MinimumUnaccompaniedAgeRequirement'),
|
||||||
|
poi.get('MaximumAgeRequirement'),
|
||||||
|
poi.get('RestrictionSummary'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upsert_rows(conn, table, columns, rows, *, location_column=None):
|
||||||
|
if not rows:
|
||||||
|
return 0
|
||||||
|
cols_sql = ', '.join(f'"{c}"' for c in columns)
|
||||||
|
update_sql = ', '.join(f'"{c}" = EXCLUDED."{c}"' for c in columns if c != '_id')
|
||||||
|
# Force the 'point' cast on the location column so multi-row VALUES doesn't
|
||||||
|
# fall back to inferring the type as text.
|
||||||
|
placeholders = ', '.join(
|
||||||
|
'%s::point' if c == location_column else '%s' for c in columns
|
||||||
|
)
|
||||||
|
template = f'({placeholders})'
|
||||||
|
query = (
|
||||||
|
f'INSERT INTO knoebels."{table}" ({cols_sql}) VALUES %s '
|
||||||
|
f'ON CONFLICT (_id) DO UPDATE SET {update_sql}'
|
||||||
|
)
|
||||||
|
with conn.cursor() as cur:
|
||||||
|
execute_values(cur, query, rows, template=template)
|
||||||
|
return len(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
|
||||||
|
|
||||||
|
logging.info('Fetching records bundle from attractions.io...')
|
||||||
|
data = fetch_records_bundle()
|
||||||
|
logging.info('Got %d Items, %d Categories', len(data['Item']), len(data['Category']))
|
||||||
|
|
||||||
|
cats = [_category_row(c) for c in data['Category']]
|
||||||
|
pois = [_poi_row(p) for p in data['Item']]
|
||||||
|
|
||||||
|
with psycopg2.connect(**db_params) as conn:
|
||||||
|
n_cats = upsert_rows(conn, 'LZ_attractions_io_categories', CATEGORY_COLUMNS, cats)
|
||||||
|
n_pois = upsert_rows(conn, 'LZ_attractions_io_poi', POI_COLUMNS, pois,
|
||||||
|
location_column='location')
|
||||||
|
conn.commit()
|
||||||
|
logging.info('Upserted %d categories, %d POIs', n_cats, n_pois)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|||||||
Reference in New Issue
Block a user