Fetch POI records from attractions.io API and upsert
Replace the local records.json read with a two-step Occasio fetch (install + data zip), then upsert Items and Categories into the knoebels schema with ON CONFLICT (_id) DO UPDATE so re-runs refresh in place. Also brings DB credentials in line with attractions_api_logger.py (env vars, default host 192.168.88.9), drops dead helpers, and uses a single shared connection. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
+153
-105
@@ -1,118 +1,166 @@
|
||||
'''
|
||||
Importing POI data from attractions.io api
|
||||
'''
|
||||
#!/usr/bin/env python3
|
||||
"""Import POI and category data from the attractions.io records bundle."""
|
||||
|
||||
from datetime import datetime, date
|
||||
import json, requests, psycopg2
|
||||
from psycopg2 import sql
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import psycopg2
|
||||
import requests
|
||||
from psycopg2.extras import execute_values
|
||||
|
||||
API_BASE = 'https://api.attractions.io/v1'
|
||||
API_KEY = '72f0ea9e-d196-508a-bee8-cce62c3228c7' # Knoebels app id (BuildConfig.API_KEY)
|
||||
APP_VERSION = '1.3.2' # bump if a newer xapk reveals drift
|
||||
APP_BUILD = 70
|
||||
NATIVE_VERSION = '1.0.129+6c346f2'
|
||||
|
||||
DB_PASSWORD = os.environ.get('DB_PASSWORD')
|
||||
if not DB_PASSWORD:
|
||||
sys.stderr.write("ERROR: DB_PASSWORD env var is required\n")
|
||||
sys.exit(1)
|
||||
|
||||
db_params = {
|
||||
'dbname': 'gp0',
|
||||
'user': 'kuhnobowls',
|
||||
'password': 'saddog095',
|
||||
'host': '192.168.88.5',
|
||||
'port': '5432',
|
||||
'options': '-c search_path=knoebels'
|
||||
'user': os.environ.get('DB_USER', 'kuhnobowls'),
|
||||
'password': DB_PASSWORD,
|
||||
'host': os.environ.get('DB_HOST', '192.168.88.9'),
|
||||
'port': os.environ.get('DB_PORT', '5432'),
|
||||
'options': '-c search_path=knoebels',
|
||||
}
|
||||
|
||||
def log_to_database(table_name, data):
|
||||
with psycopg2.connect(**db_params) as conn:
|
||||
with conn.cursor() as cursor:
|
||||
columns = len(data[0])
|
||||
query = sql.SQL("INSERT INTO {table} VALUES ({values})").format(
|
||||
table=sql.Identifier(table_name),
|
||||
values=sql.SQL(', ').join(sql.Placeholder() * columns)
|
||||
CATEGORY_COLUMNS = ('_id', 'name', 'parent')
|
||||
|
||||
POI_COLUMNS = (
|
||||
'_id', 'name', 'abbreviated_name', 'summary', 'keywords', 'default_image',
|
||||
'location', 'featured', 'wayfinding_enabled', 'visible_on_map', 'category',
|
||||
'parent', 'menu_url', 'minimum_height_requirement',
|
||||
'minimum_unaccompanied_height_requirement', 'maximum_height_requirement',
|
||||
'minimum_age_requirement', 'minimum_unaccompanied_age_requirement',
|
||||
'maximum_age_requirement', 'restriction_summary',
|
||||
)
|
||||
|
||||
|
||||
def _occasio_headers(installation_token=None):
|
||||
auth = f'Attractions-Io api-key="{API_KEY}"'
|
||||
if installation_token:
|
||||
auth += f', installation-token="{installation_token}"'
|
||||
return {
|
||||
'Authorization': auth,
|
||||
'X-Idempotency-Key': str(uuid.uuid4()),
|
||||
'Date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
||||
'Occasio-Platform': 'Android',
|
||||
'Occasio-Platform-Version': '14',
|
||||
'Occasio-App-Version': APP_VERSION,
|
||||
'Occasio-App-Build': str(APP_BUILD),
|
||||
'Occasio-Native-Version': NATIVE_VERSION,
|
||||
}
|
||||
|
||||
|
||||
def fetch_records_bundle():
|
||||
"""Two-step Occasio data fetch: register an installation, then pull the data zip."""
|
||||
install_resp = requests.post(
|
||||
f'{API_BASE}/installation',
|
||||
headers=_occasio_headers(),
|
||||
files={
|
||||
'device_identifier': (None, '123'),
|
||||
'user_identifier': (None, str(uuid.uuid4())),
|
||||
'app_build': (None, str(APP_BUILD)),
|
||||
'app_version': (None, APP_VERSION),
|
||||
},
|
||||
)
|
||||
install_resp.raise_for_status()
|
||||
token = install_resp.json()['token']
|
||||
|
||||
data_resp = requests.get(
|
||||
f'{API_BASE}/data',
|
||||
headers=_occasio_headers(installation_token=token),
|
||||
)
|
||||
data_resp.raise_for_status()
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(data_resp.content)) as zf:
|
||||
with zf.open('records.json') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _location_to_point(loc):
|
||||
# Location arrives as 'lat,lon'; Postgres point input wants '(x,y)'.
|
||||
if not loc:
|
||||
return None
|
||||
return f'({loc})'
|
||||
|
||||
|
||||
def _category_row(cat):
|
||||
return (cat['_id'], cat['Name'], cat.get('Parent'))
|
||||
|
||||
|
||||
def _poi_row(poi):
|
||||
return (
|
||||
poi['_id'],
|
||||
poi.get('Name'),
|
||||
poi.get('AbbreviatedName'),
|
||||
poi.get('Summary'),
|
||||
poi.get('Keywords'),
|
||||
poi.get('DefaultImage'),
|
||||
_location_to_point(poi.get('Location')),
|
||||
poi.get('Featured'),
|
||||
poi.get('WayfindingEnabled'),
|
||||
poi.get('VisibleOnMap'),
|
||||
poi.get('Category'),
|
||||
poi.get('Parent'),
|
||||
poi.get('MenuURL'),
|
||||
poi.get('MinimumHeightRequirement'),
|
||||
poi.get('MinimumUnaccompaniedHeightRequirement'),
|
||||
poi.get('MaximumHeightRequirement'),
|
||||
poi.get('MinimumAgeRequirement'),
|
||||
poi.get('MinimumUnaccompaniedAgeRequirement'),
|
||||
poi.get('MaximumAgeRequirement'),
|
||||
poi.get('RestrictionSummary'),
|
||||
)
|
||||
|
||||
cursor.executemany(query, data)
|
||||
conn.commit()
|
||||
|
||||
def generic_query(q):
|
||||
ret = []
|
||||
def upsert_rows(conn, table, columns, rows, *, location_column=None):
|
||||
if not rows:
|
||||
return 0
|
||||
cols_sql = ', '.join(f'"{c}"' for c in columns)
|
||||
update_sql = ', '.join(f'"{c}" = EXCLUDED."{c}"' for c in columns if c != '_id')
|
||||
# Force the 'point' cast on the location column so multi-row VALUES doesn't
|
||||
# fall back to inferring the type as text.
|
||||
placeholders = ', '.join(
|
||||
'%s::point' if c == location_column else '%s' for c in columns
|
||||
)
|
||||
template = f'({placeholders})'
|
||||
query = (
|
||||
f'INSERT INTO knoebels."{table}" ({cols_sql}) VALUES %s '
|
||||
f'ON CONFLICT (_id) DO UPDATE SET {update_sql}'
|
||||
)
|
||||
with conn.cursor() as cur:
|
||||
execute_values(cur, query, rows, template=template)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
|
||||
|
||||
logging.info('Fetching records bundle from attractions.io...')
|
||||
data = fetch_records_bundle()
|
||||
logging.info('Got %d Items, %d Categories', len(data['Item']), len(data['Category']))
|
||||
|
||||
cats = [_category_row(c) for c in data['Category']]
|
||||
pois = [_poi_row(p) for p in data['Item']]
|
||||
|
||||
with psycopg2.connect(**db_params) as conn:
|
||||
with conn.cursor() as cursor:
|
||||
cursor.execute(q)
|
||||
rows = cursor.fetchall()
|
||||
col_names = [desc[0] for desc in cursor.description]
|
||||
n_cats = upsert_rows(conn, 'LZ_attractions_io_categories', CATEGORY_COLUMNS, cats)
|
||||
n_pois = upsert_rows(conn, 'LZ_attractions_io_poi', POI_COLUMNS, pois,
|
||||
location_column='location')
|
||||
conn.commit()
|
||||
logging.info('Upserted %d categories, %d POIs', n_cats, n_pois)
|
||||
|
||||
for row in rows:
|
||||
ret.append(row)
|
||||
|
||||
return ret
|
||||
|
||||
data = None
|
||||
|
||||
with open('records.json', 'r', encoding='utf-8') as file:
|
||||
data = json.load(file)
|
||||
|
||||
rows = []
|
||||
|
||||
for poi in data['Item']:
|
||||
aoi_id = poi['_id']
|
||||
Name = poi['Name']
|
||||
AbbreviatedName = poi['AbbreviatedName']
|
||||
Summary = poi['Summary']
|
||||
Keywords = poi['Keywords']
|
||||
DefaultImage = poi['DefaultImage']
|
||||
Location = poi['Location']
|
||||
Featured = poi['Featured']
|
||||
WayfindingEnabled = poi['WayfindingEnabled']
|
||||
VisibleOnMap = poi['VisibleOnMap']
|
||||
Category = poi['Category']
|
||||
Parent = poi['Parent']
|
||||
|
||||
try:
|
||||
MenuURL = poi['MenuURL']
|
||||
except:
|
||||
MenuURL = None
|
||||
|
||||
try:
|
||||
MinimumHeightRequirement = poi['MinimumHeightRequirement']
|
||||
except:
|
||||
MinimumHeightRequirement = None
|
||||
|
||||
try:
|
||||
MinimumUnaccompaniedHeightRequirement = poi['MinimumUnaccompaniedHeightRequirement']
|
||||
except:
|
||||
MinimumUnaccompaniedHeightRequirement = None
|
||||
|
||||
try:
|
||||
MaximumHeightRequirement = poi['MaximumHeightRequirement']
|
||||
except:
|
||||
MaximumHeightRequirement = None
|
||||
|
||||
try:
|
||||
MinimumAgeRequirement = poi['MinimumAgeRequirement']
|
||||
except:
|
||||
MinimumAgeRequirement = None
|
||||
|
||||
try:
|
||||
MinimumUnaccompaniedAgeRequirement = poi['MinimumUnaccompaniedAgeRequirement']
|
||||
except:
|
||||
MinimumUnaccompaniedAgeRequirement = None
|
||||
|
||||
try:
|
||||
MaximumAgeRequirement = poi['MaximumAgeRequirement']
|
||||
except:
|
||||
MaximumAgeRequirement = None
|
||||
|
||||
try:
|
||||
RestrictionSummary = poi['RestrictionSummary']
|
||||
except:
|
||||
RestrictionSummary = None
|
||||
|
||||
row = [aoi_id, Name, AbbreviatedName, Summary, Keywords, DefaultImage, Location, Featured, WayfindingEnabled, VisibleOnMap, Category, Parent, MenuURL, MinimumHeightRequirement, MinimumUnaccompaniedHeightRequirement, MaximumHeightRequirement, MinimumAgeRequirement, MinimumUnaccompaniedAgeRequirement, MaximumAgeRequirement, RestrictionSummary]
|
||||
rows.append(row)
|
||||
|
||||
cats = []
|
||||
|
||||
for cat in data['Category']:
|
||||
aoi_id = cat['_id']
|
||||
Name = cat['Name']
|
||||
Parent = cat['Parent']
|
||||
row = [aoi_id, Name, Parent]
|
||||
cats.append(row)
|
||||
|
||||
log_to_database('LZ_attractions_io_categories', cats)
|
||||
#log_to_database('LZ_attractions_io_poi', rows)
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user