kuh-no-bowls/import_poi_data.py

#!/usr/bin/env python3
"""Import POI and category data from the attractions.io records bundle."""

import io
import json
import logging
import os
import sys
import uuid
import zipfile
from datetime import datetime, timezone

import psycopg2
import requests
from psycopg2.extras import execute_values

API_BASE = 'https://api.attractions.io/v1'
API_KEY = '72f0ea9e-d196-508a-bee8-cce62c3228c7'  # Knoebels app id (BuildConfig.API_KEY)
APP_VERSION = '1.3.2'   # bump if a newer xapk reveals drift
APP_BUILD = 70
NATIVE_VERSION = '1.0.129+6c346f2'

DB_PASSWORD = os.environ.get('DB_PASSWORD')
if not DB_PASSWORD:
    sys.stderr.write("ERROR: DB_PASSWORD env var is required\n")
    sys.exit(1)

db_params = {
    'dbname': 'gp0',
    'user': os.environ.get('DB_USER', 'kuhnobowls'),
    'password': DB_PASSWORD,
    'host': os.environ.get('DB_HOST', '192.168.88.9'),
    'port': os.environ.get('DB_PORT', '5432'),
    'options': '-c search_path=knoebels',
}

CATEGORY_COLUMNS = ('_id', 'name', 'parent')

POI_COLUMNS = (
    '_id', 'name', 'abbreviated_name', 'summary', 'keywords', 'default_image',
    'location', 'featured', 'wayfinding_enabled', 'visible_on_map', 'category',
    'parent', 'menu_url', 'minimum_height_requirement',
    'minimum_unaccompanied_height_requirement', 'maximum_height_requirement',
    'minimum_age_requirement', 'minimum_unaccompanied_age_requirement',
    'maximum_age_requirement', 'restriction_summary',
)


def _occasio_headers(installation_token=None):
    auth = f'Attractions-Io api-key="{API_KEY}"'
    if installation_token:
        auth += f', installation-token="{installation_token}"'
    return {
        'Authorization': auth,
        'X-Idempotency-Key': str(uuid.uuid4()),
        'Date': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'Occasio-Platform': 'Android',
        'Occasio-Platform-Version': '14',
        'Occasio-App-Version': APP_VERSION,
        'Occasio-App-Build': str(APP_BUILD),
        'Occasio-Native-Version': NATIVE_VERSION,
    }


def fetch_records_bundle():
    """Two-step Occasio data fetch: register an installation, then pull the data zip."""
    install_resp = requests.post(
        f'{API_BASE}/installation',
        headers=_occasio_headers(),
        files={
            'device_identifier': (None, '123'),
            'user_identifier': (None, str(uuid.uuid4())),
            'app_build': (None, str(APP_BUILD)),
            'app_version': (None, APP_VERSION),
        },
    )
    install_resp.raise_for_status()
    token = install_resp.json()['token']

    data_resp = requests.get(
        f'{API_BASE}/data',
        headers=_occasio_headers(installation_token=token),
    )
    data_resp.raise_for_status()

    with zipfile.ZipFile(io.BytesIO(data_resp.content)) as zf:
        with zf.open('records.json') as f:
            return json.load(f)


def _location_to_point(loc):
    # Location arrives as 'lat,lon'; Postgres point input wants '(x,y)'.
    if not loc:
        return None
    return f'({loc})'


def _category_row(cat):
    return (cat['_id'], cat['Name'], cat.get('Parent'))


def _poi_row(poi):
    return (
        poi['_id'],
        poi.get('Name'),
        poi.get('AbbreviatedName'),
        poi.get('Summary'),
        poi.get('Keywords'),
        poi.get('DefaultImage'),
        _location_to_point(poi.get('Location')),
        poi.get('Featured'),
        poi.get('WayfindingEnabled'),
        poi.get('VisibleOnMap'),
        poi.get('Category'),
        poi.get('Parent'),
        poi.get('MenuURL'),
        poi.get('MinimumHeightRequirement'),
        poi.get('MinimumUnaccompaniedHeightRequirement'),
        poi.get('MaximumHeightRequirement'),
        poi.get('MinimumAgeRequirement'),
        poi.get('MinimumUnaccompaniedAgeRequirement'),
        poi.get('MaximumAgeRequirement'),
        poi.get('RestrictionSummary'),
    )


def upsert_rows(conn, table, columns, rows, *, location_column=None):
    if not rows:
        return 0
    cols_sql = ', '.join(f'"{c}"' for c in columns)
    update_sql = ', '.join(f'"{c}" = EXCLUDED."{c}"' for c in columns if c != '_id')
    # Force the 'point' cast on the location column so multi-row VALUES doesn't
    # fall back to inferring the type as text.
    placeholders = ', '.join(
        '%s::point' if c == location_column else '%s' for c in columns
    )
    template = f'({placeholders})'
    query = (
        f'INSERT INTO knoebels."{table}" ({cols_sql}) VALUES %s '
        f'ON CONFLICT (_id) DO UPDATE SET {update_sql}'
    )
    with conn.cursor() as cur:
        execute_values(cur, query, rows, template=template)
    return len(rows)


def main():
    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)

    logging.info('Fetching records bundle from attractions.io...')
    data = fetch_records_bundle()
    logging.info('Got %d Items, %d Categories', len(data['Item']), len(data['Category']))

    cats = [_category_row(c) for c in data['Category']]
    pois = [_poi_row(p) for p in data['Item']]

    with psycopg2.connect(**db_params) as conn:
        n_cats = upsert_rows(conn, 'LZ_attractions_io_categories', CATEGORY_COLUMNS, cats)
        n_pois = upsert_rows(conn, 'LZ_attractions_io_poi', POI_COLUMNS, pois,
                             location_column='location')
        conn.commit()
    logging.info('Upserted %d categories, %d POIs', n_cats, n_pois)


if __name__ == '__main__':
    main()