pca_svd: exclude torn-down powersurge from the PCA, trim attractions.io category names (trailing-space gotcha), time-side validation of PC1-PC3 interpretations, per-ride R^2 of the rank-3 reconstruction, blog-ready PC2xPC3 scatter. writeup: PC1/PC2/PC3 narrative sections. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
552 KiB
552 KiB
In [24]:
import os
from pathlib import Path
import pandas as pd
from dotenv import load_dotenv
from sqlalchemy import create_engine, text
load_dotenv(Path.cwd() / ".env")
engine = create_engine(
"postgresql+psycopg2://{user}:{pw}@{host}:{port}/{db}".format(
user=os.environ["PG_USER"], pw=os.environ["PG_PASSWORD"],
host=os.environ["PG_HOST"], port=os.environ["PG_PORT"], db=os.environ["PG_DB"],
)
)
df = pd.read_sql(
text(
"""
SELECT a.time_stamp,
r.ride AS ride_name,
a.queue_time AS queue_time_sec
FROM knoebels."LZ_attractions_io" a
JOIN knoebels.ride_master r ON r.attractions_dot_io_id = a._id
WHERE a.queue_time IS NOT NULL
"""
),
engine,
)
df["queue_time_min"] = df["queue_time_sec"] / 60.0
df["time_stamp"] = pd.to_datetime(df["time_stamp"])
def ride_time_matrix(df, freq="5min"):
"""Long table -> (time x ride) matrix of mean wait in minutes per time bin."""
snapped = df.assign(ts=df["time_stamp"].dt.floor(freq))
return (
snapped.pivot_table(index="ts", columns="ride_name",
values="queue_time_min", aggfunc="mean")
.sort_index()
.ffill() # carry the last known wait forward
.fillna(0) # anything still missing (pre-open) = no line
)
df_matrix = ride_time_matrix(df).drop(columns=["powersurge"], errors="ignore")
print(f"{len(df):,} readings -> matrix of {df_matrix.shape[0]:,} time-snapshots x {df_matrix.shape[1]} rides")
#df_matrix.iloc[:3, :5]
df_matrix.info()657,498 readings -> matrix of 12,535 time-snapshots x 56 rides <class 'pandas.DataFrame'> DatetimeIndex: 12535 entries, 2024-05-15 12:20:00 to 2026-05-31 17:30:00 Data columns (total 56 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 antique_cars 12535 non-null float64 1 balloon_race 12535 non-null float64 2 black_diamond 12535 non-null float64 3 bumper_cars 12535 non-null float64 4 cosmotron 12535 non-null float64 5 downdraft 12535 non-null float64 6 fandango 12535 non-null float64 7 flyer 12535 non-null float64 8 flying_tigers 12535 non-null float64 9 flying_turns 12535 non-null float64 10 galleon 12535 non-null float64 11 giant_flume 12535 non-null float64 12 giant_wheel 12535 non-null float64 13 goin'_buggy 12535 non-null float64 14 grand_carousel 12535 non-null float64 15 hand_cars 12535 non-null float64 16 haunted_mansion 12535 non-null float64 17 helicopters 12535 non-null float64 18 impulse 12535 non-null float64 19 italian_trapeze 12535 non-null float64 20 jet_skyfighter 12535 non-null float64 21 kiddie_boats 12535 non-null float64 22 kiddie_bumper_cars 12535 non-null float64 23 kiddie_firetrucks 12535 non-null float64 24 kiddie_himalaya 12535 non-null float64 25 kiddie_wheel 12535 non-null float64 26 kiddie_whip 12535 non-null float64 27 kozmo's_kurves 12535 non-null float64 28 looper 12535 non-null float64 29 merry_mixer 12535 non-null float64 30 motor_boats 12535 non-null float64 31 ole_smokey 12535 non-null float64 32 panther_cars 12535 non-null float64 33 paradrop 12535 non-null float64 34 paratrooper 12535 non-null float64 35 pete's_fleet 12535 non-null float64 36 phoenix 12535 non-null float64 37 pioneer_train 12535 non-null float64 38 pony_carts 12535 non-null float64 39 red_baron 12535 non-null float64 40 ribbit 12535 non-null float64 41 roto_jets 12535 non-null float64 42 s&g_carousel 12535 non-null float64 43 satellite 12535 non-null float64 44 scenic_skyway 12535 non-null float64 45 sklooosh 12535 non-null float64 46 spanish_bambini 12535 non-null float64 47 stratosfear 12535 non-null float64 48 super_round-up 12535 non-null float64 49 tea_cups 12535 non-null float64 50 tilt-a-whirl 12535 non-null float64 51 tornado 12535 non-null float64 52 tumbling_timbers 12535 non-null float64 53 twister 12535 non-null float64 54 umbrella_ride 12535 non-null float64 55 whipper 12535 non-null float64 dtypes: float64(56) memory usage: 5.5 MB
In [17]:
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
scaled = scaler.fit_transform(df_matrix) # each column now mean 0, variance 1
pd.DataFrame(scaled, columns=df_matrix.columns).describe().loc[["mean", "std"]].iloc[:, :4].round(2)Out [17]:
| ride_name | antique_cars | balloon_race | black_diamond | bumper_cars |
|---|---|---|---|---|
| mean | 0.0 | -0.0 | -0.0 | -0.0 |
| std | 1.0 | 1.0 | 1.0 | 1.0 |
In [18]:
from sklearn.decomposition import PCA
pca = PCA(n_components=3)
principal_components = pca.fit_transform(scaled)
scores = pd.DataFrame(principal_components, index=df_matrix.index,
columns=["PC1", "PC2", "PC3"])
loadings = pd.DataFrame(pca.components_.T, index=df_matrix.columns,
columns=["PC1", "PC2", "PC3"])
for i, pc in enumerate(["PC1", "PC2", "PC3"]):
print(f"{pc}: {pca.explained_variance_ratio_[i]:5.1%} of all variance")
print(f"{'':4} {pca.explained_variance_ratio_[:3].sum():5.1%} captured by just 3 axes (out of 56)")PC1: 40.4% of all variance
PC2: 13.0% of all variance
PC3: 6.7% of all variance
60.1% captured by just 3 axes (out of 56)
In [19]:
import numpy as np
import matplotlib.pyplot as plt
def validation_plot(real_quantity, pc_score, score_name, xlabel):
"""Put a PC score and the real-world quantity it should track on the same
standardized scale. If the PC *were* that quantity every dot would sit on the
dashed 45° line; the red line is the actual fit (its slope is the correlation
r). The closer red hugs the diagonal, the stronger the validation."""
zx = (real_quantity - real_quantity.mean()) / real_quantity.std()
zy = (pc_score - pc_score.mean()) / pc_score.std()
r = real_quantity.corr(pc_score)
fig, ax = plt.subplots(figsize=(7, 6.5))
ax.scatter(zx, zy, s=4, alpha=0.12, color="steelblue")
line = np.array([-3, 3])
ax.plot(line, line, "k--", lw=1.3, label="perfect match (y = x)")
ax.plot(line, r * line, color="crimson", lw=2.5,
label=f"actual relationship (r = {r:.2f})")
ax.set(xlim=(-3, 3), ylim=(-3, 3), aspect="equal",
xlabel=f"{xlabel} (standardized)",
ylabel=f"{score_name} score (standardized)")
ax.set_title(f"{score_name}: does it track the real-world quantity?")
ax.legend(loc="upper left")
plt.tight_layout()
park_mean = df_matrix.mean(axis=1)
validation_plot(park_mean, scores["PC1"], "PC1", "Park-wide mean wait")In [20]:
from sklearn.cluster import KMeans
# category tag, with the sneaky trailing space trimmed off
category = pd.read_sql(
text(
"""
SELECT r.ride AS ride_name, trim(c.name) AS category
FROM knoebels.ride_master r
JOIN knoebels."LZ_attractions_io_poi" p ON p._id = r.attractions_dot_io_id
LEFT JOIN knoebels."LZ_attractions_io_categories" c ON c._id = p.category
"""
),
engine,
).set_index("ride_name")["category"]
coords = pd.read_sql(
text(
"""
SELECT r.ride AS ride_name,
(p.location)[0]::float AS lat, (p.location)[1]::float AS lon
FROM knoebels.ride_master r
JOIN knoebels."LZ_attractions_io_poi" p ON p._id = r.attractions_dot_io_id
WHERE p.location IS NOT NULL
"""
),
engine,
).set_index("ride_name")
annotated = loadings.join(category.rename("category"))
located = annotated.join(coords, how="inner")
# k-means on STANDARDIZED coordinates only -> reproducible spatial areas
located["area"] = KMeans(n_clusters=5, n_init=10, random_state=0).fit_predict(
StandardScaler().fit_transform(located[["lat", "lon"]])
)
kiddie_area = located.groupby("area")["PC2"].mean().idxmin()
kiddie_rides = located.index[located["area"] == kiddie_area]
main_rides = located.index[located["area"] != kiddie_area]
# Does the map agree? Left: each ride at its real GPS point, colored by PC2
# loading. Right: areas from clustering lat/lon alone — no queue data, no PCA.
fig, axes = plt.subplots(1, 2, figsize=(15, 7))
sc = axes[0].scatter(located["lon"], located["lat"], c=located["PC2"],
cmap="RdBu_r", s=140, edgecolor="black", lw=0.4)
axes[0].set(title="Each ride at its real location, colored by PC2 loading\n"
"(blue = Kiddieland end, red = main-midway end)",
xlabel="Longitude", ylabel="Latitude")
plt.colorbar(sc, ax=axes[0], label="PC2 loading")
for ride, row in located.iterrows():
axes[0].annotate(ride, (row["lon"], row["lat"]), fontsize=6, alpha=0.6,
xytext=(3, 3), textcoords="offset points")
for area, g in located.groupby("area"):
label = f"Area {area}" + (" (Kiddieland)" if area == kiddie_area else "")
axes[1].scatter(g["lon"], g["lat"], label=label, s=140, alpha=0.85)
axes[1].set(title="k-means on standardized lat/lon alone\n(no queue data, no PCA)",
xlabel="Longitude", ylabel="Latitude")
axes[1].legend()
plt.tight_layout()
print(f"Kiddieland ({len(kiddie_rides)} rides): {', '.join(sorted(kiddie_rides))}")Kiddieland (19 rides): goin'_buggy, hand_cars, haunted_mansion, helicopters, jet_skyfighter, kiddie_boats, kiddie_bumper_cars, kiddie_firetrucks, kiddie_wheel, kiddie_whip, ole_smokey, panther_cars, pete's_fleet, pony_carts, red_baron, ribbit, s&g_carousel, spanish_bambini, umbrella_ride
In [21]:
contrast = df_matrix[main_rides].mean(axis=1) - df_matrix[kiddie_rides].mean(axis=1)
validation_plot(contrast, scores["PC2"], "PC2", "Main-midway − Kiddieland wait")In [22]:
daytime_rides = loadings.index[loadings["PC3"] > 0.1]
evening_rides = loadings.index[loadings["PC3"] < -0.1]
hour = df_matrix.index.hour
park_by_hour = df_matrix.mean(axis=1).groupby(hour).mean()
fig, axes = plt.subplots(1, 2, figsize=(14, 5))
# left: the axis itself, by hour
scores.groupby(scores.index.hour)["PC3"].mean().plot(ax=axes[0], marker="o")
axes[0].axhline(0, color="grey", lw=0.5)
axes[0].set(title="PC3 score by hour (+ daytime / − evening)",
xlabel="Hour (local)", ylabel="Mean PC3 score")
# right: each group's wait as a share of the park-wide level, de-trended
for name, grp, color in [("daytime rides (PC3 +)", daytime_rides, "tab:orange"),
("evening rides (PC3 −)", evening_rides, "tab:blue")]:
(df_matrix[grp].mean(axis=1).groupby(hour).mean() / park_by_hour).plot(
ax=axes[1], marker="o", label=name, color=color)
axes[1].axhline(1.0, color="grey", lw=0.5)
axes[1].set(title="Group wait ÷ park-wide wait, by hour",
xlabel="Hour (local)", ylabel="Share of park-wide mean wait")
axes[1].legend()
plt.tight_layout()In [ ]:
# PC2 calls these two rides almost identical (both lean "main midway").
# PC3 sends them to opposite ends.
loadings.loc[["tilt-a-whirl", "giant_wheel"], ["PC2", "PC3"]].round(3)In [23]:
from matplotlib.patches import Ellipse
# the tight Kiddieland pile = rides loading hard-negative on PC2
kiddie_core = loadings.index[loadings["PC2"] < -0.13]
kx, ky = loadings.loc[kiddie_core, "PC2"], loadings.loc[kiddie_core, "PC3"]
fig, ax = plt.subplots(figsize=(11, 8.5))
cat_colors = {"Family": "tab:blue", "Kiddie": "tab:orange", "Thrill": "tab:red"}
for c, g in annotated.dropna(subset=["category"]).groupby("category"):
ax.scatter(g["PC2"], g["PC3"], label=c, color=cat_colors.get(c, "grey"),
s=80, alpha=0.75, edgecolor="white", zorder=3)
# one callout for the dense kiddie pile, instead of 13 overlapping labels
w, h = (kx.max() - kx.min()) + 0.07, (ky.max() - ky.min()) + 0.07
ax.add_patch(Ellipse((kx.mean(), ky.mean()), width=w, height=h, facecolor="tab:orange",
alpha=0.08, edgecolor="tab:orange", ls="--", lw=1.5, zorder=1))
ax.annotate("Kiddieland\n(tight cluster of kiddie rides)", (kx.mean(), ky.mean() - h / 2),
ha="center", va="top", fontsize=9, color="tab:orange", fontweight="bold")
# label only the rides that carry the story
plain = {"giant_wheel": (6, 5), "grand_carousel": (8, -3), "impulse": (7, -3),
"bumper_cars": (7, 4), "motor_boats": (7, 0), "paratrooper": (7, 0),
"tea_cups": (7, 0), "tilt-a-whirl": (7, -2), "roto_jets": (-42, 6)}
for ride, (dx, dy) in plain.items():
p = annotated.loc[ride]
ax.annotate(ride, (p["PC2"], p["PC3"]), fontsize=8.5,
xytext=(dx, dy), textcoords="offset points", zorder=4)
# the two Family-tagged rides that behave like kiddie rides — pull labels out with arrows
misfiled = {"s&g_carousel": (-70, -30), "kiddie_himalaya": (-78, 20)}
for ride, (dx, dy) in misfiled.items():
p = annotated.loc[ride]
ax.annotate(ride + " (tagged Family)", (p["PC2"], p["PC3"]), fontsize=8.5,
color="tab:blue", xytext=(dx, dy), textcoords="offset points",
arrowprops=dict(arrowstyle="->", lw=0.7, color="tab:blue"), zorder=5)
ax.axhline(0, color="grey", lw=0.5)
ax.axvline(0, color="grey", lw=0.5)
ax.set_xlabel("PC2 Kiddieland corner ← → main midway")
ax.set_ylabel("PC3 evening crowd ← → daytime / school-group daypart")
ax.set_title("Knoebels rides in PC2 × PC3 space\n"
"(position comes from queue patterns alone; color = the park's category tag)")
ax.legend(title="Category tag", loc="lower left")
plt.tight_layout()