Add whole-database snapshot mode

Adds `kb_exfiltrator snapshot <db> [endpoint] [timeout-secs]`, which uploads a
consistent copy of the whole database instead of just extracted highlights. The
server can then parse whatever it wants, so adding a new statistic never means
touching the device again.

Uses VACUUM INTO rather than copying the file. Nickel keeps the database in WAL
mode and recent commits live in KoboReader.sqlite-wal until a checkpoint, so
copying the bare .sqlite silently loses reading progress — measured at 19 hours
of drift on the live device. Copying all three files instead is non-atomic: a
checkpoint landing mid-copy yields a pre-checkpoint main file plus a post-reset
WAL, losing data with no error. VACUUM INTO runs in a read transaction, so it
sees WAL-resident commits and emits one compacted journal_mode=delete file.

Also fixes the failure mode that left a process hung for 16 days: the server
stopped responding and std.http.Client has no timeout, so the process blocked
forever while holding the SQLite handle open. Snapshot mode closes the database
before any network I/O, and a watchdog thread hard-exits after a deadline
covering the whole run (snapshotting can stall on a locked database too).

Two portability constraints, both specific to the Kobo Elipsa:

  - Linux 4.9.77 predates statx (4.11). Zig's File.stat() and getEndPos() both
    issue it, returning ENOSYS as a bare error.Unexpected — after the snapshot
    is already written, so it presents as a post-write failure. readToEndAlloc
    with no size hint uses plain read() calls instead.
  - usize is 32-bit on armv7, so a u64 stat size will not coerce. Only the ARM
    build catches this.

Legacy invocations are unchanged, so the existing NickelMenu item keeps working.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

src/main.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const snapshot = @import("snapshot.zig");
 const c = @cImport({
     @cInclude("sqlite3.h");
 });
@@ -125,28 +126,79 @@ const Highlight = struct {
     }
 };
 
+// Zig only collects `test` blocks from the test unit's root file, so pull in the
+// snapshot module's tests explicitly.
+test {
+    _ = @import("snapshot.zig");
+}
+
+const usage =
+    \\Usage:
+    \\  kb_exfiltrator snapshot   <db> [endpoint] [timeout-secs]
+    \\      Upload a consistent whole-database snapshot (VACUUM INTO). This is the
+    \\      pipeline's primary mode: the server parses it, so adding new stats never
+    \\      requires touching the device. Omit endpoint to leave the snapshot on disk.
+    \\
+    \\  kb_exfiltrator highlights <db> [endpoint]
+    \\      Legacy mode: extract highlights and POST them as JSON.
+    \\
+    \\  kb_exfiltrator <db> [endpoint]
+    \\      Backwards-compatible alias for `highlights`.
+    \\
+;
+
 pub fn main() !void {
     var gpa = std.heap.GeneralPurposeAllocator(.{}){};
     defer _ = gpa.deinit();
     const allocator = gpa.allocator();
 
-    // Get command line args
     const args = try std.process.argsAlloc(allocator);
     defer std.process.argsFree(allocator, args);
 
     if (args.len < 2) {
-        std.debug.print("Usage: {s} <path-to-kobo-database.sqlite> [flask-endpoint-url]\n", .{args[0]});
+        std.debug.print("{s}", .{usage});
-        std.debug.print("Example: {s} /mnt/kobo/.kobo/KoboReader.sqlite\n", .{args[0]});
-        std.debug.print("Example: {s} /mnt/kobo/.kobo/KoboReader.sqlite https://myserver.com/api/highlights\n", .{args[0]});
         return error.MissingArgument;
     }
 
-    const db_path = args[1];
+    if (std.mem.eql(u8, args[1], "snapshot")) {
-    const endpoint_url = if (args.len >= 3) args[2] else null;
+        if (args.len < 3) {
+            std.debug.print("{s}", .{usage});
+            return error.MissingArgument;
+        }
+        const timeout = if (args.len >= 5)
+            std.fmt.parseInt(u64, args[4], 10) catch snapshot.default_timeout_secs
+        else
+            snapshot.default_timeout_secs;
+        return snapshot.run(
+            allocator,
+            args[2],
+            if (args.len >= 4) args[3] else null,
+            snapshot.default_snapshot_path,
+            timeout,
+        );
+    }
+
+    // `highlights <db> [endpoint]`, or the legacy `<db> [endpoint]` form.
+    const legacy = std.mem.eql(u8, args[1], "highlights");
+    const rest = if (legacy) args[2..] else args[1..];
+    if (rest.len < 1) {
+        std.debug.print("{s}", .{usage});
+        return error.MissingArgument;
+    }
+    return runHighlights(allocator, rest[0], if (rest.len >= 2) rest[1] else null);
+}
+
+fn runHighlights(
+    allocator: std.mem.Allocator,
+    db_path: []const u8,
+    endpoint_url: ?[]const u8,
+) !void {
+    // Open database. db_path arrives as a plain slice, so re-terminate it for the C API.
+    const db_path_z = try allocator.dupeZ(u8, db_path);
+    defer allocator.free(db_path_z);
 
-    // Open database
     var db: ?*c.sqlite3 = null;
-    const rc = c.sqlite3_open(db_path.ptr, &db);
+    const rc = c.sqlite3_open(db_path_z.ptr, &db);
     if (rc != c.SQLITE_OK) {
         std.debug.print("Failed to open database: {s}\n", .{c.sqlite3_errmsg(db)});
         return error.DatabaseOpenFailed;

src/snapshot.zig

@@ -0,0 +1,183 @@
+//! Whole-database snapshot mode.
+//!
+//! Produces a single, internally-consistent copy of KoboReader.sqlite and uploads it.
+//!
+//! Why a snapshot rather than copying the raw files: Nickel keeps the database in WAL mode,
+//! and recent commits live in `KoboReader.sqlite-wal` until a checkpoint. Copying the bare
+//! `.sqlite` silently loses hours of reading progress (measured: 19h of drift on a live
+//! device). Copying all three files instead is non-atomic — a checkpoint landing mid-copy
+//! yields a pre-checkpoint main file plus a post-reset WAL, losing data with no error.
+//!
+//! `VACUUM INTO` runs inside a read transaction, so it sees WAL-resident commits and emits
+//! one compacted `journal_mode=delete` file. One file, one consistent read, no reassembly.
+
+const std = @import("std");
+const c = @cImport({
+    @cInclude("sqlite3.h");
+});
+
+pub const default_snapshot_path = "/tmp/kobo_snapshot.sqlite";
+pub const default_timeout_secs: u64 = 120;
+
+/// Upper bound on a snapshot we're willing to buffer. The live database is ~2.4 MB; this
+/// is a guard against pathological growth, not a real expectation.
+const max_snapshot_bytes: usize = 256 * 1024 * 1024;
+
+/// Hard upper bound on the whole run.
+///
+/// v1 hung for 16 days: the server stopped responding and `std.http.Client` has no timeout,
+/// so the process blocked forever *while holding the SQLite handle open*. This guarantees the
+/// process dies instead. Snapshotting can stall too (a locked database), so the watchdog
+/// covers the entire run, not just the upload.
+fn watchdog(secs: u64) void {
+    std.Thread.sleep(secs * std.time.ns_per_s);
+    std.debug.print("timeout: exceeded {d}s, aborting\n", .{secs});
+    std.process.exit(2);
+}
+
+fn sha256Hex(data: []const u8, out: *[64]u8) void {
+    var digest: [32]u8 = undefined;
+    std.crypto.hash.sha2.Sha256.hash(data, &digest, .{});
+    const hex = "0123456789abcdef";
+    for (digest, 0..) |byte, i| {
+        out[i * 2] = hex[byte >> 4];
+        out[i * 2 + 1] = hex[byte & 0x0F];
+    }
+}
+
+/// SQL string literals escape a single quote by doubling it.
+fn quoteSqlLiteral(allocator: std.mem.Allocator, s: []const u8) ![]u8 {
+    var out: std.ArrayList(u8) = .empty;
+    errdefer out.deinit(allocator);
+    for (s) |ch| {
+        if (ch == '\'') try out.append(allocator, '\'');
+        try out.append(allocator, ch);
+    }
+    return out.toOwnedSlice(allocator);
+}
+
+fn deviceName(buf: *[std.posix.HOST_NAME_MAX]u8) []const u8 {
+    return std.posix.gethostname(buf) catch "kobo";
+}
+
+/// Write a consistent snapshot of `db_path` to `snap_path`, replacing any existing file.
+fn writeSnapshot(allocator: std.mem.Allocator, db_path: []const u8, snap_path: []const u8) !void {
+    // VACUUM INTO refuses to overwrite. A leftover file from a killed run would otherwise
+    // make every subsequent sync fail.
+    std.fs.cwd().deleteFile(snap_path) catch |err| switch (err) {
+        error.FileNotFound => {},
+        else => return err,
+    };
+
+    const db_path_z = try allocator.dupeZ(u8, db_path);
+    defer allocator.free(db_path_z);
+
+    var db: ?*c.sqlite3 = null;
+    if (c.sqlite3_open(db_path_z.ptr, &db) != c.SQLITE_OK) {
+        std.debug.print("failed to open database: {s}\n", .{c.sqlite3_errmsg(db)});
+        _ = c.sqlite3_close(db);
+        return error.DatabaseOpenFailed;
+    }
+    // Closed explicitly below rather than by defer: the handle must be released before any
+    // network I/O so a stalled upload can never pin the database, its -wal and its -shm.
+    errdefer _ = c.sqlite3_close(db);
+
+    const escaped = try quoteSqlLiteral(allocator, snap_path);
+    defer allocator.free(escaped);
+    const sql = try std.fmt.allocPrintSentinel(allocator, "VACUUM INTO '{s}';", .{escaped}, 0);
+    defer allocator.free(sql);
+
+    var errmsg: [*c]u8 = null;
+    if (c.sqlite3_exec(db, sql.ptr, null, null, &errmsg) != c.SQLITE_OK) {
+        std.debug.print("VACUUM INTO failed: {s}\n", .{errmsg});
+        c.sqlite3_free(errmsg);
+        return error.SnapshotFailed;
+    }
+
+    if (c.sqlite3_close(db) != c.SQLITE_OK) return error.DatabaseCloseFailed;
+}
+
+pub fn run(
+    allocator: std.mem.Allocator,
+    db_path: []const u8,
+    endpoint: ?[]const u8,
+    snap_path: []const u8,
+    timeout_secs: u64,
+) !void {
+    const guard = try std.Thread.spawn(.{}, watchdog, .{timeout_secs});
+    guard.detach();
+
+    try writeSnapshot(allocator, db_path, snap_path);
+    // Database handle is released from here on.
+
+    const payload = blk: {
+        const file = try std.fs.cwd().openFile(snap_path, .{});
+        defer file.close();
+        // Deliberately NOT File.stat()/getEndPos(): both issue the `statx` syscall, which
+        // does not exist before Linux 4.11. The Kobo Elipsa runs 4.9.77, where it returns
+        // ENOSYS and surfaces as a bare `error.Unexpected`. readToEndAlloc with no size
+        // hint grows the buffer using plain read() calls instead.
+        break :blk try file.readToEndAlloc(allocator, max_snapshot_bytes);
+    };
+    defer allocator.free(payload);
+
+    var sha: [64]u8 = undefined;
+    sha256Hex(payload, &sha);
+
+    var host_buf: [std.posix.HOST_NAME_MAX]u8 = undefined;
+    const device = deviceName(&host_buf);
+
+    std.debug.print("snapshot: {d} bytes, sha256={s}\n", .{ payload.len, sha });
+
+    // Without an endpoint this is the inspect/debug path, so the file is deliberately
+    // left in place. Cleanup is registered only once we know we're uploading.
+    const url = endpoint orelse {
+        std.debug.print("no endpoint given; snapshot left at {s}\n", .{snap_path});
+        return;
+    };
+    defer std.fs.cwd().deleteFile(snap_path) catch {};
+
+    std.debug.print("uploading to {s} ...\n", .{url});
+
+    var client: std.http.Client = .{ .allocator = allocator };
+    defer client.deinit();
+
+    const result = try client.fetch(.{
+        .location = .{ .uri = try std.Uri.parse(url) },
+        .method = .POST,
+        .payload = payload,
+        .headers = .{ .content_type = .{ .override = "application/octet-stream" } },
+        .extra_headers = &.{
+            .{ .name = "x-kobo-sha256", .value = &sha },
+            .{ .name = "x-kobo-device", .value = device },
+        },
+    });
+
+    const status = @intFromEnum(result.status);
+    if (status < 200 or status >= 300) {
+        std.debug.print("server returned HTTP {d}\n", .{status});
+        return error.HttpRequestFailed;
+    }
+    std.debug.print("ok: uploaded snapshot (HTTP {d})\n", .{status});
+}
+
+test "quoteSqlLiteral doubles single quotes" {
+    const a = std.testing.allocator;
+
+    const plain = try quoteSqlLiteral(a, "/tmp/snap.sqlite");
+    defer a.free(plain);
+    try std.testing.expectEqualStrings("/tmp/snap.sqlite", plain);
+
+    const tricky = try quoteSqlLiteral(a, "/tmp/o'brien.sqlite");
+    defer a.free(tricky);
+    try std.testing.expectEqualStrings("/tmp/o''brien.sqlite", tricky);
+}
+
+test "sha256Hex matches known digest" {
+    var out: [64]u8 = undefined;
+    sha256Hex("abc", &out);
+    try std.testing.expectEqualStrings(
+        "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad",
+        &out,
+    );
+}