From f55e9d02536b2f45943529bed85b58db40580a27 Mon Sep 17 00:00:00 2001 From: Wesley Ray Date: Sat, 1 Aug 2026 18:10:33 -0400 Subject: [PATCH] Add whole-database snapshot mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `kb_exfiltrator snapshot [endpoint] [timeout-secs]`, which uploads a consistent copy of the whole database instead of just extracted highlights. The server can then parse whatever it wants, so adding a new statistic never means touching the device again. Uses VACUUM INTO rather than copying the file. Nickel keeps the database in WAL mode and recent commits live in KoboReader.sqlite-wal until a checkpoint, so copying the bare .sqlite silently loses reading progress — measured at 19 hours of drift on the live device. Copying all three files instead is non-atomic: a checkpoint landing mid-copy yields a pre-checkpoint main file plus a post-reset WAL, losing data with no error. VACUUM INTO runs in a read transaction, so it sees WAL-resident commits and emits one compacted journal_mode=delete file. Also fixes the failure mode that left a process hung for 16 days: the server stopped responding and std.http.Client has no timeout, so the process blocked forever while holding the SQLite handle open. Snapshot mode closes the database before any network I/O, and a watchdog thread hard-exits after a deadline covering the whole run (snapshotting can stall on a locked database too). Two portability constraints, both specific to the Kobo Elipsa: - Linux 4.9.77 predates statx (4.11). Zig's File.stat() and getEndPos() both issue it, returning ENOSYS as a bare error.Unexpected — after the snapshot is already written, so it presents as a post-write failure. readToEndAlloc with no size hint uses plain read() calls instead. - usize is 32-bit on armv7, so a u64 stat size will not coerce. Only the ARM build catches this. Legacy invocations are unchanged, so the existing NickelMenu item keeps working. Co-Authored-By: Claude Opus 5 --- src/main.zig | 68 +++++++++++++++--- src/snapshot.zig | 183 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 src/snapshot.zig diff --git a/src/main.zig b/src/main.zig index 5c7709d..077529e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const snapshot = @import("snapshot.zig"); const c = @cImport({ @cInclude("sqlite3.h"); }); @@ -125,28 +126,79 @@ const Highlight = struct { } }; +// Zig only collects `test` blocks from the test unit's root file, so pull in the +// snapshot module's tests explicitly. +test { + _ = @import("snapshot.zig"); +} + +const usage = + \\Usage: + \\ kb_exfiltrator snapshot [endpoint] [timeout-secs] + \\ Upload a consistent whole-database snapshot (VACUUM INTO). This is the + \\ pipeline's primary mode: the server parses it, so adding new stats never + \\ requires touching the device. Omit endpoint to leave the snapshot on disk. + \\ + \\ kb_exfiltrator highlights [endpoint] + \\ Legacy mode: extract highlights and POST them as JSON. + \\ + \\ kb_exfiltrator [endpoint] + \\ Backwards-compatible alias for `highlights`. + \\ +; + pub fn main() !void { var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); const allocator = gpa.allocator(); - // Get command line args const args = try std.process.argsAlloc(allocator); defer std.process.argsFree(allocator, args); if (args.len < 2) { - std.debug.print("Usage: {s} [flask-endpoint-url]\n", .{args[0]}); - std.debug.print("Example: {s} /mnt/kobo/.kobo/KoboReader.sqlite\n", .{args[0]}); - std.debug.print("Example: {s} /mnt/kobo/.kobo/KoboReader.sqlite https://myserver.com/api/highlights\n", .{args[0]}); + std.debug.print("{s}", .{usage}); return error.MissingArgument; } - const db_path = args[1]; - const endpoint_url = if (args.len >= 3) args[2] else null; + if (std.mem.eql(u8, args[1], "snapshot")) { + if (args.len < 3) { + std.debug.print("{s}", .{usage}); + return error.MissingArgument; + } + const timeout = if (args.len >= 5) + std.fmt.parseInt(u64, args[4], 10) catch snapshot.default_timeout_secs + else + snapshot.default_timeout_secs; + return snapshot.run( + allocator, + args[2], + if (args.len >= 4) args[3] else null, + snapshot.default_snapshot_path, + timeout, + ); + } + + // `highlights [endpoint]`, or the legacy ` [endpoint]` form. + const legacy = std.mem.eql(u8, args[1], "highlights"); + const rest = if (legacy) args[2..] else args[1..]; + if (rest.len < 1) { + std.debug.print("{s}", .{usage}); + return error.MissingArgument; + } + return runHighlights(allocator, rest[0], if (rest.len >= 2) rest[1] else null); +} + +fn runHighlights( + allocator: std.mem.Allocator, + db_path: []const u8, + endpoint_url: ?[]const u8, +) !void { + // Open database. db_path arrives as a plain slice, so re-terminate it for the C API. + const db_path_z = try allocator.dupeZ(u8, db_path); + defer allocator.free(db_path_z); - // Open database var db: ?*c.sqlite3 = null; - const rc = c.sqlite3_open(db_path.ptr, &db); + const rc = c.sqlite3_open(db_path_z.ptr, &db); if (rc != c.SQLITE_OK) { std.debug.print("Failed to open database: {s}\n", .{c.sqlite3_errmsg(db)}); return error.DatabaseOpenFailed; diff --git a/src/snapshot.zig b/src/snapshot.zig new file mode 100644 index 0000000..4f74e90 --- /dev/null +++ b/src/snapshot.zig @@ -0,0 +1,183 @@ +//! Whole-database snapshot mode. +//! +//! Produces a single, internally-consistent copy of KoboReader.sqlite and uploads it. +//! +//! Why a snapshot rather than copying the raw files: Nickel keeps the database in WAL mode, +//! and recent commits live in `KoboReader.sqlite-wal` until a checkpoint. Copying the bare +//! `.sqlite` silently loses hours of reading progress (measured: 19h of drift on a live +//! device). Copying all three files instead is non-atomic — a checkpoint landing mid-copy +//! yields a pre-checkpoint main file plus a post-reset WAL, losing data with no error. +//! +//! `VACUUM INTO` runs inside a read transaction, so it sees WAL-resident commits and emits +//! one compacted `journal_mode=delete` file. One file, one consistent read, no reassembly. + +const std = @import("std"); +const c = @cImport({ + @cInclude("sqlite3.h"); +}); + +pub const default_snapshot_path = "/tmp/kobo_snapshot.sqlite"; +pub const default_timeout_secs: u64 = 120; + +/// Upper bound on a snapshot we're willing to buffer. The live database is ~2.4 MB; this +/// is a guard against pathological growth, not a real expectation. +const max_snapshot_bytes: usize = 256 * 1024 * 1024; + +/// Hard upper bound on the whole run. +/// +/// v1 hung for 16 days: the server stopped responding and `std.http.Client` has no timeout, +/// so the process blocked forever *while holding the SQLite handle open*. This guarantees the +/// process dies instead. Snapshotting can stall too (a locked database), so the watchdog +/// covers the entire run, not just the upload. +fn watchdog(secs: u64) void { + std.Thread.sleep(secs * std.time.ns_per_s); + std.debug.print("timeout: exceeded {d}s, aborting\n", .{secs}); + std.process.exit(2); +} + +fn sha256Hex(data: []const u8, out: *[64]u8) void { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data, &digest, .{}); + const hex = "0123456789abcdef"; + for (digest, 0..) |byte, i| { + out[i * 2] = hex[byte >> 4]; + out[i * 2 + 1] = hex[byte & 0x0F]; + } +} + +/// SQL string literals escape a single quote by doubling it. +fn quoteSqlLiteral(allocator: std.mem.Allocator, s: []const u8) ![]u8 { + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(allocator); + for (s) |ch| { + if (ch == '\'') try out.append(allocator, '\''); + try out.append(allocator, ch); + } + return out.toOwnedSlice(allocator); +} + +fn deviceName(buf: *[std.posix.HOST_NAME_MAX]u8) []const u8 { + return std.posix.gethostname(buf) catch "kobo"; +} + +/// Write a consistent snapshot of `db_path` to `snap_path`, replacing any existing file. +fn writeSnapshot(allocator: std.mem.Allocator, db_path: []const u8, snap_path: []const u8) !void { + // VACUUM INTO refuses to overwrite. A leftover file from a killed run would otherwise + // make every subsequent sync fail. + std.fs.cwd().deleteFile(snap_path) catch |err| switch (err) { + error.FileNotFound => {}, + else => return err, + }; + + const db_path_z = try allocator.dupeZ(u8, db_path); + defer allocator.free(db_path_z); + + var db: ?*c.sqlite3 = null; + if (c.sqlite3_open(db_path_z.ptr, &db) != c.SQLITE_OK) { + std.debug.print("failed to open database: {s}\n", .{c.sqlite3_errmsg(db)}); + _ = c.sqlite3_close(db); + return error.DatabaseOpenFailed; + } + // Closed explicitly below rather than by defer: the handle must be released before any + // network I/O so a stalled upload can never pin the database, its -wal and its -shm. + errdefer _ = c.sqlite3_close(db); + + const escaped = try quoteSqlLiteral(allocator, snap_path); + defer allocator.free(escaped); + const sql = try std.fmt.allocPrintSentinel(allocator, "VACUUM INTO '{s}';", .{escaped}, 0); + defer allocator.free(sql); + + var errmsg: [*c]u8 = null; + if (c.sqlite3_exec(db, sql.ptr, null, null, &errmsg) != c.SQLITE_OK) { + std.debug.print("VACUUM INTO failed: {s}\n", .{errmsg}); + c.sqlite3_free(errmsg); + return error.SnapshotFailed; + } + + if (c.sqlite3_close(db) != c.SQLITE_OK) return error.DatabaseCloseFailed; +} + +pub fn run( + allocator: std.mem.Allocator, + db_path: []const u8, + endpoint: ?[]const u8, + snap_path: []const u8, + timeout_secs: u64, +) !void { + const guard = try std.Thread.spawn(.{}, watchdog, .{timeout_secs}); + guard.detach(); + + try writeSnapshot(allocator, db_path, snap_path); + // Database handle is released from here on. + + const payload = blk: { + const file = try std.fs.cwd().openFile(snap_path, .{}); + defer file.close(); + // Deliberately NOT File.stat()/getEndPos(): both issue the `statx` syscall, which + // does not exist before Linux 4.11. The Kobo Elipsa runs 4.9.77, where it returns + // ENOSYS and surfaces as a bare `error.Unexpected`. readToEndAlloc with no size + // hint grows the buffer using plain read() calls instead. + break :blk try file.readToEndAlloc(allocator, max_snapshot_bytes); + }; + defer allocator.free(payload); + + var sha: [64]u8 = undefined; + sha256Hex(payload, &sha); + + var host_buf: [std.posix.HOST_NAME_MAX]u8 = undefined; + const device = deviceName(&host_buf); + + std.debug.print("snapshot: {d} bytes, sha256={s}\n", .{ payload.len, sha }); + + // Without an endpoint this is the inspect/debug path, so the file is deliberately + // left in place. Cleanup is registered only once we know we're uploading. + const url = endpoint orelse { + std.debug.print("no endpoint given; snapshot left at {s}\n", .{snap_path}); + return; + }; + defer std.fs.cwd().deleteFile(snap_path) catch {}; + + std.debug.print("uploading to {s} ...\n", .{url}); + + var client: std.http.Client = .{ .allocator = allocator }; + defer client.deinit(); + + const result = try client.fetch(.{ + .location = .{ .uri = try std.Uri.parse(url) }, + .method = .POST, + .payload = payload, + .headers = .{ .content_type = .{ .override = "application/octet-stream" } }, + .extra_headers = &.{ + .{ .name = "x-kobo-sha256", .value = &sha }, + .{ .name = "x-kobo-device", .value = device }, + }, + }); + + const status = @intFromEnum(result.status); + if (status < 200 or status >= 300) { + std.debug.print("server returned HTTP {d}\n", .{status}); + return error.HttpRequestFailed; + } + std.debug.print("ok: uploaded snapshot (HTTP {d})\n", .{status}); +} + +test "quoteSqlLiteral doubles single quotes" { + const a = std.testing.allocator; + + const plain = try quoteSqlLiteral(a, "/tmp/snap.sqlite"); + defer a.free(plain); + try std.testing.expectEqualStrings("/tmp/snap.sqlite", plain); + + const tricky = try quoteSqlLiteral(a, "/tmp/o'brien.sqlite"); + defer a.free(tricky); + try std.testing.expectEqualStrings("/tmp/o''brien.sqlite", tricky); +} + +test "sha256Hex matches known digest" { + var out: [64]u8 = undefined; + sha256Hex("abc", &out); + try std.testing.expectEqualStrings( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + &out, + ); +}