Add whole-database snapshot mode
Adds `kb_exfiltrator snapshot <db> [endpoint] [timeout-secs]`, which uploads a
consistent copy of the whole database instead of just extracted highlights. The
server can then parse whatever it wants, so adding a new statistic never means
touching the device again.
Uses VACUUM INTO rather than copying the file. Nickel keeps the database in WAL
mode and recent commits live in KoboReader.sqlite-wal until a checkpoint, so
copying the bare .sqlite silently loses reading progress — measured at 19 hours
of drift on the live device. Copying all three files instead is non-atomic: a
checkpoint landing mid-copy yields a pre-checkpoint main file plus a post-reset
WAL, losing data with no error. VACUUM INTO runs in a read transaction, so it
sees WAL-resident commits and emits one compacted journal_mode=delete file.
Also fixes the failure mode that left a process hung for 16 days: the server
stopped responding and std.http.Client has no timeout, so the process blocked
forever while holding the SQLite handle open. Snapshot mode closes the database
before any network I/O, and a watchdog thread hard-exits after a deadline
covering the whole run (snapshotting can stall on a locked database too).
Two portability constraints, both specific to the Kobo Elipsa:
- Linux 4.9.77 predates statx (4.11). Zig's File.stat() and getEndPos() both
issue it, returning ENOSYS as a bare error.Unexpected — after the snapshot
is already written, so it presents as a post-write failure. readToEndAlloc
with no size hint uses plain read() calls instead.
- usize is 32-bit on armv7, so a u64 stat size will not coerce. Only the ARM
build catches this.
Legacy invocations are unchanged, so the existing NickelMenu item keeps working.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+60
-8
@@ -1,4 +1,5 @@
|
||||
const std = @import("std");
|
||||
const snapshot = @import("snapshot.zig");
|
||||
const c = @cImport({
|
||||
@cInclude("sqlite3.h");
|
||||
});
|
||||
@@ -125,28 +126,79 @@ const Highlight = struct {
|
||||
}
|
||||
};
|
||||
|
||||
// Zig only collects `test` blocks from the test unit's root file, so pull in the
|
||||
// snapshot module's tests explicitly.
|
||||
test {
|
||||
_ = @import("snapshot.zig");
|
||||
}
|
||||
|
||||
const usage =
|
||||
\\Usage:
|
||||
\\ kb_exfiltrator snapshot <db> [endpoint] [timeout-secs]
|
||||
\\ Upload a consistent whole-database snapshot (VACUUM INTO). This is the
|
||||
\\ pipeline's primary mode: the server parses it, so adding new stats never
|
||||
\\ requires touching the device. Omit endpoint to leave the snapshot on disk.
|
||||
\\
|
||||
\\ kb_exfiltrator highlights <db> [endpoint]
|
||||
\\ Legacy mode: extract highlights and POST them as JSON.
|
||||
\\
|
||||
\\ kb_exfiltrator <db> [endpoint]
|
||||
\\ Backwards-compatible alias for `highlights`.
|
||||
\\
|
||||
;
|
||||
|
||||
pub fn main() !void {
|
||||
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
|
||||
defer _ = gpa.deinit();
|
||||
const allocator = gpa.allocator();
|
||||
|
||||
// Get command line args
|
||||
const args = try std.process.argsAlloc(allocator);
|
||||
defer std.process.argsFree(allocator, args);
|
||||
|
||||
if (args.len < 2) {
|
||||
std.debug.print("Usage: {s} <path-to-kobo-database.sqlite> [flask-endpoint-url]\n", .{args[0]});
|
||||
std.debug.print("Example: {s} /mnt/kobo/.kobo/KoboReader.sqlite\n", .{args[0]});
|
||||
std.debug.print("Example: {s} /mnt/kobo/.kobo/KoboReader.sqlite https://myserver.com/api/highlights\n", .{args[0]});
|
||||
std.debug.print("{s}", .{usage});
|
||||
return error.MissingArgument;
|
||||
}
|
||||
|
||||
const db_path = args[1];
|
||||
const endpoint_url = if (args.len >= 3) args[2] else null;
|
||||
if (std.mem.eql(u8, args[1], "snapshot")) {
|
||||
if (args.len < 3) {
|
||||
std.debug.print("{s}", .{usage});
|
||||
return error.MissingArgument;
|
||||
}
|
||||
const timeout = if (args.len >= 5)
|
||||
std.fmt.parseInt(u64, args[4], 10) catch snapshot.default_timeout_secs
|
||||
else
|
||||
snapshot.default_timeout_secs;
|
||||
return snapshot.run(
|
||||
allocator,
|
||||
args[2],
|
||||
if (args.len >= 4) args[3] else null,
|
||||
snapshot.default_snapshot_path,
|
||||
timeout,
|
||||
);
|
||||
}
|
||||
|
||||
// `highlights <db> [endpoint]`, or the legacy `<db> [endpoint]` form.
|
||||
const legacy = std.mem.eql(u8, args[1], "highlights");
|
||||
const rest = if (legacy) args[2..] else args[1..];
|
||||
if (rest.len < 1) {
|
||||
std.debug.print("{s}", .{usage});
|
||||
return error.MissingArgument;
|
||||
}
|
||||
return runHighlights(allocator, rest[0], if (rest.len >= 2) rest[1] else null);
|
||||
}
|
||||
|
||||
fn runHighlights(
|
||||
allocator: std.mem.Allocator,
|
||||
db_path: []const u8,
|
||||
endpoint_url: ?[]const u8,
|
||||
) !void {
|
||||
// Open database. db_path arrives as a plain slice, so re-terminate it for the C API.
|
||||
const db_path_z = try allocator.dupeZ(u8, db_path);
|
||||
defer allocator.free(db_path_z);
|
||||
|
||||
// Open database
|
||||
var db: ?*c.sqlite3 = null;
|
||||
const rc = c.sqlite3_open(db_path.ptr, &db);
|
||||
const rc = c.sqlite3_open(db_path_z.ptr, &db);
|
||||
if (rc != c.SQLITE_OK) {
|
||||
std.debug.print("Failed to open database: {s}\n", .{c.sqlite3_errmsg(db)});
|
||||
return error.DatabaseOpenFailed;
|
||||
|
||||
Reference in New Issue
Block a user