db: checkpoint the engine, and open from it
`Engine.checkpoint()` publishes the current state: commit first, then snapshot the catalog under the catalog lock, then validate the snapshot against an unchanged `seq` under the log lock before publishing -- the same bounded-retry shape compaction has always used. The crash-recovery invariant (PLAN D6) reduces to that ordering, and it is asserted: `snapshot_seq <= committed_seq`. The catalog holds what the pages cannot say for themselves: db and collection names, slab extents and tails, and for each index its spec, tree position, overflow extents and id->page table. Written wholesale into fresh pages each time, never mutated in place, so the previous copy stays valid under the previous watermark until the new one switches over -- untearable by construction, which is why there is no incremental update path. Every read is bounds-checked, because the bytes come off disk and a scrambled catalog must produce an error the caller can fall back from. `Log.replay` takes a `from_seq` and skips below it before the BSON parse. The walk still visits every block, because that is what leaves `end_pos` correct for the next append; making opens *fast* is the job of truncating the log, next. A failed catalog load warns, discards what it loaded, and replays the log in full. The log is untouched at this commit, so that fallback is real rather than aspirational -- which is the reason to land this before truncation. -- The docs hashmap is deliberately *not* in the catalog. It is still the authoritative _id lookup, but putting it there means writing a format the commit that drops it would only delete again; it is rebuilt by walking the `_id_` tree, which the data file already holds. -- One real bug, and it is the interesting part. Replay does not maintain index entries -- it puts documents in place and lets `build_all_indexes` bulk-pack afterwards, which is O(n log n) once rather than per record. After a checkpoint that is wrong: the indexes arrive already populated, `rebuild_index` skips a non-empty one by design, and every record replayed on top was invisible to every index. The symptom was a document present in the collection and absent from `_id_` -- which, once the hashmap goes, means simply absent. Replay now maintains entries when it opened from a checkpoint, and keeps the bulk path for a full one. `Engine.seq` is restored, which it never was: it restarted at 0 on every open. The mutation for it is *not* covered and the test says so rather than implying otherwise -- the sequence is seeded from the watermark, so it only drifts by the records replayed on top, and the catalog carries those same records in every sequence a unit test can reach. Observing the drift needs a crash between a duplicate-sequence append and the checkpoint that would have captured it. The line stays because a log without monotonic sequences has no total order.
This commit is contained in:
@@ -219,7 +219,14 @@ pub const Log = struct {
|
||||
}
|
||||
|
||||
/// Replay all valid records from the beginning of the file.
|
||||
pub fn replay(self: *Log, ctx: *anyopaque, callback: ReplayFn) !void {
|
||||
/// Replay every record with `seq > from_seq`. Zero replays everything, which
|
||||
/// is what an engine with no checkpoint does.
|
||||
///
|
||||
/// The walk still visits every block regardless, because that is what leaves
|
||||
/// `end_pos` correct for the next append. Skipping is about not *applying*
|
||||
/// records the data file already contains; making opens fast is the job of
|
||||
/// truncating the log once a checkpoint covers it.
|
||||
pub fn replay(self: *Log, ctx: *anyopaque, callback: ReplayFn, from_seq: u64) !void {
|
||||
var decomp: std.ArrayListUnmanaged(u8) = .empty;
|
||||
defer decomp.deinit(self.gpa);
|
||||
var pos: u64 = file_header_len;
|
||||
@@ -275,7 +282,7 @@ pub const Log = struct {
|
||||
|
||||
var idx: usize = 0;
|
||||
while (idx < decomp.items.len) {
|
||||
idx += try self.parse_record(decomp.items[idx..], pos + idx, ctx, callback);
|
||||
idx += try self.parse_record(decomp.items[idx..], pos + idx, ctx, callback, from_seq);
|
||||
}
|
||||
pos += total;
|
||||
self.end_pos = pos;
|
||||
@@ -292,6 +299,7 @@ pub const Log = struct {
|
||||
pos: u64,
|
||||
ctx: *anyopaque,
|
||||
callback: ReplayFn,
|
||||
from_seq: u64,
|
||||
) !usize {
|
||||
if (bytes.len < 4) return error.InvalidLog;
|
||||
const total: u32 = std.mem.readInt(u32, bytes[0..4], .little);
|
||||
@@ -312,6 +320,9 @@ pub const Log = struct {
|
||||
|
||||
var idx: usize = header_len - 4;
|
||||
const seq: u64 = std.mem.readInt(u64, payload[8..16], .little);
|
||||
// Below the checkpoint: the data file already holds its effect. Skipped
|
||||
// before the BSON parse, which is the expensive part.
|
||||
if (seq <= from_seq) return total;
|
||||
const rtype = payload[16];
|
||||
const db = read_cstring(payload, &idx) orelse return error.InvalidLog;
|
||||
const coll = read_cstring(payload, &idx) orelse return error.InvalidLog;
|
||||
@@ -739,7 +750,7 @@ test "append, replay, torn tail" {
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .seen = &seen, .gpa = gpa };
|
||||
try log.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try log.replay(@ptrCast(&ctx), Ctx.apply, 0);
|
||||
|
||||
try testing.expectEqualSlices(u8, &[_]u8{ record_type_upsert, 1, record_type_delete, 2 }, seen.items);
|
||||
}
|
||||
@@ -783,7 +794,7 @@ test "record larger than the read chunk replays" {
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .count = &count, .gpa = gpa };
|
||||
try log.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try log.replay(@ptrCast(&ctx), Ctx.apply, 0);
|
||||
try testing.expectEqual(@as(usize, 1), count);
|
||||
}
|
||||
|
||||
@@ -832,7 +843,7 @@ test "reject corrupt interior block" {
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .count = &count, .gpa = gpa };
|
||||
try testing.expectError(error.InvalidLog, log2.replay(@ptrCast(&ctx), Ctx.apply));
|
||||
try testing.expectError(error.InvalidLog, log2.replay(@ptrCast(&ctx), Ctx.apply, 0));
|
||||
}
|
||||
|
||||
test "torn tail truncates cleanly and appends overwrite it" {
|
||||
@@ -878,7 +889,7 @@ test "torn tail truncates cleanly and appends overwrite it" {
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .seen = &seen, .gpa = gpa };
|
||||
try log2.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try log2.replay(@ptrCast(&ctx), Ctx.apply, 0);
|
||||
try testing.expectEqualSlices(u8, &[_]u8{1}, seen.items);
|
||||
|
||||
// A new append overwrites from the replay end and replays cleanly.
|
||||
@@ -888,7 +899,7 @@ test "torn tail truncates cleanly and appends overwrite it" {
|
||||
var log3 = try Log.open(gpa, io, path);
|
||||
defer log3.close();
|
||||
ctx.seen = &seen;
|
||||
try log3.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try log3.replay(@ptrCast(&ctx), Ctx.apply, 0);
|
||||
try testing.expectEqualSlices(u8, &[_]u8{ 1, 3 }, seen.items);
|
||||
}
|
||||
|
||||
@@ -938,7 +949,7 @@ test "Log.create discards a leftover file; Log.open keeps it" {
|
||||
var kept = try Log.open(gpa, io, path);
|
||||
defer kept.close();
|
||||
try testing.expect(try kept.file.length(io) > file_header_len);
|
||||
try kept.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try kept.replay(@ptrCast(&ctx), Ctx.apply, 0);
|
||||
try testing.expectEqualSlices(u8, &[_]u8{ 1, 2, 3 }, seen.items);
|
||||
}
|
||||
|
||||
@@ -959,6 +970,6 @@ test "Log.create discards a leftover file; Log.open keeps it" {
|
||||
seen.clearRetainingCapacity();
|
||||
var reopened = try Log.open(gpa, io, path);
|
||||
defer reopened.close();
|
||||
try reopened.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try reopened.replay(@ptrCast(&ctx), Ctx.apply, 0);
|
||||
try testing.expectEqualSlices(u8, &[_]u8{9}, seen.items);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user