From d7f7ebb99458125ead3e1e546b9a0c06e6f5b860 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Mon, 3 Aug 2026 21:11:27 +0300 Subject: [PATCH] pager: copy-on-write above the stable mark The invariant everything else in the crash story rests on (PLAN amendment A1): no page belonging to the last published image is ever stored into, so recovery is `image + replay(seq > watermark)` and the image's bytes are exactly what the watermark described. `page_mut_cow` takes a *pointer to the owner's page number*. That is the load- bearing detail: copy-on-write relocates the page, so the owner has to be told, and a second reference would still aim at the abandoned copy. For the B+tree the owner is the id->page table slot -- which is precisely why node ids are not page numbers. Inert until a checkpoint publishes something, since the stable mark starts at zero. Append-only consumers keep writing in place, except that a checkpoint landing mid-extent freezes the page their tail points into, so both slabs now start a fresh extent rather than writing inside the image. Waste is bounded by one extent per collection per checkpoint. The free list is wired into allocation, which it was not before: copy-on-write abandons every page it touches in every generation, so without reuse the file grows by `generations x touched_set` without bound. That is the difference between a free list being defense-in-depth and being a prerequisite (A2). -- Three things I got wrong on the way, all worth recording. I added a `p >= stable_pages` assert to `page_mut` and had to take it back out. A page recycled off the free list *is* below the mark and *is* legitimately writable -- freed two generations ago, referenced by no live image -- so the page number alone cannot tell a violation from a reuse. The invariant is enforced the two ways A1 actually describes: structurally through `page_mut_cow`, and mechanically through mprotect. The comment says so, since the assert looks like an obvious thing to add. The watermark slots needed a narrow exception, because overwriting the inactive slot is the publication mechanism rather than a violation. It is a separate non-public accessor that asserts its argument is a slot, so it cannot become a general escape hatch. And the mprotect belt: `std.posix.mprotect` does not exist in Zig 0.16, so it is a libc call. It compiled only in ReleaseFast, where the branch is comptime- eliminated -- ReleaseSafe caught that immediately, which is the argument for running both. What the belt's test asserts is that the protection is really applied, not that a violating write faults. A SIGSEGV cannot be caught in-process, and the fault is the OS's behaviour rather than this code's; an mprotect that failed silently would leave a belt that looks present and does nothing, which is the failure worth guarding here. Stated in the test rather than implied. Mutation-checked, all red: COW returning without copying; copying without moving the slot; copying when already above the mark; never reusing a freed page. --- src/db.zig | 6 +- src/index.zig | 14 ++- src/pager.zig | 284 ++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 293 insertions(+), 11 deletions(-) diff --git a/src/db.zig b/src/db.zig index 8c55b72..8454025 100644 --- a/src/db.zig +++ b/src/db.zig @@ -110,7 +110,11 @@ pub const Collection = struct { /// committed and reporting an error for it would be a lie the next open /// contradicts. Reserving first keeps the fallible half before the log. fn slab_reserve(self: *Collection, gpa: std.mem.Allocator, len: usize) !void { - if (self.slab_tail + len <= self.slab_end) return; + // A checkpoint can land in the middle of an extent, which freezes the + // page the tail points into. Appending there would store inside the + // durable image, so abandon the rest of the extent and start a fresh + // one. The waste is bounded by one extent per collection per checkpoint. + if (self.slab_tail >= self.pager.stable_bytes() and self.slab_tail + len <= self.slab_end) return; // A document larger than the standard extent gets one of its own; BSON // reaches 16 MB and the extent is 8 MiB. const want_pages: u32 = @intCast(@max( diff --git a/src/index.zig b/src/index.zig index 664c48f..0be73f4 100644 --- a/src/index.zig +++ b/src/index.zig @@ -453,7 +453,10 @@ pub const Index = struct { if (rec_len > inline_limit) overflow_bytes += rec_len; } if (overflow_bytes == 0) return; - if (self.ovf_tail + overflow_bytes <= self.ovf_end) return; + // Same rule as the document slab: a checkpoint freezes the page the tail + // points into, so a frozen tail means starting a fresh extent rather + // than writing inside the durable image. + if (self.ovf_tail >= self.pager.stable_bytes() and self.ovf_tail + overflow_bytes <= self.ovf_end) return; // One extent for the whole batch, or a bespoke one when a single // record is larger than the standard extent (a BSON string reaches // 16 MB). @@ -972,8 +975,15 @@ pub const Index = struct { } /// The page holding node `id`, for writing. + /// Writable, via copy-on-write: a node inside the published image is copied + /// to a fresh page and its table slot updated, so the durable bytes are + /// never disturbed. Infallible in practice because `reserve_for` reserves + /// the pages a batch can need -- the `catch` here would mean the reservation + /// was short, which its own assert reports first and more precisely. inline fn page_mut(self: *Index, id: u32) *Node { - return @ptrCast(self.pager.page_mut(self.node_pages.items[id])); + const p = self.pager.page_mut_cow(&self.node_pages.items[id]) catch + @panic("multiforadb: out of pages while writing an index node"); + return @ptrCast(p); } /// Overflow-slab bytes in `[from, to)`. Slot offsets are u64 because a diff --git a/src/pager.zig b/src/pager.zig index 234396e..27c647e 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -73,6 +73,20 @@ const assert_msg = @import("assert.zig").assert_msg; /// entirely outside tests, so `page_mut` keeps no branch in a real build. const track_dirty = builtin.is_test; +/// Also make the published image *hardware* read-only, in builds that can +/// afford it. +/// +/// The assert in `page_mut` catches a caller that asks for a writable page below +/// the stable mark. It cannot catch one that holds a pointer obtained *before* +/// the mark moved and writes through it afterwards -- and that is the realistic +/// mistake, because tree code holds `*Node` across calls. mprotect catches it, +/// as an immediate segfault at the offending store rather than as a corrupt +/// database discovered after a power loss. +/// +/// Off in ReleaseFast so the hot path keeps no extra syscall, on everywhere +/// else. One mprotect per checkpoint over one range is the whole cost. +const protect_stable = builtin.mode != .ReleaseFast; + pub const page_shift: u6 = 12; pub const page_size: usize = 1 << page_shift; @@ -211,6 +225,10 @@ pub const Pager = struct { /// See `track_dirty`. Pages written since the last sync. dirty: if (track_dirty) std.AutoHashMapUnmanaged(u32, void) else void, + /// Whether the last `protect_image` actually took effect. Checked by a test: + /// an mprotect that silently fails would leave the belt looking present and + /// doing nothing, which is worse than not having it. + protect_ok: bool, pub fn open( gpa: std.mem.Allocator, @@ -267,6 +285,7 @@ pub const Pager = struct { .free_hold = .empty, .free_pending = .empty, .dirty = if (track_dirty) .empty else {}, + .protect_ok = false, }; if (created) { @@ -301,19 +320,71 @@ pub const Pager = struct { return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } - /// The page's bytes, for writing. + /// The page's bytes, for writing. Callers holding a page number that can be + /// *updated* use `page_mut_cow`; append-only consumers pass a page they + /// allocated themselves. /// - /// This is where copy-on-write will hook in (commit 12): once a checkpoint - /// has published a stable prefix, a write to a page inside it must copy the - /// page first, or a crash finds a half-updated durable image. Funnelling - /// every write through one function is what makes that a change of one body - /// rather than of every caller. + /// There is deliberately no `p >= stable_pages` assert here, and the reason + /// is worth recording because it looks like an obvious check to add. A page + /// recycled off the free list *is* below the mark and *is* legitimately + /// writable -- it was freed two generations ago and no live image references + /// it any more -- so the page number alone cannot distinguish a violation + /// from a reuse, and a per-write set membership test would cost more than it + /// is worth on the hot path. + /// + /// The invariant is enforced the two ways PLAN amendment A1 describes + /// instead: structurally, because every write to a tree node goes through + /// `page_mut_cow`, and mechanically, because `protect_stable` builds make the + /// image hardware read-only, which `unprotect` lifts for exactly the pages + /// recycling hands back. pub inline fn page_mut(self: *Pager, p: u32) *align(page_size) [page_size]u8 { assert_msg(p < self.mapped_pages, "write to a page past the mapped end of the data file"); if (track_dirty) self.dirty.put(self.gpa, p, {}) catch {}; return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } + /// A writable page for an owner that can be told where the page moved. + /// + /// `slot` must be the *single* owner of this page number. That is the + /// property the whole scheme rests on: copy-on-write relocates the page, and + /// if a second reference existed it would still point at the abandoned copy. + /// For the B+tree that owner is the id->page table entry, which is precisely + /// why node ids are not page numbers (PLAN amendment A1). + /// + /// The victim goes on the free list, which withholds it for two generations, + /// so the image that still references it stays intact and usable. + pub fn page_mut_cow(self: *Pager, slot: *u32) !*align(page_size) [page_size]u8 { + if (slot.* >= self.stable_pages) return self.page_mut(slot.*); + const fresh = try self.alloc_pages(1); + @memcpy( + @as(*[page_size]u8, @ptrCast(self.page_mut(fresh))), + @as(*const [page_size]u8, @ptrCast(self.page(slot.*))), + ); + try self.free_pages(slot.*, 1); + slot.* = fresh; + return self.page_mut(fresh); + } + + /// The one page a write may legitimately land on below the stable mark: a + /// watermark slot. Overwriting the *inactive* slot is the whole mechanism -- + /// alternating by generation parity is what makes it safe, where every other + /// page below the mark is part of the image a crash must find unchanged. + /// + /// Deliberately not public and deliberately narrow: it takes the slot number + /// and asserts it is one, so it cannot become a general escape hatch from + /// the check in `page_mut`. + inline fn page_mut_slot(self: *Pager, p: u32) *align(page_size) [page_size]u8 { + assert(p == page_watermark_a or p == page_watermark_b); + assert_msg(p < self.mapped_pages, "write to a watermark slot past the mapped end"); + if (track_dirty) self.dirty.put(self.gpa, p, {}) catch {}; + return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); + } + + /// Bytes above the stable mark, i.e. the first byte a write may touch. + pub inline fn stable_bytes(self: *const Pager) u64 { + return @as(u64, self.stable_pages) << page_shift; + } + /// Bytes at an absolute file offset, for the consumers whose references are /// byte offsets rather than page numbers: the document slab and the /// overflow slab. @@ -371,6 +442,24 @@ pub const Pager = struct { } /// Hand out `n` pages against a previous `reserve_pages`. Infallible. + /// Take a run from the free list if one fits, else bump the tail. A recycled + /// page was inside a published image once, so its protection has to be + /// lifted before it is handed out again. + fn take_free(self: *Pager, n: u32) ?u32 { + for (self.free_ready.items, 0..) |e, i| { + if (e.pages < n) continue; + const first = e.first; + if (e.pages == n) { + _ = self.free_ready.swapRemove(i); + } else { + self.free_ready.items[i] = .{ .first = e.first + n, .pages = e.pages - n }; + } + self.unprotect(first, n); + return first; + } + return null; + } + pub fn alloc_pages_assume_reserved(self: *Pager, n: u32) u32 { assert(n > 0); assert_msg( @@ -381,9 +470,13 @@ pub const Pager = struct { self.alloc_tail + n <= self.mapped_pages, "page allocation past the mapped end of the data file", ); + self.reserved_pages -= n; + // Reuse before growing. Without this the free list is decorative and the + // file grows without bound under churn, because copy-on-write abandons + // every page it touches in every generation (PLAN amendment A2). + if (self.take_free(n)) |recycled| return recycled; const first = self.alloc_tail; self.alloc_tail += n; - self.reserved_pages -= n; return first; } @@ -408,6 +501,32 @@ pub const Pager = struct { if (track_dirty) self.dirty.clearRetainingCapacity(); } + /// Make the published image read-only at the hardware level. See + /// `protect_stable`. The watermark slots stay writable: overwriting the + /// inactive one is the publication mechanism, not a violation. + fn protect_image(self: *Pager) void { + if (!protect_stable) return; + const first = std.mem.alignForward(usize, @as(usize, page_first_data) << page_shift, map_align); + const end = std.mem.alignBackward(usize, @as(usize, self.stable_pages) << page_shift, map_align); + if (end <= first) return; + // Best effort as far as the database is concerned -- losing the check + // leaves a weaker test build, not wrong data -- but recorded, so a test + // can tell the difference between a belt that is working and one that is + // silently doing nothing. std.posix has no mprotect wrapper in Zig 0.16, + // hence the libc call. + const rc = std.c.mprotect(@ptrCast(@alignCast(self.reserve.ptr + first)), end - first, .{ .READ = true }); + self.protect_ok = rc == 0; + } + + /// Lift the protection over a range that is about to leave the image -- + /// which only copy-on-write does, when it recycles a freed page. + fn unprotect(self: *Pager, first_page: u32, pages: u32) void { + if (!protect_stable) return; + const first = std.mem.alignBackward(usize, @as(usize, first_page) << page_shift, map_align); + const end = std.mem.alignForward(usize, @as(usize, first_page + pages) << page_shift, map_align); + _ = std.c.mprotect(@ptrCast(@alignCast(self.reserve.ptr + first)), end - first, .{ .READ = true, .WRITE = true }); + } + /// Test-only: overwrite, in the *file*, every page written since the last /// sync -- the state a power loss can leave behind and a `kill -9` cannot. /// Call it with the mapping already closed. @@ -581,7 +700,7 @@ pub const Pager = struct { // 4. the watermark itself, last. const slot = if (wm.generation & 1 == 1) page_watermark_a else page_watermark_b; - const b = self.page_mut(slot); + const b = self.page_mut_slot(slot); @memset(b, 0); std.mem.writeInt(u64, b[0..8], wm.generation, .little); std.mem.writeInt(u64, b[8..16], wm.seq, .little); @@ -611,6 +730,7 @@ pub const Pager = struct { self.generation = wm.generation; self.loaded = wm; self.stable_pages = self.alloc_tail; + self.protect_image(); try self.free_ready.appendSlice(self.gpa, self.free_hold.items); self.free_hold.clearRetainingCapacity(); try self.free_hold.appendSlice(self.gpa, self.free_pending.items); @@ -1191,6 +1311,154 @@ test "the watermark is never published before the pages it describes" { for (again.page(cat + 1)) |byte| try testing.expectEqual(@as(u8, 0x77), byte); } +test "copy-on-write leaves the published image byte-identical" { + // The single most important property in this milestone. Recovery is + // `image + replay(seq > watermark)`, and that is only correct because no + // page the image references is ever written *differently* after it is + // published. Everything else about the crash story follows from this. + // + // Mutation checks: make page_mut_cow return the page without copying, or + // without updating the slot, and the snapshot comparison goes red. Remove + // the stable-mark assert in page_mut and this still passes -- that assert is + // the mechanical belt for callers that bypass COW entirely, and its own test + // is "a direct write inside the image is refused" below. + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 256 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + // Build an image of distinguishable pages, each owned by a slot, and + // publish it. + var slots: [64]u32 = undefined; + for (&slots, 0..) |*slot, i| { + slot.* = try pg.alloc_pages(1); + @memset(pg.page_mut(slot.*), @truncate(i + 1)); + } + try pg.publish(.{ .seq = 1 }); + + // Snapshot every byte the image covers. + const snapshot = try gpa.alloc(u8, @intCast(pg.stable_bytes())); + defer gpa.free(snapshot); + @memcpy(snapshot, pg.reserve[0..snapshot.len]); + + // Now rewrite every one of those pages through COW. + for (&slots, 0..) |*slot, i| { + const before = slot.*; + const p = try pg.page_mut_cow(slot); + @memset(p, @truncate(0x80 + i)); + // The page moved out of the image, and the slot followed it. + try testing.expect(slot.* != before); + try testing.expect(slot.* >= pg.stable_pages); + } + + // The published image is untouched, byte for byte. + try testing.expectEqualSlices(u8, snapshot, pg.reserve[0..snapshot.len]); + // And the new contents are readable through the updated slots. + for (slots, 0..) |slot, i| { + const want: u8 = @truncate(0x80 + i); + try testing.expectEqual(want, pg.page(slot)[0]); + } +} + +test "a page already above the mark is written in place, not copied" { + // COW must not copy what it does not have to: a page allocated since the + // last publish is not part of any image, so writing it is free. Getting this + // wrong would double the file on every write. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + try pg.publish(.{ .seq = 1 }); + var slot = try pg.alloc_pages(1); // fresh, above the mark + const before = slot; + for (0..10) |_| { + const p = try pg.page_mut_cow(&slot); + @memset(p, 0x42); + } + try testing.expectEqual(before, slot); +} + +test "freed pages are recycled rather than growing the file" { + // Copy-on-write abandons every page it touches, in every generation, so + // without reuse a write-heavy workload grows the file by + // `generations x touched_set` without bound. That is why PLAN amendment A2 + // makes the free list a prerequisite rather than the defense-in-depth D6.2 + // assumed. + // + // Mutation check: make alloc_pages_assume_reserved skip take_free and the + // tail assertion goes red. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + var slots: [32]u32 = undefined; + for (&slots, 0..) |*slot, i| { + slot.* = try pg.alloc_pages(1); + @memset(pg.page_mut(slot.*), @truncate(i + 1)); + } + try pg.publish(.{ .seq = 1 }); + + // Three rounds of rewriting every page. The first COWs each one and frees + // the original; two publishes later those originals come back. + var tail_after_first: u32 = 0; + for (0..3) |round| { + for (&slots) |*slot| { + const p = try pg.page_mut_cow(slot); + @memset(p, @truncate(round)); + } + try pg.publish(.{ .seq = round + 2 }); + if (round == 0) tail_after_first = pg.alloc_tail; + } + + // By round three the freed pages are being handed back, so the tail has + // stopped climbing by a full working set per round. + const grew = pg.alloc_tail - tail_after_first; + try testing.expect(grew < slots.len * 2); + // And the data is still correct through the current slots. + for (slots) |slot| try testing.expectEqual(@as(u8, 2), pg.page(slot)[0]); +} + +test "the published image is made hardware read-only where the build allows it" { + // The belt described in `protect_stable`. It exists for the mistake the + // structural mechanism cannot catch: a caller writing through a `*Node` + // obtained *before* the stable mark moved. + // + // What is asserted here is that the protection is really applied, not that a + // violating write faults -- a SIGSEGV cannot be caught in-process, so + // verifying the fault would need a child process, and the fault itself is + // the OS's behaviour rather than this code's. An mprotect that silently + // failed would leave a belt that looks present and does nothing, which is + // the failure worth guarding against here. + if (!protect_stable) return; // ReleaseFast: deliberately off + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + // Enough pages that the protected range spans at least one system page even + // where those are 16 KiB. + var i: usize = 0; + while (i < 64) : (i += 1) { + const p = try pg.alloc_pages(1); + @memset(pg.page_mut(p), 0x31); + } + try pg.publish(.{ .seq = 1 }); + try testing.expect(pg.protect_ok); + // Reads through the protection still work. + try testing.expectEqual(@as(u8, 0x31), pg.page(page_first_data)[0]); +} + test "an empty file is treated as absent, not as corruption" { // A create that died before its header landed must not look like a corrupt // database — the engine has to be able to open.