From 23b283ff449873c53bad22140ceecdd7f565d853 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 16:33:00 +0300 Subject: [PATCH] db: the slab knows where its dead bytes are Inert on its own: nothing is reclaimed yet and no behaviour changes. What changes is that a collection can now answer *where* its garbage is, which is the precondition for handing any of it back. A slab extent becomes a `SlabRun`: the same two u32s plus a dense array of dead-byte counters, one per `map_align` window. The window is the unit because it is the smallest thing that can be given back at all -- `mark_appendable` refuses an unaligned start and `protect_stable` rounds outwards -- so a counter never exceeds `map_align` and its width follows from that. Two bytes per window is the entire memory cost: 2.7 MB for a 21 GB slab on 16 KiB pages. The shapes that track dead *documents* instead (an interval set, a free-run list) cost gigabytes at the 200-byte document scale of D7.3, and would make `evict_doc` allocate after the write is already committed, which is a failure with nowhere to go. `mark_dead` is therefore infallible, and is called from the two places slab dies: `evict_doc`, after every index entry naming the bytes is gone, and `note_skip`, for what the appender writes off at a checkpoint or when it abandons the tail of an extent. Ordering `mark_dead` last in `evict_doc` is what will make reclamation by counting alone sound -- a window reaches `map_align` dead only once every document touching it has been through there. The run list is now sorted by page number rather than allocation order. That was free while an extent could only be appended to; a recycled run arrives *below* one the collection already owns, and `run_of` is a binary search. Sortedness and non-overlap are asserted at the single point runs enter. Two counters accompany it. `dead_unlocated` holds garbage that has no window: the head and tail of a run outside its whole windows, and -- the larger share -- everything that died before the last restart. It exists so one identity stays exact: sum of window counters + dead_unlocated == slab_used - live_bytes Left side is where, right side is how much; reclamation reads the first and the compaction trigger reads the second, and a drift between them is either a rebuild firing on a clean database or a window handed back with a live document in it. `reclaimed_bytes` is inert here and exists for the churn gate, which cannot otherwise tell "the ratio improved because reclamation worked" from "the ratio improved for another reason". The catalog is byte-identical: still `u32 count, (u32 first, u32 pages)*`, so `catalog_version` stays 1 and there is no second read path. The window map is deliberately not persisted -- an open puts the whole amount into `dead_unlocated` instead. The consequence runs one way: a forgotten dead byte is a window that is not handed back, never a live window that is. Reading inserts sorted rather than appending, so a catalog written before this commit loads into an ordered list. Five tests. The accounting identity across both kinds of death; run edges counted but not placed, driven against `mark_dead` directly since the alignment of a real extent is the allocator's business; documents never straddling a run, over a collection with an oversized document in a run of its own; a run recycled to a lower address keeping the list ordered and findable; and a restart forgetting where the garbage is but not how much. Mutations, each red on its own: drop `mark_dead` from `note_skip` (30824 vs 0), from `evict_doc` (151304 vs 30824), the run-tail branch (16384 vs 12288), the run-head branch (26 tests crash on the underflowed window index), the sorted insert (overlap assert fires), and the `dead_unlocated` line in `read_catalog` (40240 vs 0). 180/180 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz, e2e 49, e2e2 concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/db.zig | 564 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 539 insertions(+), 25 deletions(-) diff --git a/src/db.zig b/src/db.zig index e4808c9..c595778 100644 --- a/src/db.zig +++ b/src/db.zig @@ -36,6 +36,67 @@ const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size; const LogKind = enum { upsert, delete, index_create, index_drop }; +/// Dead bytes in one `map_align` window. The window is the unit of reclamation +/// -- a whole system page is the smallest thing `mark_appendable` and +/// `protect_stable` can hand back -- so a counter never exceeds `map_align`, +/// and the width follows from that rather than being chosen. +/// +/// Two bytes per window on the usual platforms. That is the entire memory cost +/// of knowing where a collection's garbage is: 2.7 MB for a 21 GB slab on +/// 16 KiB pages, 10.6 MB on 4 KiB ones. The alternative shapes -- an interval +/// set, a free-run list -- cost memory proportional to the number of *dead +/// documents*, which for 200-byte documents at that scale is gigabytes, and +/// would make `evict_doc` allocate after the write is already committed. +const WindowDead = if (pgr.map_align <= std.math.maxInt(u16)) u16 else u32; + +/// A run of pages a collection's slab owns, plus where its garbage is. +/// +/// This replaced a bare `pgr.Extent` because an extent can only be given back +/// whole, and a churning collection almost never empties one. A run is split +/// instead: the windows inside it with nothing live left go to the pager and +/// the run becomes two shorter ones. So the list is kept sorted by page number, +/// which makes `run_of` a binary search and the sortedness itself an assert -- +/// allocation order stopped being meaningful once a run could be recycled to a +/// *lower* address than one already owned. +const SlabRun = struct { + first: u32, + pages: u32, + /// The run's start, rounded up to `map_align`: the first offset that begins + /// a whole window. `alloc_pages` works in 4 KiB pages, so a run need not + /// start on a system page. + window_first: u64, + /// Dead bytes per window, `dead[i]` covering + /// `[window_first + i*map_align, +map_align)`. `map_align` means the window + /// holds nothing live and can be handed back. + dead: []WindowDead, + + /// Windows wholly inside the pages `[first, first+pages)`. The bytes + /// outside them -- below `window_first`, and the tail after the last whole + /// window -- are real slab that documents do live in; their garbage is + /// counted in `Collection.dead_unlocated` instead, because it can never be + /// reclaimed on its own. + fn window_count(first: u32, pages: u32) usize { + const from = @as(u64, first) << pgr.page_shift; + const to = from + (@as(u64, pages) << pgr.page_shift); + const wf = std.mem.alignForward(u64, from, pgr.map_align); + const we = std.mem.alignBackward(u64, to, pgr.map_align); + return if (we > wf) @intCast((we - wf) / pgr.map_align) else 0; + } + + fn start(self: SlabRun) u64 { + return @as(u64, self.first) << pgr.page_shift; + } + + fn end(self: SlabRun) u64 { + return (@as(u64, self.first) + self.pages) << pgr.page_shift; + } + + /// One past the last byte covered by a window counter. + fn window_end(self: SlabRun) u64 { + return self.window_first + self.dead.len * pgr.map_align; + } +}; + pub const Collection = struct { /// Documents live as canonical BSON bytes in the data file, in extents this /// collection owns; the map holds each document's offset. Those are @@ -53,8 +114,9 @@ pub const Collection = struct { doc_count: u64, /// The data file this collection's documents live in. pager: *pgr.Pager, - /// Extents owned by this collection's slab, in allocation order. - slab_extents: std.ArrayListUnmanaged(pgr.Extent), + /// Page runs owned by this collection's slab, sorted by page number, each + /// carrying the map of where its dead bytes are. See `SlabRun`. + slab_runs: std.ArrayListUnmanaged(SlabRun), /// Absolute file offset of the next document write, and the end of the /// extent it falls in. slab_tail: u64, @@ -72,6 +134,25 @@ pub const Collection = struct { /// collection so dropping one can move the right amount from the engine's /// live total to its dead total. live_bytes: u64, + /// Garbage this collection knows it has but cannot place in a window: the + /// edges of a run that fall outside any whole window, and -- the larger + /// share -- everything that died before the last restart, since the window + /// map is not persisted. + /// + /// It exists to keep one identity exact: + /// + /// sum of every window counter + dead_unlocated == slab_used - live_bytes + /// + /// Without it the two halves of the accounting would drift apart at every + /// open, and there would be no assert that could tell drift from a lost + /// update. What it costs is only that garbage from before a restart is not + /// reclaimed window-wise; it still arms compaction like any other. + dead_unlocated: u64, + /// Slab handed back to the pager by window reclamation, cumulative for the + /// life of the process. Purely an observation: it is what distinguishes + /// "the ratio improved because reclamation worked" from "the ratio improved + /// for some other reason", which is the only way to read the churn gate. + reclaimed_bytes: u64, /// Secondary indexes (persisted through the log). Heap-allocated, so an /// `*Index` handed out by `find_index` or `create_index` stays valid when /// a sibling index is dropped. Held by value, `orderedRemove` memmoved the @@ -116,11 +197,13 @@ pub const Collection = struct { var self: Collection = .{ .doc_count = 0, .pager = pager, - .slab_extents = .empty, + .slab_runs = .empty, .slab_tail = 0, .slab_end = 0, .slab_used = 0, .live_bytes = 0, + .dead_unlocated = 0, + .reclaimed_bytes = 0, .hold = .{}, .indexes = .empty, .id_index = undefined, @@ -146,6 +229,117 @@ pub const Collection = struct { return null; } + /// Take ownership of a page run, keeping `slab_runs` sorted by page number. + /// The only place a run enters the list, so the sort order and the window + /// map are established together and cannot disagree. + fn insert_run(self: *Collection, gpa: std.mem.Allocator, first: u32, pages: u32) !void { + const dead = try gpa.alloc(WindowDead, SlabRun.window_count(first, pages)); + errdefer gpa.free(dead); + @memset(dead, 0); + var at: usize = 0; + while (at < self.slab_runs.items.len and self.slab_runs.items[at].first < first) at += 1; + // A recycled run must not overlap one this collection already owns: + // that would be the pager handing out pages twice, and the symptom + // would be a document quietly overwritten rather than anything failing. + if (at > 0) { + const prev = self.slab_runs.items[at - 1]; + assert_msg(prev.first + prev.pages <= first, "a slab run overlaps the one below it"); + } + if (at < self.slab_runs.items.len) { + assert_msg(first + pages <= self.slab_runs.items[at].first, "a slab run overlaps the one above it"); + } + try self.slab_runs.insert(gpa, at, .{ + .first = first, + .pages = pages, + .window_first = std.mem.alignForward(u64, @as(u64, first) << pgr.page_shift, pgr.map_align), + .dead = dead, + }); + } + + /// The run holding `off`, or null if no run does. Binary search, which the + /// sorted list is for: `mark_dead` runs once per evicted document, and a + /// collection with a fragmented slab can own thousands of runs. + fn run_of(self: *const Collection, off: u64) ?usize { + const page: u32 = @intCast(off >> pgr.page_shift); + var lo: usize = 0; + var hi: usize = self.slab_runs.items.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + const r = self.slab_runs.items[mid]; + if (page < r.first) { + hi = mid; + } else if (page >= r.first + r.pages) { + lo = mid + 1; + } else { + return mid; + } + } + return null; + } + + /// Record that `[off, off+len)` of slab is garbage. + /// + /// Infallible, and that is the constraint the whole representation was + /// chosen around: the two callers are `evict_doc`, which runs after the + /// log record is already durable, and the appender's skip accounting. An + /// allocation here would be a failure with nowhere to report it. + /// + /// Bytes that fall outside a whole window -- the head of a run before its + /// first window boundary, and the tail after its last -- go to + /// `dead_unlocated`. They are not lost, only unreclaimable on their own. + fn mark_dead(self: *Collection, off: u64, len: u64) void { + if (len == 0) return; + const ri = self.run_of(off) orelse { + assert_msg(false, "dead slab bytes fall outside every run the collection owns"); + unreachable; + }; + const r = &self.slab_runs.items[ri]; + const stop = off + len; + // A document is written inside one extent by construction + // (`slab_reserve` never lets an append cross `slab_end`), so a dead + // range that crosses a run boundary means an offset from a different + // layout -- a stale index entry, which is the failure the layout epoch + // exists to prevent. + assert_msg(stop <= r.end(), "a dead slab range crosses the end of the run holding it"); + var pos = off; + if (pos < r.window_first) { + const n = @min(stop, r.window_first) - pos; + self.dead_unlocated += n; + pos += n; + } + const win_end = r.window_end(); + while (pos < stop and pos < win_end) { + const w: usize = @intCast((pos - r.window_first) / pgr.map_align); + const w_end = r.window_first + (w + 1) * pgr.map_align; + const n = @min(stop, w_end) - pos; + // A window cannot hold more dead bytes than it has bytes. Tripping + // this means the same range was marked twice -- a double eviction, + // or a recycled offset marked against the previous owner's map. + assert_msg(r.dead[w] + n <= pgr.map_align, "a slab window holds more dead bytes than it has"); + r.dead[w] += @intCast(n); + pos += n; + } + if (pos < stop) self.dead_unlocated += stop - pos; + } + + /// Garbage this collection has placed in windows. Walks every window, so it + /// belongs to the reclamation scan and to tests, not to a hot path. + fn dead_located(self: *const Collection) u64 { + var sum: u64 = 0; + for (self.slab_runs.items) |r| { + for (r.dead) |d| sum += d; + } + return sum; + } + + /// Drop the window maps and the run list. The pages themselves are the + /// caller's business -- a drop hands them to the pager, a rebuild has + /// already done so. + fn free_runs(self: *Collection, gpa: std.mem.Allocator) void { + for (self.slab_runs.items) |r| gpa.free(r.dead); + self.slab_runs.clearRetainingCapacity(); + } + /// Append `bytes` to the slab, returning its flat offset. The last /// segment holds up to `slab_segment_size`; a full one starts the next. /// Make room for a document of `len` bytes, so the append that follows @@ -210,7 +404,7 @@ pub const Collection = struct { )); try self.pager.reserve_pages(&self.hold, want_pages); const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages); - try self.slab_extents.append(gpa, .{ .first = first, .pages = want_pages }); + try self.insert_run(gpa, first, want_pages); self.slab_tail = @as(u64, first) << pgr.page_shift; self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift); return skipped; @@ -230,8 +424,15 @@ pub const Collection = struct { /// Two collections churning against a checkpoint every 32 MiB skip up to a /// system page each per checkpoint, and an abandoned extent tail can be /// most of 8 MiB. Counted here, that garbage arms compaction like any other. + /// + /// Skipped slab always starts at the cursor -- all three callers write off + /// the bytes in front of it and then move it -- so this is also where the + /// window map learns about it. That matters more for the abandoned tail + /// than for the round-up: most of 8 MiB of a run is whole windows, dead on + /// arrival, and reclaiming them is free. fn note_skip(self: *Collection, bytes: u64) u64 { self.slab_used += bytes; + self.mark_dead(self.slab_tail, bytes); return bytes; } @@ -619,10 +820,11 @@ pub const Engine = struct { coll.indexes.deinit(self.gpa); // Give the slab's pages back. They become reusable two generations // later, so a fallback to the previous image still finds them intact. - for (coll.slab_extents.items) |e| { - self.pager.free_pages(e.first, e.pages) catch {}; + for (coll.slab_runs.items) |r| { + self.pager.free_pages(r.first, r.pages) catch {}; } - coll.slab_extents.deinit(self.gpa); + coll.free_runs(self.gpa); + coll.slab_runs.deinit(self.gpa); self.gpa.destroy(coll); } @@ -700,6 +902,11 @@ pub const Engine = struct { coll.doc_count -= 1; assert_msg(coll.live_bytes >= old_bytes.len, "evicting a document would underflow the collection's live bytes"); coll.live_bytes -= old_bytes.len; + // Last, and after every index entry naming these bytes is gone. That + // ordering is what makes window reclamation safe to do by counting + // alone: a window only reaches `map_align` dead once every document + // touching it has been through here, so nothing reachable is inside it. + coll.mark_dead(off, old_bytes.len); } /// `bson.encode_key` of a stored document's `_id`, owned by the caller. @@ -1582,16 +1789,21 @@ pub const Engine = struct { try coll.lock.lock(self.io); defer coll.lock.unlock(self.io); - const old_extents = try self.gpa.dupe(pgr.Extent, coll.slab_extents.items); + var old_extents = try self.gpa.alloc(pgr.Extent, coll.slab_runs.items.len); defer self.gpa.free(old_extents); + for (coll.slab_runs.items, 0..) |r, i| old_extents[i] = .{ .first = r.first, .pages = r.pages }; // Fresh slab. The old extents stay allocated until the free list // releases them, two generations on. - coll.slab_extents.clearRetainingCapacity(); + coll.free_runs(self.gpa); coll.slab_tail = 0; coll.slab_end = 0; coll.slab_used = 0; coll.live_bytes = 0; + // The rebuild is the one place the located and unlocated halves are + // both reset: every byte it copies is live, so a fresh slab has no + // garbage to place. Anything the copy skips is marked as it happens. + coll.dead_unlocated = 0; // Walk in _id order, which is also the order the new slab ends up in -- // so a later scan reads it sequentially. @@ -1848,7 +2060,7 @@ pub const Engine = struct { while (coll_it.next()) |ce| { const coll = ce.value_ptr.*; // Everything below this line is written by a collection's own - // writer under its own lock, and `slab_extents` is an ArrayList + // writer under its own lock, and `slab_runs` is an ArrayList // that `slab_reserve` appends to -- so reading it under only the // shared catalog lock could walk a slice a concurrent append had // already reallocated. Lock order is catalog then collection, @@ -1866,10 +2078,14 @@ pub const Engine = struct { "a collection cannot hold more live bytes than it ever appended", ); dead_sum += coll.slab_used - coll.live_bytes; - try put_u32(gpa, out, @intCast(coll.slab_extents.items.len)); - for (coll.slab_extents.items) |e| { - try put_u32(gpa, out, e.first); - try put_u32(gpa, out, e.pages); + // Runs, not extents, but the same two u32s: only the window map + // is new and it is deliberately not persisted (see + // `dead_unlocated`), so `catalog_version` stays 1 and there is + // no second read path to keep working. + try put_u32(gpa, out, @intCast(coll.slab_runs.items.len)); + for (coll.slab_runs.items) |r| { + try put_u32(gpa, out, r.first); + try put_u32(gpa, out, r.pages); } try put_u32(gpa, out, @intCast(coll.indexes.items.len + 1)); try write_index_catalog(gpa, out, &coll.id_index); @@ -1950,12 +2166,21 @@ pub const Engine = struct { if (coll.slab_used < coll.live_bytes) return error.CorruptCatalog; self.live_bytes += coll.live_bytes; self.dead_bytes += coll.slab_used - coll.live_bytes; + // An open knows *that* the collection has garbage but not + // *where*: the window map is rebuilt empty, and the whole + // amount starts out unlocated. The consequence runs one way -- + // a forgotten dead byte is a window that is not handed back, + // never a live window that is. + coll.dead_unlocated = coll.slab_used - coll.live_bytes; const nex = try r.read_u32(); var e: u32 = 0; while (e < nex) : (e += 1) { const first = try r.read_u32(); const pages = try r.read_u32(); - try coll.slab_extents.append(self.gpa, .{ .first = first, .pages = pages }); + // Sorted insert rather than append: a catalog written + // before runs were address-ordered holds them in + // allocation order, and `run_of` is a binary search. + try coll.insert_run(self.gpa, first, pages); } const nix = try r.read_u32(); // Index 0 is the implicit _id_, already created by @@ -2787,8 +3012,8 @@ test "an append after a checkpoint keeps its extent instead of abandoning it" { try engine.commit(); const coll = engine.get_collection("app", "users").?; - try testing.expectEqual(@as(usize, 1), coll.slab_extents.items.len); - const extent_start = @as(u64, coll.slab_extents.items[0].first) << pgr.page_shift; + try testing.expectEqual(@as(usize, 1), coll.slab_runs.items.len); + const extent_start = coll.slab_runs.items[0].start(); try engine.checkpoint(); const tail_at_checkpoint = coll.slab_tail; @@ -2801,7 +3026,7 @@ test "an append after a checkpoint keeps its extent instead of abandoning it" { try engine.insert("app", "users", &second, &env.gen); try engine.commit(); - try testing.expectEqual(@as(usize, 1), coll.slab_extents.items.len); + try testing.expectEqual(@as(usize, 1), coll.slab_runs.items.len); // A page or two for the tree's copy-on-write is expected; a whole slab // extent is the regression this guards against. try testing.expect(engine.pager.alloc_tail < tail_before + slab_extent_pages); @@ -2831,6 +3056,300 @@ test "an append after a checkpoint keeps its extent instead of abandoning it" { } } +/// A document of roughly `size` bytes, so a test can fill extents without +/// writing tens of thousands of records. +fn make_padded(gpa: std.mem.Allocator, id: i32, size: usize) !bson.Document { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const a = arena.allocator(); + const pad = try a.alloc(u8, size); + @memset(pad, 'x'); + const pairs = try a.alloc(bson.Pair, 2); + pairs[0] = .{ .key = try a.dupe(u8, "_id"), .value = .{ .int32 = id } }; + pairs[1] = .{ .key = try a.dupe(u8, "pad"), .value = .{ .string = pad } }; + return .{ .arena = arena, .pairs = pairs }; +} + +/// Every dead byte the collection knows about, placed or not. +fn dead_total(coll: *const Collection) u64 { + return coll.dead_located() + coll.dead_unlocated; +} + +test "every dead slab byte is counted in exactly one place" { + // The identity the window map rests on: + // + // sum of window counters + dead_unlocated == slab_used - live_bytes + // + // The left side is where the garbage is, the right side is how much there + // is; reclamation reads the first and the compaction trigger reads the + // second, so a drift between them is a rebuild that fires on a clean + // database or a window that is handed back with a document in it. + // + // Both kinds of death are exercised: evicted documents, and the slab the + // appender writes off when a checkpoint freezes the page its cursor points + // into. + // + // Mutation check: drop the `mark_dead` call from `note_skip`, or the + // `mark_dead` call from `evict_doc` -- each removes one of the two ways + // slab dies and the sides part company by that amount. The run *edges* are + // covered separately, by the test below, because whether a run's start is + // `map_align`-aligned is up to the allocator and not something an + // engine-level test can arrange. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // no rebuild may intervene + try engine.lock(); + defer engine.unlock(); + + var i: i32 = 0; + while (i < 40) : (i += 1) { + var d = try make_padded(gpa, i, 6000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + // A checkpoint every few documents, so the appender keeps having to + // round its cursor up to a system page and skipping the bytes between. + if (@mod(i, 7) == 6) { + try engine.commit(); + try engine.checkpoint(); + } + } + try engine.commit(); + + const coll = engine.get_collection("app", "c").?; + // Some skipping must actually have happened, or the test proves only the + // easy half. 6000-byte documents never land flush against a page boundary. + try testing.expect(coll.slab_used > coll.live_bytes); + try testing.expectEqual(coll.slab_used - coll.live_bytes, dead_total(coll)); + + // Now the other kind: evictions. + i = 0; + while (i < 40) : (i += 2) { + try testing.expect(try engine.remove_by_id("app", "c", .{ .int32 = i })); + } + try engine.commit(); + try testing.expectEqual(coll.slab_used - coll.live_bytes, dead_total(coll)); + // And the evicted bytes are mostly placeable: 6000-byte documents are far + // smaller than a window, so they fall inside one rather than off its edge. + try testing.expect(coll.dead_located() > coll.dead_unlocated); +} + +test "dead bytes outside a whole window are counted but not placed" { + // A run is allocated in 4 KiB pages but reclaimed in `map_align` windows, + // so unless the allocator happens to hand back an aligned run there is a + // head below its first window boundary and a tail above its last. Bytes + // that die there can never be given back on their own -- but they are + // still garbage, and if they were simply dropped the amount of garbage the + // collection reports would fall short of the amount it has, which is a + // compaction that never fires. + // + // Driven against `mark_dead` directly: the alignment of a real slab extent + // is the allocator's business and an engine-level test cannot arrange for + // an unaligned one. + // + // Mutation checks, each red on its own: drop the trailing `if (pos < + // stop)` and the tail bytes go uncounted; drop the leading `if (pos < + // r.window_first)` and the window index underflows instead, which takes + // down half the suite. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, env.io, tmp.path); + defer engine.deinit(); + try engine.lock(); + defer engine.unlock(); + + var d = try make_doc(gpa, 1, "alice"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + const coll = engine.get_collection("app", "c").?; + + // A run deliberately starting one 4 KiB page past a window boundary, long + // enough to hold two whole windows plus a partial one at each end. + const pages_per_window: u32 = @intCast(pgr.map_align / pgr.page_size); + if (pages_per_window < 2) return error.SkipZigTest; // no edges to test + const owned = coll.slab_runs.items[0]; + const aligned = std.mem.alignForward(u32, owned.first + owned.pages + 8, pages_per_window); + try coll.insert_run(gpa, aligned + 1, 3 * pages_per_window); + const ri = coll.run_of(@as(u64, aligned + 1) << pgr.page_shift).?; + const r = coll.slab_runs.items[ri]; + try testing.expectEqual(@as(usize, 2), r.dead.len); + try testing.expect(r.window_first > r.start()); + try testing.expect(r.window_end() < r.end()); + + const before = coll.dead_unlocated; + // The head, one whole window, and the tail. + coll.mark_dead(r.start(), r.window_first - r.start()); + coll.mark_dead(r.window_first, pgr.map_align); + coll.mark_dead(r.window_end(), r.end() - r.window_end()); + + try testing.expectEqual(@as(u64, pgr.map_align), coll.dead_located()); + try testing.expectEqual( + before + (r.window_first - r.start()) + (r.end() - r.window_end()), + coll.dead_unlocated, + ); + // The whole window is full and the one beside it untouched: the head and + // tail bytes did not leak into a counter that would hand a window back. + try testing.expectEqual(@as(WindowDead, pgr.map_align), r.dead[0]); + try testing.expectEqual(@as(WindowDead, 0), r.dead[1]); +} + +test "a slab run holds whole documents" { + // `mark_dead` charges a document to the run holding its first byte and + // asserts the rest is in the same run. That is only sound because + // `slab_reserve` never lets an append cross `slab_end` -- so check it + // against a collection that owns several runs, including one taken for a + // single oversized document. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); + try engine.lock(); + defer engine.unlock(); + + var small = try make_padded(gpa, 1, 1000); + defer small.deinit(); + try engine.insert("app", "c", &small, &env.gen); + // Larger than the standard 8 MiB extent, so it gets a run of its own and + // the next document forces a third. + var huge = try make_padded(gpa, 2, 9 * 1024 * 1024); + defer huge.deinit(); + try engine.insert("app", "c", &huge, &env.gen); + var after = try make_padded(gpa, 3, 1000); + defer after.deinit(); + try engine.insert("app", "c", &after, &env.gen); + try engine.commit(); + + const coll = engine.get_collection("app", "c").?; + try testing.expect(coll.slab_runs.items.len >= 2); + // Sorted by page number, and non-overlapping. + for (coll.slab_runs.items[1..], 0..) |r, k| { + const prev = coll.slab_runs.items[k]; + try testing.expect(prev.first + prev.pages <= r.first); + } + var it = coll.id_index.iter(); + while (it.next()) |entry| { + const ri = coll.run_of(entry.off) orelse return error.TestUnexpectedResult; + const r = coll.slab_runs.items[ri]; + try testing.expect(entry.off + coll.doc_bytes(entry.off).len <= r.end()); + } +} + +test "a slab run recycled to a lower address keeps the list sorted" { + // Runs used to be held in allocation order, which was fine while an extent + // could only be appended. Reclamation makes a recycled run arrive at an + // address *below* one the collection already owns, and `run_of` is a binary + // search -- so the list has to be ordered by page, not by age. + // + // Exercised on the structure directly: producing a lower-addressed + // allocation through the engine needs the free list to be primed, which is + // 3.3's business, and this invariant should hold before then. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, env.io, tmp.path); + defer engine.deinit(); + try engine.lock(); + defer engine.unlock(); + + var d = try make_doc(gpa, 1, "alice"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + const coll = engine.get_collection("app", "c").?; + + // Three more runs, arriving out of order and clear of the one the insert + // took. They are never written to, so no pages need to exist. + const base: u32 = coll.slab_runs.items[0].first + coll.slab_runs.items[0].pages + 16; + try coll.insert_run(gpa, base + 200, 8); + try coll.insert_run(gpa, base, 8); + try coll.insert_run(gpa, base + 100, 8); + try testing.expectEqual(@as(usize, 4), coll.slab_runs.items.len); + for (coll.slab_runs.items[1..], 0..) |r, k| { + try testing.expect(coll.slab_runs.items[k].first < r.first); + } + // And every one of them is findable at its own address, which is the point + // of the ordering. + for ([_]u32{ base, base + 100, base + 200 }) |first| { + const off = @as(u64, first) << pgr.page_shift; + const ri = coll.run_of(off) orelse return error.TestUnexpectedResult; + try testing.expectEqual(first, coll.slab_runs.items[ri].first); + } + try testing.expect(coll.run_of(@as(u64, base + 8) << pgr.page_shift) == null); +} + +test "a restart forgets where the garbage is, not that there is any" { + // The window map is deliberately not persisted: it would be a new catalog + // field, a new version, and a second read path, to save re-deriving + // something the collection can live without. What must survive is the + // *amount*, because that is what arms compaction -- so an open puts the + // whole of it into `dead_unlocated` and the identity still holds with every + // window counter at zero. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var dead_before: u64 = 0; + { + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); + try engine.lock(); + defer engine.unlock(); + var i: i32 = 0; + while (i < 20) : (i += 1) { + var d = try make_padded(gpa, i, 4000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + i = 0; + while (i < 20) : (i += 2) { + _ = try engine.remove_by_id("app", "c", .{ .int32 = i }); + } + try engine.commit(); + const coll = engine.get_collection("app", "c").?; + try testing.expect(coll.dead_located() > 0); + dead_before = coll.slab_used - coll.live_bytes; + try engine.checkpoint(); + } + + var engine2 = try Engine.open(gpa, io, tmp.path); + defer engine2.deinit(); + const coll = engine2.get_collection("app", "c").?; + try testing.expectEqual(dead_before, coll.slab_used - coll.live_bytes); + try testing.expectEqual(@as(u64, 0), coll.dead_located()); + try testing.expectEqual(dead_before, coll.dead_unlocated); + try testing.expectEqual(dead_before, dead_total(coll)); + // The runs came back too, and in a shape `run_of` can use. + var it = coll.id_index.iter(); + while (it.next()) |entry| try testing.expect(coll.run_of(entry.off) != null); +} + test "a replace that changes nothing is not a write" { // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm. // Red on all three: the log grows, the document is superseded so the engine @@ -3401,7 +3920,7 @@ test "a checkpoint runs alongside writers on several collections" { // counters while holding only the *shared catalog* lock -- and a writer // holds that same lock shared, taking the collection's lock exclusively. // So the snapshot walked structures its owner was free to mutate, and - // `slab_extents` is an ArrayList a new extent appends to: a reallocation + // `slab_runs` is an ArrayList a new extent inserts into: a reallocation // mid-walk leaves the serializer reading freed memory. // // Several collections rather than one, because the interesting overlap is a @@ -4485,12 +5004,7 @@ fn id_key_for(gpa: std.mem.Allocator, v: bson.Value) ![]u8 { /// index from one still holding pre-rebuild offsets -- the old bytes are on the /// free list rather than overwritten, so reading them still succeeds. fn offset_in_extents(coll: *const Collection, off: u64) bool { - for (coll.slab_extents.items) |e| { - const first = @as(u64, e.first) << pgr.page_shift; - const end = first + (@as(u64, e.pages) << pgr.page_shift); - if (off >= first and off < end) return true; - } - return false; + return coll.run_of(off) != null; } /// Document bytes at an absolute file offset, without needing the Collection.