diff --git a/src/db.zig b/src/db.zig index be696d3..84b1cce 100644 --- a/src/db.zig +++ b/src/db.zig @@ -50,6 +50,11 @@ pub const Collection = struct { /// rebuild. `slab_tail` cannot answer that -- it is an absolute file offset, /// so it jumps forward whenever a fresh extent is taken. slab_used: u64, + /// Of those bytes, the ones still reachable. `slab_used - live_bytes` is + /// this collection's slab garbage, which only a rebuild reclaims. Kept per + /// collection so dropping one can move the right amount from the engine's + /// live total to its dead total. + live_bytes: u64, /// Secondary indexes (persisted through the log). Heap-allocated, so an /// `*Index` handed out by `find_index` or `create_index` stays valid when /// a sibling index is dropped. Held by value, `orderedRemove` memmoved the @@ -85,6 +90,7 @@ pub const Collection = struct { .slab_tail = 0, .slab_end = 0, .slab_used = 0, + .live_bytes = 0, .indexes = .empty, .id_index = undefined, }; @@ -122,7 +128,12 @@ pub const Collection = struct { // page the tail points into. Appending there would store inside the // durable image, so abandon the rest of the extent and start a fresh // one. The waste is bounded by one extent per collection per checkpoint. - if (self.slab_tail >= self.pager.stable_bytes() and self.slab_tail + len <= self.slab_end) return; + // + // `is_unpublished_at` rather than a comparison against the stable mark: + // an extent recycled off the free list starts *below* the mark and is + // still writable. Asking the mark meant every recycled extent was thrown + // away after one document, so churn never reused anything. + if (self.pager.is_unpublished_at(self.slab_tail) and self.slab_tail + len <= self.slab_end) return; // A document larger than the standard extent gets one of its own; BSON // reaches 16 MB and the extent is 8 MiB. const want_pages: u32 = @intCast(@max( @@ -147,6 +158,9 @@ pub const Collection = struct { @memcpy(self.pager.bytes_mut(off, bytes.len), bytes); self.slab_tail += bytes.len; self.slab_used += bytes.len; + // Here rather than at the call site: a rebuild appends through this same + // path, and its copies are live by definition. + self.live_bytes += bytes.len; return off; } @@ -240,6 +254,13 @@ pub const Engine = struct { /// rewrite is worth doing — see `note_compact`. live_docs: u64 = 0, dead_docs: u64 = 0, + /// The same question in bytes, about the *data file* rather than the log. + /// Once a checkpoint truncates the log, the log no longer holds the garbage + /// -- the doc slab does, and only a rebuild reclaims it. These are what + /// `note_compact` gates on; counting documents would let one collection of + /// 16 KiB documents and one of 40 B documents look identical. + live_bytes: u64 = 0, + dead_bytes: u64 = 0, /// Set when the log has grown enough since the last checkpoint to be worth /// reclaiming. Read by the write epilogue and the TTL monitor, both of which /// run without holding a collection lock. @@ -314,6 +335,9 @@ pub const Engine = struct { engine.seq = replay_from; engine.committed_seq = replay_from; engine.live_docs = engine.pager.loaded.live_docs; + // Without this a restart forgets its garbage, and a churned + // database would never compact again. + engine.dead_bytes = engine.pager.loaded.dead_bytes; } } @@ -352,6 +376,9 @@ pub const Engine = struct { assert_msg(self.live_docs >= coll.doc_count, "dropping a collection would underflow the engine's live count"); self.live_docs -= coll.doc_count; self.dead_docs += coll.doc_count; + assert_msg(self.live_bytes >= coll.live_bytes, "dropping a collection would underflow the engine's live bytes"); + self.live_bytes -= coll.live_bytes; + self.dead_bytes += coll.live_bytes; coll.id_index.deinit(self.gpa); for (coll.indexes.items) |ix| { ix.deinit(self.gpa); @@ -388,6 +415,17 @@ pub const Engine = struct { /// regenerating them from it, which is far cheaper than scanning. /// `id_enc` is `bson.encode_key` of the document's `_id` -- the canonical /// encoding, which is what the `_id_` index is keyed on. + /// Copy a new document's bytes into the collection's slab and count them as + /// live at both levels. `Collection.slab_append` maintains the collection's + /// own total (a rebuild appends through it too, and its copies are live by + /// definition); the engine's total only moves when a document actually + /// becomes live, which a rebuild's copies do not. + fn publish_doc_bytes(self: *Engine, coll: *Collection, bytes: []const u8) u64 { + const off = coll.slab_append(bytes); + self.live_bytes += bytes.len; + return off; + } + fn evict_doc(self: *Engine, coll: *Collection, id_enc: []const u8) void { const off = coll.id_index.lookup_exact(id_enc) orelse return; // Resolve the bytes before any mutation; the slab is untouched by @@ -399,8 +437,13 @@ pub const Engine = struct { assert_msg(self.live_docs >= 1, "evicting a document would underflow the engine's live count"); self.live_docs -= 1; self.dead_docs += 1; + assert_msg(self.live_bytes >= old_bytes.len, "evicting a document would underflow the engine's live bytes"); + self.live_bytes -= old_bytes.len; + self.dead_bytes += old_bytes.len; assert_msg(coll.doc_count >= 1, "evicting a document would underflow the collection's count"); coll.doc_count -= 1; + assert_msg(coll.live_bytes >= old_bytes.len, "evicting a document would underflow the collection's live bytes"); + coll.live_bytes -= old_bytes.len; } /// `bson.encode_key` of a stored document's `_id`, owned by the caller. @@ -743,7 +786,7 @@ pub const Engine = struct { // 7. Publish the document and its entries: copy the bytes into the // slab and record the offset. Infallible from here. - const off = coll.slab_append(doc_bytes); + const off = self.publish_doc_bytes(coll, doc_bytes); self.live_docs += 1; coll.doc_count += 1; for (built_list.items) |*b| { @@ -1100,21 +1143,24 @@ pub const Engine = struct { } fn note_compact(self: *Engine) void { - // The threshold counts data volume (uncompressed record bytes), not - // the on-disk size: a compressed log would otherwise stay under any - // byte threshold and never compact its garbage. - if (self.log.data_bytes < self.compact_threshold) return; - // Only rewrite when enough of the log is actually garbage. The old - // rule fired on bytes appended, which is the wrong question twice - // over: a 1 GB bulk load has no garbage at all yet would compact - // ~64 times under a 16 MiB threshold (rewriting 1 GB each time, - // hence quadratic), while a small collection rewritten in place - // accumulates garbage indefinitely without ever hitting the count. + // Garbage is measured in the *data file*, not the log. This used to read + // `log.data_bytes`, which was the right question while the log was the + // only copy of the data -- but a checkpoint now truncates the log and + // `truncate_to_header` resets that counter, so the first gate stopped + // being reachable and compaction silently never fired again. The churn + // gate caught it: 50% churn over three rounds left the data file at 4.1x + // the live data, with a trigger that had been dead since the log started + // being reclaimed. // - // Garbage share is dead / (live + dead); this fires at ~20%, so the - // file stays near 1.25x the live data and each compaction is paid - // for by the space it reclaims. - if (self.dead_docs * 4 < self.live_docs) return; + // Absolute volume first: a rewrite costs a full copy of the live data, + // so it is not worth doing for a few kilobytes however bad the ratio. + if (self.dead_bytes < self.compact_threshold) return; + // Then the share, dead / (live + dead), firing at ~20%: the file stays + // near 1.25x the live data and each rebuild is paid for by the space it + // reclaims. Bytes rather than document counts, because a rewrite copies + // bytes -- 100k evicted 40 B documents are not worth the same rebuild as + // 100k evicted 16 KiB ones. + if (self.dead_bytes * 4 < self.live_bytes) return; self.compact_pending.store(true, .release); } @@ -1184,10 +1230,24 @@ pub const Engine = struct { if (rebuild_err) |err| return err; self.dead_docs = 0; + // Every collection's slab was just repacked to hold only live bytes. + self.dead_bytes = 0; // Publish the rebuilt layout, which is also what reclaims the log. Until // this lands the old pages are still referenced by the previous // watermark, so a crash mid-rebuild simply loses the rebuild. try self.checkpoint(); + // And again, to walk the pages the rebuild just abandoned the rest of the + // way down the free list: one publish moves them from `pending` to + // `hold`, a second from `hold` to `ready`. Without this the space a + // rebuild reclaims is not reusable until two unrelated checkpoints have + // happened, so the next rebuild grows the file instead of reusing it -- + // measured as ~1.2x of extra steady-state size under sustained churn. + // + // Safe for the same reason the delay exists: what the second publish + // releases is the pages the *pre-rebuild* image referenced, and that + // image is no longer the fallback -- the first publish made the rebuilt + // one current and the one before it the fallback. Both remain intact. + try self.checkpoint(); } /// Copy one collection's live documents into fresh extents and rebuild every @@ -1209,6 +1269,7 @@ pub const Engine = struct { coll.slab_tail = 0; coll.slab_end = 0; coll.slab_used = 0; + coll.live_bytes = 0; // Walk in _id order, which is also the order the new slab ends up in -- // so a later scan reads it sequentially. @@ -1347,6 +1408,13 @@ pub const Engine = struct { fn write_catalog(self: *Engine, out: *std.ArrayListUnmanaged(u8)) !void { const gpa = self.gpa; + // The engine's live-byte total is by definition the sum over + // collections, and `read_catalog` rebuilds it that way. Check it here, + // where every collection is being walked regardless: a divergence means + // some path published or evicted bytes at one level and not the other, + // and the visible symptom would be a compaction trigger that fires + // never or always. + var live_sum: u64 = 0; try put_u32(gpa, out, catalog_magic); try put_u32(gpa, out, catalog_version); try put_u64(gpa, out, self.live_docs); @@ -1363,6 +1431,12 @@ pub const Engine = struct { try put_u64(gpa, out, coll.slab_tail); try put_u64(gpa, out, coll.slab_end); try put_u64(gpa, out, coll.slab_used); + try put_u64(gpa, out, coll.live_bytes); + live_sum += coll.live_bytes; + assert_msg( + coll.live_bytes <= coll.slab_used, + "a collection cannot hold more live bytes than it ever appended", + ); try put_u32(gpa, out, @intCast(coll.slab_extents.items.len)); for (coll.slab_extents.items) |e| { try put_u32(gpa, out, e.first); @@ -1373,6 +1447,10 @@ pub const Engine = struct { for (coll.indexes.items) |ix| try write_index_catalog(gpa, out, ix); } } + assert_msg( + live_sum == self.live_bytes, + "the engine's live-byte total must equal the sum over collections", + ); try put_u64(gpa, out, std.hash.XxHash3.hash(0, out.items)); } @@ -1437,6 +1515,10 @@ pub const Engine = struct { coll.slab_tail = try r.read_u64(); coll.slab_end = try r.read_u64(); coll.slab_used = try r.read_u64(); + coll.live_bytes = try r.read_u64(); + // The engine's total is the sum over collections rather than a + // separately stored field, so the two cannot disagree. + self.live_bytes += coll.live_bytes; const nex = try r.read_u32(); var e: u32 = 0; while (e < nex) : (e += 1) { @@ -1570,6 +1652,8 @@ pub const Engine = struct { self.dbs.clearRetainingCapacity(); self.live_docs = 0; self.dead_docs = 0; + self.live_bytes = 0; + self.dead_bytes = 0; self.seq = 0; self.committed_seq = 0; } @@ -1621,6 +1705,7 @@ pub const Engine = struct { .catalog_page = first, .catalog_len = buf.items.len, .live_docs = self.live_docs, + .dead_bytes = self.dead_bytes, }) catch |err| { self.log_lock.unlock(self.io); return err; @@ -1747,7 +1832,7 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an const doc_bytes = try serialize_doc(self.gpa, doc); defer self.gpa.free(doc_bytes); try coll.slab_reserve(self.gpa, doc_bytes.len); - const off = coll.slab_append(doc_bytes); + const off = self.publish_doc_bytes(coll, doc_bytes); self.live_docs += 1; coll.doc_count += 1; // The `_id_` entry is added *now*, not after replay: it is the only @@ -1939,6 +2024,162 @@ test "compaction reclaims garbage but leaves a garbage-free log alone" { try testing.expect(engine.log.data_bytes < after_insert * 2); } +test "compaction still triggers after a checkpoint has truncated the log" { + // Mutation: gate `note_compact` on `self.log.data_bytes` (what it read + // before the checkpoint existed) instead of `self.dead_bytes`. Red, because + // `truncate_to_header` zeroes that counter at every checkpoint -- the trigger + // then never fires and the doc slab grows without bound. This test exists + // because the churn gate measured exactly that: 4.1x live data. + // + // Second mutation: drop `dead_bytes` from the watermark, or from the restore + // beside `live_docs` in `open`. Red on the reopened engine below. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + // 200 documents of a few dozen bytes each, so the volume gate has to be + // small enough for their garbage to clear it. + engine.compact_threshold = 1024; + try engine.lock(); + defer engine.unlock(); + + for (0..200) |i| { + var d = try make_doc(gpa, @intCast(i), "x"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + + // A checkpoint, which is what truncates the log. From here on the log says + // nothing about how much garbage the database holds. + try engine.checkpoint(); + try testing.expectEqual(@as(u64, 0), engine.log.data_bytes); + const live_after_load = engine.live_bytes; + try testing.expect(live_after_load > 0); + + // Now make garbage. Every replace supersedes a document, so its slab bytes + // are dead: the data file holds them and only a rebuild reclaims them. + _ = engine.take_compact(); + for (0..200) |i| { + var d = try make_doc(gpa, @intCast(i), "yy"); + defer d.deinit(); + try engine.replace("app", "c", &d, &env.gen); + } + try engine.commit(); + + try testing.expectEqual(live_after_load + 200, engine.live_bytes); + try testing.expectEqual(live_after_load, engine.dead_bytes); + try testing.expect(engine.dead_bytes >= engine.compact_threshold); + try testing.expect(engine.take_compact()); + + // And the rebuild actually clears the garbage it was called for. + try engine.compact(); + try testing.expectEqual(@as(u64, 0), engine.dead_bytes); + try testing.expectEqual(@as(u64, 200), engine.live_docs); + // The engine's live total is the sum over collections, both after a rebuild + // and after the catalog round trip below. + const coll = engine.get_collection("app", "c").?; + try testing.expectEqual(engine.live_bytes, coll.live_bytes); + try testing.expectEqual(coll.slab_used, coll.live_bytes); +} + +test "a rebuild leaves the space it reclaimed ready to reuse" { + // Mutation check: delete the second `checkpoint()` at the end of `compact`. + // Red -- one publish only moves the abandoned extents from `pending` to + // `hold`, so nothing is reusable and the next rebuild grows the file instead. + // Measured on the churn gate as ~1.2x of extra steady-state size (3.58x live + // data against 2.47x) under sustained update churn. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = 1024; + try engine.lock(); + defer engine.unlock(); + + for (0..300) |i| { + var d = try make_doc(gpa, @intCast(i), "x"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + for (0..300) |i| { + var d = try make_doc(gpa, @intCast(i), "yy"); + defer d.deinit(); + try engine.replace("app", "c", &d, &env.gen); + } + try engine.commit(); + try testing.expect(engine.take_compact()); + try engine.compact(); + + // The rebuild abandoned the old slab extent and every page the old trees + // occupied. Those must be handed back, not merely queued. + try testing.expect(engine.pager.free_ready_pages() > 0); + + // And the next allocation actually uses them rather than the tail. + const tail_before = engine.pager.alloc_tail; + for (0..300) |i| { + var d = try make_doc(gpa, @intCast(i), "zzz"); + defer d.deinit(); + try engine.replace("app", "c", &d, &env.gen); + } + try engine.commit(); + try testing.expect(engine.pager.alloc_tail < tail_before + engine.pager.free_ready_pages() + 64); + try testing.expectEqual(@as(u64, 300), engine.live_docs); +} + +test "reopen carries the garbage counter across a restart" { + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var dead_before: u64 = 0; + var live_before: u64 = 0; + { + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // never rebuild here + try engine.lock(); + for (0..100) |i| { + var d = try make_doc(gpa, @intCast(i), "x"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + for (0..50) |i| { + var d = try make_doc(gpa, @intCast(i), "yy"); + defer d.deinit(); + try engine.replace("app", "c", &d, &env.gen); + } + try engine.commit(); + try engine.checkpoint(); + dead_before = engine.dead_bytes; + live_before = engine.live_bytes; + engine.unlock(); + try testing.expect(dead_before > 0); + } + + var engine2 = try Engine.open(gpa, io, tmp.path); + defer engine2.deinit(); + try testing.expectEqual(dead_before, engine2.dead_bytes); + try testing.expectEqual(live_before, engine2.live_bytes); +} + test "reopen replays log" { var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); diff --git a/src/index.zig b/src/index.zig index ae0acb2..4de40bf 100644 --- a/src/index.zig +++ b/src/index.zig @@ -456,7 +456,10 @@ pub const Index = struct { // Same rule as the document slab: a checkpoint freezes the page the tail // points into, so a frozen tail means starting a fresh extent rather // than writing inside the durable image. - if (self.ovf_tail >= self.pager.stable_bytes() and self.ovf_tail + overflow_bytes <= self.ovf_end) return; + // Same reasoning as the document slab: a recycled extent is below the + // stable mark and still writable, so ask whether these bytes are in the + // published image rather than where they sit. + if (self.pager.is_unpublished_at(self.ovf_tail) and self.ovf_tail + overflow_bytes <= self.ovf_end) return; // One extent for the whole batch, or a bespoke one when a single // record is larger than the standard extent (a BSON string reaches // 16 MB). diff --git a/src/pager.zig b/src/pager.zig index 90f5670..0d9309a 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -209,8 +209,28 @@ pub const Pager = struct { /// Pages below this belong to the last published image and must never be /// stored into (PLAN amendment A1). Zero until a checkpoint publishes one, /// which is why copy-on-write is inert before then. + /// + /// A *bound*, not the membership test: see `unpublished`. stable_pages: u32, + /// Pages handed out since the last publish, and therefore not referenced by + /// any durable image -- free to be written in place however low their page + /// number is. + /// + /// `p >= stable_pages` was used for this and is not the same question. It is + /// right for tail-bumped pages and wrong for recycled ones, which come off + /// the free list *below* the mark and are nonetheless writable. The + /// difference is not cosmetic: with the numeric test, every write to a + /// recycled node page copied it and freed it again, and every recycled slab + /// extent was abandoned after a single document -- so nothing was ever really + /// reused and the file grew without bound under churn. The churn gate + /// measured 7.2x live data over six rounds, still climbing linearly. + /// + /// One bit per page: 32 KiB per GiB of database, one load on the COW path + /// against the 4 KiB copy it avoids. Cleared wholesale at each publish, + /// which is exactly when every allocated page becomes part of the image. + unpublished: std.DynamicBitSetUnmanaged, + /// Free pages, in three stages. `ready` may be handed out now; `hold` was /// freed one generation ago; `pending` was freed during this generation. /// @@ -281,12 +301,16 @@ pub const Pager = struct { .loaded = .{}, .generation = 0, .stable_pages = 0, + .unpublished = .{}, .free_ready = .empty, .free_hold = .empty, .free_pending = .empty, .dirty = if (track_dirty) .empty else {}, .protect_ok = false, }; + // The bit set is the one heap allocation `self` owns before `deinit` can + // be reached: a rejected header returns from the middle of this function. + errdefer self.unpublished.deinit(gpa); if (created) { try self.grow_to(page_first_data); @@ -303,6 +327,7 @@ pub const Pager = struct { } pub fn deinit(self: *Pager) void { + self.unpublished.deinit(self.gpa); self.free_ready.deinit(self.gpa); self.free_hold.deinit(self.gpa); self.free_pending.deinit(self.gpa); @@ -324,13 +349,12 @@ pub const Pager = struct { /// *updated* use `page_mut_cow`; append-only consumers pass a page they /// allocated themselves. /// - /// There is deliberately no `p >= stable_pages` assert here, and the reason - /// is worth recording because it looks like an obvious check to add. A page - /// recycled off the free list *is* below the mark and *is* legitimately - /// writable -- it was freed two generations ago and no live image references - /// it any more -- so the page number alone cannot distinguish a violation - /// from a reuse, and a per-write set membership test would cost more than it - /// is worth on the hot path. + /// There is deliberately no assert on the page number here. A page recycled + /// off the free list is below the stable mark and is legitimately writable -- + /// it was freed two generations ago and no live image references it any more + /// -- so the page number alone cannot distinguish a violation from a reuse. + /// `unpublished` can, but asserting on it here would only restate what + /// `page_mut_cow` has already decided one frame up. /// /// The invariant is enforced the two ways PLAN amendment A1 describes /// instead: structurally, because every write to a tree node goes through @@ -354,15 +378,30 @@ pub const Pager = struct { /// The victim goes on the free list, which withholds it for two generations, /// so the image that still references it stays intact and usable. pub fn page_mut_cow(self: *Pager, slot: *u32) !*align(page_size) [page_size]u8 { - if (slot.* >= self.stable_pages) return self.page_mut(slot.*); - const fresh = try self.alloc_pages(1); + if (self.is_unpublished(slot.*)) return self.page_mut(slot.*); + const copy = try self.alloc_pages(1); @memcpy( - @as(*[page_size]u8, @ptrCast(self.page_mut(fresh))), + @as(*[page_size]u8, @ptrCast(self.page_mut(copy))), @as(*const [page_size]u8, @ptrCast(self.page(slot.*))), ); try self.free_pages(slot.*, 1); - slot.* = fresh; - return self.page_mut(fresh); + slot.* = copy; + return self.page_mut(copy); + } + + /// Whether this page was handed out since the last publish, and so may be + /// written in place. See the `unpublished` field. + pub inline fn is_unpublished(self: *const Pager, p: u32) bool { + return p < self.unpublished.bit_length and self.unpublished.isSet(p); + } + + /// The same question for the byte-offset consumers: may an append at `off` + /// land in place, or does its page belong to the durable image? Used by the + /// document slab and the overflow slab, which would otherwise have to guess + /// from the offset (and, using `stable_bytes`, guessed wrong for every + /// recycled extent). + pub inline fn is_unpublished_at(self: *const Pager, off: u64) bool { + return self.is_unpublished(@intCast(off >> page_shift)); } /// The one page a write may legitimately land on below the stable mark: a @@ -445,19 +484,60 @@ pub const Pager = struct { /// Take a run from the free list if one fits, else bump the tail. A recycled /// page was inside a published image once, so its protection has to be /// lifted before it is handed out again. + /// Best fit, not first fit. First fit lets single-page requests cannibalise + /// the large runs: copy-on-write asks for one page thousands of times per + /// generation, each carving a page off the front of whatever run comes first, + /// and a 2048-page slab extent is shaved to nothing while never being usable + /// as an extent. That is what the churn gate saw -- the free list draining to + /// zero every generation with the file still growing by the full write volume. + /// Smallest-sufficient keeps the big runs whole for the consumers that need + /// them, and there is nothing else that wants the one-page holes. fn take_free(self: *Pager, n: u32) ?u32 { + // The `pages == n` early exit is the exact-match shortcut only; removing + // it must leave behaviour identical, and turning the loop into a plain + // first-fit `break` is the mutation the extent test names. + var best: ?usize = null; for (self.free_ready.items, 0..) |e, i| { if (e.pages < n) continue; - const first = e.first; - if (e.pages == n) { - _ = self.free_ready.swapRemove(i); - } else { - self.free_ready.items[i] = .{ .first = e.first + n, .pages = e.pages - n }; - } - self.unprotect(first, n); - return first; + if (best == null or e.pages < self.free_ready.items[best.?].pages) best = i; + if (e.pages == n) break; // cannot do better than exact } - return null; + const i = best orelse return null; + const e = self.free_ready.items[i]; + const first = e.first; + if (e.pages == n) { + _ = self.free_ready.swapRemove(i); + } else { + self.free_ready.items[i] = .{ .first = e.first + n, .pages = e.pages - n }; + } + self.unprotect(first, n); + return first; + } + + /// Merge runs that touch, so the holes single-page frees leave behind can add + /// up to an extent again. Without it the free list only ever fragments: every + /// generation returns thousands of one-page copy-on-write victims, and an + /// 8 MiB slab extent request never finds a home however much space is free. + /// Once per publish, over a list whose length is the generation's free count. + fn coalesce_free_ready(self: *Pager) void { + const items = self.free_ready.items; + if (items.len < 2) return; + std.mem.sort(Extent, items, {}, struct { + fn less(_: void, a: Extent, b: Extent) bool { + return a.first < b.first; + } + }.less); + var w: usize = 0; + for (items[1..]) |e| { + const prev = &items[w]; + if (prev.first + prev.pages == e.first) { + prev.pages += e.pages; + } else { + w += 1; + items[w] = e; + } + } + self.free_ready.items.len = w + 1; } pub fn alloc_pages_assume_reserved(self: *Pager, n: u32) u32 { @@ -474,12 +554,26 @@ pub const Pager = struct { // Reuse before growing. Without this the free list is decorative and the // file grows without bound under churn, because copy-on-write abandons // every page it touches in every generation (PLAN amendment A2). - if (self.take_free(n)) |recycled| return recycled; + if (self.take_free(n)) |recycled| { + self.mark_unpublished(recycled, n); + return recycled; + } const first = self.alloc_tail; self.alloc_tail += n; + self.mark_unpublished(first, n); return first; } + fn mark_unpublished(self: *Pager, first: u32, n: u32) void { + // `grow_to` sizes the set to the mapping, and `reserve_pages` has already + // grown the mapping past this run, so the range is in bounds. + assert_msg( + @as(usize, first) + n <= self.unpublished.bit_length, + "allocated a page outside the unpublished set", + ); + self.unpublished.setRangeValue(.{ .start = first, .end = first + n }, true); + } + /// Bytes currently allocated, i.e. the extent of the live image. pub fn allocated_bytes(self: *const Pager) u64 { return @as(u64, self.alloc_tail) << page_shift; @@ -584,6 +678,10 @@ pub const Pager = struct { self.file_pages = new_pages; } + // Before the mapping grows, so a failure here cannot leave pages + // reachable that the set has no bit for. + try self.unpublished.resize(self.gpa, new_pages, false); + const off: usize = @as(usize, self.mapped_pages) << page_shift; const len: usize = @intCast(new_len - (@as(u64, self.mapped_pages) << page_shift)); _ = try std.posix.mmap( @@ -749,6 +847,10 @@ pub const Pager = struct { self.free_hold.clearRetainingCapacity(); try self.free_hold.appendSlice(self.gpa, self.free_pending.items); self.free_pending.clearRetainingCapacity(); + self.coalesce_free_ready(); + // Every page handed out so far is now part of the published image, so + // nothing may be written in place any more until it is allocated afresh. + self.unpublished.setRangeValue(.{ .start = 0, .end = self.unpublished.bit_length }, false); } // -- free list ---------------------------------------------------------- @@ -1398,6 +1500,120 @@ test "a page already above the mark is written in place, not copied" { try testing.expectEqual(before, slot); } +test "a recycled page is written in place rather than copied again" { + // Mutation check: put `slot.* >= self.stable_pages` back in `page_mut_cow` + // in place of `is_unpublished`. Red -- a recycled page has a low page number + // and would be copied and freed all over again, so nothing is ever really + // reused. That is not a hypothetical: it is what the churn gate measured as + // a data file growing linearly and without bound, at 7.2x live data and + // still climbing when the run was stopped. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + // Free a page and let it walk all the way to reusable. + const doomed = try pg.alloc_pages(1); + @memset(pg.page_mut(doomed), 0xAA); + try pg.publish(.{ .seq = 1 }); + try pg.free_pages(doomed, 1); + try pg.publish(.{ .seq = 2 }); + try pg.publish(.{ .seq = 3 }); + try testing.expect(pg.free_ready_pages() >= 1); + + // Recycling hands it back, below the stable mark. + var slot = try pg.alloc_pages(1); + try testing.expectEqual(doomed, slot); + try testing.expect(slot < pg.stable_pages); + @memset(pg.page_mut(slot), 0xBB); + + // A write through the COW path must land in place: the page is not part of + // any published image, whatever its number. + const tail_before = pg.alloc_tail; + const before = slot; + const p = try pg.page_mut_cow(&slot); + @memset(p, 0xCC); + try testing.expectEqual(before, slot); + try testing.expectEqual(tail_before, pg.alloc_tail); + try testing.expectEqual(@as(u8, 0xCC), pg.page(slot)[0]); +} + +test "one-page requests do not carve up the runs the extents need" { + // Mutation check: make `take_free` first-fit again (take the first extent + // with `pages >= n`). Red -- the single-page allocations below shave the + // large run down and the extent request has to grow the file. + // + // This is the shape the churn gate hit: copy-on-write asks for one page + // thousands of times per generation while the document slab asks for + // 2048-page extents, so first fit dismantled every run before an extent + // could use it. The free list drained to empty every generation and the file + // still grew by the whole write volume. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + // A large run, plus enough single-page holes to serve the small requests. + // The holes are kept apart from each other and from the run by pages that + // are never freed -- otherwise coalescing merges the lot into one run and the + // test stops being about fit at all. + const run = try pg.alloc_pages(64); + _ = try pg.alloc_pages(1); // separator, never freed + var holes: [8]u32 = undefined; + for (&holes) |*h| { + h.* = try pg.alloc_pages(1); + _ = try pg.alloc_pages(1); // separator, never freed + } + try pg.publish(.{ .seq = 1 }); + try pg.free_pages(run, 64); + for (holes) |h| try pg.free_pages(h, 1); + try pg.publish(.{ .seq = 2 }); + try pg.publish(.{ .seq = 3 }); + + // Eight one-page allocations must come out of the eight holes. + for (0..holes.len) |_| _ = try pg.alloc_pages(1); + const tail_before = pg.alloc_tail; + + // So the run is still whole and the extent request is served from it. + const reused = try pg.alloc_pages(64); + try testing.expectEqual(run, reused); + try testing.expectEqual(tail_before, pg.alloc_tail); +} + +test "freed pages that touch merge back into a usable run" { + // Mutation check: drop the `coalesce_free_ready()` call from `publish`. Red + // -- the four one-page frees below stay four separate holes and the run of + // four has to come from the tail. Over a real workload the free list only + // ever fragments, so an 8 MiB extent request never finds a home however much + // free space has accumulated. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + + var pages: [4]u32 = undefined; + for (&pages) |*x| x.* = try pg.alloc_pages(1); + try pg.publish(.{ .seq = 1 }); + // Freed out of order, one page at a time, which is how copy-on-write frees. + try pg.free_pages(pages[2], 1); + try pg.free_pages(pages[0], 1); + try pg.free_pages(pages[3], 1); + try pg.free_pages(pages[1], 1); + try pg.publish(.{ .seq = 2 }); + try pg.publish(.{ .seq = 3 }); + + const tail_before = pg.alloc_tail; + const run = try pg.alloc_pages(4); + try testing.expectEqual(pages[0], run); + try testing.expectEqual(tail_before, pg.alloc_tail); +} + test "freed pages are recycled rather than growing the file" { // Copy-on-write abandons every page it touches, in every generation, so // without reuse a write-heavy workload grows the file by