src/db.zig

@@ -34,6 +34,16 @@ const assert_msg = @import("assert.zig").assert_msg;
 /// Slack is bounded by one extent per collection.
 const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size;
 
+/// Shortest run worth taking off the free list for a slab, 1 MiB. Below this
+/// the extent is exhausted after a few documents, and every exhaustion writes
+/// off whatever is left of the one before it.
+const slab_run_min_pages: u32 = slab_extent_pages / 8;
+
+/// Pages a slab allocation of `len` bytes needs at minimum.
+fn pages_for(len: usize) u32 {
+    return @intCast((len + pgr.page_size - 1) / pgr.page_size);
+}
+
 const LogKind = enum { upsert, delete, index_create, index_drop };
 
 /// Dead bytes in one `map_align` window. The window is the unit of reclamation
@@ -533,15 +543,22 @@ pub const Collection = struct {
         const skipped = self.note_skip(self.slab_end - self.slab_tail);
         // A document larger than the standard extent gets one of its own; BSON
         // reaches 16 MB and the extent is 8 MiB.
-        const want_pages: u32 = @intCast(@max(
+        const want_pages: u32 = @max(slab_extent_pages, pages_for(len));
-            slab_extent_pages,
-            (len + pgr.page_size - 1) / pgr.page_size,
-        ));
         try self.pager.reserve_pages(&self.hold, want_pages);
-        const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages);
+        // Off the free list first, or window reclamation is decorative: the
-        try self.insert_run(gpa, first, want_pages);
+        // pages come back, nothing asks for them in a shape they arrive in, and
-        self.slab_tail = @as(u64, first) << pgr.page_shift;
+        // the file grows by the whole write volume anyway. A floor of 1 MiB,
-        self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift);
+        // because a shorter extent is exhausted after a handful of documents
+        // and every exhaustion abandons what is left of it -- and because the
+        // floor is what makes trimming a larger run harmless.
+        const min_pages: u32 = @min(want_pages, @max(pages_for(len), slab_run_min_pages));
+        const run = self.pager.alloc_slab_run(&self.hold, min_pages, want_pages) orelse pgr.Extent{
+            .first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages),
+            .pages = want_pages,
+        };
+        try self.insert_run(gpa, run.first, run.pages);
+        self.slab_tail = @as(u64, run.first) << pgr.page_shift;
+        self.slab_end = self.slab_tail + (@as(u64, run.pages) << pgr.page_shift);
         return skipped;
     }
 
@@ -3670,6 +3687,71 @@ test "a reclaimed slab window is not reusable until two publishes later" {
     try testing.expect(in_extents(engine.pager.free_ready.items, gone));
 }
 
+test "a churning collection reuses its slab instead of growing the file" {
+    // The one that decides whether any of this was worth doing. Reclamation can
+    // be working perfectly -- windows counted, pages handed back,
+    // `reclaimed_bytes` climbing -- and the file still grow by the whole write
+    // volume, because nothing asks for the pages in the shape they come back
+    // in. That is what `alloc_slab_run` is for, and this is what says so.
+    //
+    // Delete-and-refill in rounds, with checkpoints per round so reclamation
+    // gets to run and what it frees becomes allocatable. The first round has to
+    // grow the file; the ones after it must not.
+    //
+    // Measured here: 2068 pages after the first round, 2083 after three more of
+    // the same volume -- 15 pages of growth against 4800 pages written.
+    //
+    // Mutation check: raise `slab_run_min_pages` to a whole extent, so no
+    // reclaimed run is ever long enough to qualify and every extent request
+    // bumps the tail. Red -- which is also the measurement that says the floor
+    // has to stay well below an extent to be any use.
+    var threaded: std.Io.Threaded = .init_single_threaded;
+    defer threaded.deinit();
+    var env = test_env(&threaded);
+    const io = env.io;
+    const gpa = testing.allocator;
+
+    var tmp = try TmpLog.init(gpa);
+    defer tmp.deinit(gpa);
+    var engine = try Engine.open(gpa, io, tmp.path);
+    defer engine.deinit();
+    engine.compact_threshold = std.math.maxInt(u64); // reuse, not rebuild
+    try engine.lock();
+    defer engine.unlock();
+
+    // Documents a whole window wide, which is the case the design is built for
+    // -- see the note on small documents in the churn gate.
+    const doc_size = pgr.map_align;
+    const per_round = 400;
+    var round: i32 = 0;
+    var tail_after_first: u32 = 0;
+    while (round < 4) : (round += 1) {
+        var i: i32 = 0;
+        while (i < per_round) : (i += 1) {
+            var d = try make_padded(gpa, round * per_round + i, doc_size);
+            defer d.deinit();
+            try engine.insert("app", "c", &d, &env.gen);
+        }
+        try engine.commit();
+        i = 0;
+        while (i < per_round) : (i += 1) {
+            _ = try engine.remove_by_id("app", "c", .{ .int32 = round * per_round + i });
+        }
+        try engine.commit();
+        // Two, so what this round freed is allocatable in the next one.
+        try engine.checkpoint();
+        try engine.checkpoint();
+        if (round == 0) tail_after_first = engine.pager.alloc_tail;
+    }
+
+    const coll = engine.get_collection("app", "c").?;
+    try testing.expect(coll.reclaimed_bytes > 0);
+    // Three more rounds of the same volume after the first. Anything left is
+    // fragmentation the windows could not cover, not the write volume.
+    const grew = engine.pager.alloc_tail - tail_after_first;
+    try testing.expect(grew < 3 * per_round * doc_size / pgr.page_size / 4);
+}
+
 test "a replace that changes nothing is not a write" {
     // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm.
     // Red on all three: the log grows, the document is superseded so the engine

src/pager.zig

@@ -704,6 +704,90 @@ pub const Pager = struct {
         return first;
     }
 
+    /// Take a run off the free list for a document slab: at least `min_pages`,
+    /// at most `max_pages`, starting and ending on a system-page boundary.
+    /// Returns null when nothing on the list qualifies, and the caller bumps the
+    /// tail instead.
+    ///
+    /// `take_free` cannot serve this, and that is the whole reason this exists.
+    /// It is deliberately best fit -- smallest sufficient run -- so that the
+    /// thousands of single-page copy-on-write requests per generation cannot
+    /// dismantle the large runs. A slab extent asks for 2048 pages, and window
+    /// reclamation gives back runs a few pages at a time, so with an exact-size
+    /// rule the free list could fill up with reclaimed slab that no slab request
+    /// would ever take: the pages come back, the file keeps growing, the ratio
+    /// does not move. That is the failure this whole milestone is measured
+    /// against.
+    ///
+    /// So this one takes a partial run when it cannot get a whole one, and is
+    /// allowed to trim a larger one. There is no cannibalisation to fear here:
+    /// the request is itself large (the caller's floor is 1 MiB), so what it
+    /// leaves behind is still a usable run rather than a hole. The one-page
+    /// requests still go through `take_free` unchanged, and its pinned mutation
+    /// test is untouched.
+    ///
+    /// The alignment is not cosmetic. `map_align` is the granularity writeback
+    /// works at and the granularity reclamation gives back at, so a run that
+    /// starts mid-system-page both wastes its first window and shares a kernel
+    /// page with whatever occupies the rest of it -- which for a page still in
+    /// the published image is the tearing `mark_appendable` refuses to risk.
+    pub fn alloc_slab_run(self: *Pager, hold: *Reservation, min_pages: u32, max_pages: u32) ?Extent {
+        assert(min_pages > 0);
+        assert(min_pages <= max_pages);
+        self.alloc_lock.lockUncancelable(self.io);
+        defer self.alloc_lock.unlock(self.io);
+        assert_msg(max_pages <= hold.pages, "a slab run request overran reserve_pages' promise");
+        // A split can leave a piece at each end, so one entry may become two.
+        // Out of memory before anything is disturbed: the caller falls back to
+        // bumping the tail, which is what it would have done anyway.
+        self.free_ready.ensureUnusedCapacity(self.gpa, 1) catch return null;
+
+        const spp: u32 = if (map_align >= page_size) @intCast(map_align / page_size) else 1;
+        var best: ?usize = null;
+        var best_first: u32 = 0;
+        var best_take: u32 = 0;
+        for (self.free_ready.items, 0..) |e, i| {
+            const from = std.mem.alignForward(u32, e.first, spp);
+            const to = std.mem.alignBackward(u32, e.first + e.pages, spp);
+            if (to <= from) continue;
+            const usable = to - from;
+            if (usable < min_pages) continue;
+            const take = @min(usable, max_pages);
+            // The longest run available, so the collection switches extents as
+            // rarely as possible -- every switch abandons what is left of the
+            // one before it. Ties go to the smallest source run, which leaves
+            // the big ones as whole as it can.
+            const better = if (best) |b|
+                take > best_take or
+                    (take == best_take and e.pages < self.free_ready.items[b].pages)
+            else
+                true;
+            if (better) {
+                best = i;
+                best_first = from;
+                best_take = take;
+            }
+        }
+        const i = best orelse return null;
+        const e = self.free_ready.items[i];
+        const head = best_first - e.first;
+        const tail_first = best_first + best_take;
+        const tail = (e.first + e.pages) - tail_first;
+        if (head > 0) {
+            self.free_ready.items[i] = .{ .first = e.first, .pages = head };
+            if (tail > 0) self.free_ready.appendAssumeCapacity(.{ .first = tail_first, .pages = tail });
+        } else if (tail > 0) {
+            self.free_ready.items[i] = .{ .first = tail_first, .pages = tail };
+        } else {
+            _ = self.free_ready.swapRemove(i);
+        }
+        self.reserved_pages -= best_take;
+        hold.pages -= best_take;
+        self.unprotect(best_first, best_take);
+        self.mark_unpublished(best_first, best_take);
+        return .{ .first = best_first, .pages = best_take };
+    }
+
     /// Merge runs that touch, so the holes single-page frees leave behind can add
     /// up to an extent again. Without it the free list only ever fragments: every
     /// generation returns thousands of one-page copy-on-write victims, and an
@@ -1971,6 +2055,65 @@ test "one-page requests do not carve up the runs the extents need" {
     try testing.expectEqual(tail_before, pg.alloc_tail);
 }
 
+test "a slab run comes off the free list aligned, or not at all" {
+    // `alloc_slab_run` is a second allocation policy in the same allocator, so
+    // what it must not do is as important as what it must:
+    //
+    //  - what it hands out starts and ends on a system-page boundary, because
+    //    that is the granularity writeback tears at and the granularity
+    //    reclamation gives back at;
+    //  - a run too short to be worth an extent is left alone;
+    //  - the pieces it trims off stay on the free list rather than leaking.
+    //
+    // Mutation checks: drop the `alignForward`/`alignBackward` and the first
+    // assertion goes red on the odd-page run; drop the `usable < min_pages`
+    // test and the short run is handed out.
+    var threaded: std.Io.Threaded = .init_single_threaded;
+    defer threaded.deinit();
+    const io = threaded.io();
+    var tp = try TmpPager.init(io, 64 << 20);
+    defer tp.deinit();
+    const pg = tp.pg();
+    const spp: u32 = @intCast(map_align / page_size);
+
+    // A long run deliberately starting one 4 KiB page past a boundary, and a
+    // short one, kept apart so coalescing cannot merge them.
+    _ = try pg.alloc_pages(std.mem.alignForward(u32, pg.alloc_tail, spp) - pg.alloc_tail + 1);
+    const long = try pg.alloc_pages(400);
+    _ = try pg.alloc_pages(1); // separator, never freed
+    const short = try pg.alloc_pages(8);
+    _ = try pg.alloc_pages(1); // separator, never freed
+    try testing.expect(long % spp != 0);
+    try pg.publish(.{ .seq = 1 });
+    try pg.free_pages(long, 400);
+    try pg.free_pages(short, 8);
+    try pg.publish(.{ .seq = 2 });
+    try pg.publish(.{ .seq = 3 });
+    const ready_before = pg.free_ready_pages();
+
+    var hold: Reservation = .{};
+    try pg.reserve_pages(&hold, 256);
+    const run = pg.alloc_slab_run(&hold, 64, 256) orelse return error.TestUnexpectedResult;
+    pg.release_reservation(&hold);
+    try testing.expectEqual(@as(u32, 0), run.first % spp);
+    try testing.expectEqual(@as(u32, 0), run.pages % spp);
+    try testing.expectEqual(@as(u32, 256), run.pages);
+    // It came out of the long run, not the short one, and past its unaligned
+    // first page.
+    try testing.expect(run.first > long);
+    try testing.expect(run.first < long + 400);
+    // Everything not handed out is still on the list.
+    try testing.expectEqual(ready_before - 256, pg.free_ready_pages());
+
+    // The short run is below the floor and stays where it is, whatever is asked
+    // of it; nothing else is left long enough either.
+    var hold2: Reservation = .{};
+    try pg.reserve_pages(&hold2, 256);
+    try testing.expect(pg.alloc_slab_run(&hold2, 200, 256) == null);
+    pg.release_reservation(&hold2);
+    try testing.expectEqual(ready_before - 256, pg.free_ready_pages());
+}
+
 test "a quiet checkpoint stops growing the file" {
     // Both streams a publish writes are allocated fresh every time, so that a
     // crash leaves the previous copy readable. Nothing gave them back, and a