diff --git a/src/db.zig b/src/db.zig index 3af904f..d3a8717 100644 --- a/src/db.zig +++ b/src/db.zig @@ -34,6 +34,16 @@ const assert_msg = @import("assert.zig").assert_msg; /// Slack is bounded by one extent per collection. const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size; +/// Shortest run worth taking off the free list for a slab, 1 MiB. Below this +/// the extent is exhausted after a few documents, and every exhaustion writes +/// off whatever is left of the one before it. +const slab_run_min_pages: u32 = slab_extent_pages / 8; + +/// Pages a slab allocation of `len` bytes needs at minimum. +fn pages_for(len: usize) u32 { + return @intCast((len + pgr.page_size - 1) / pgr.page_size); +} + const LogKind = enum { upsert, delete, index_create, index_drop }; /// Dead bytes in one `map_align` window. The window is the unit of reclamation @@ -533,15 +543,22 @@ pub const Collection = struct { const skipped = self.note_skip(self.slab_end - self.slab_tail); // A document larger than the standard extent gets one of its own; BSON // reaches 16 MB and the extent is 8 MiB. - const want_pages: u32 = @intCast(@max( - slab_extent_pages, - (len + pgr.page_size - 1) / pgr.page_size, - )); + const want_pages: u32 = @max(slab_extent_pages, pages_for(len)); try self.pager.reserve_pages(&self.hold, want_pages); - const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages); - try self.insert_run(gpa, first, want_pages); - self.slab_tail = @as(u64, first) << pgr.page_shift; - self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift); + // Off the free list first, or window reclamation is decorative: the + // pages come back, nothing asks for them in a shape they arrive in, and + // the file grows by the whole write volume anyway. A floor of 1 MiB, + // because a shorter extent is exhausted after a handful of documents + // and every exhaustion abandons what is left of it -- and because the + // floor is what makes trimming a larger run harmless. + const min_pages: u32 = @min(want_pages, @max(pages_for(len), slab_run_min_pages)); + const run = self.pager.alloc_slab_run(&self.hold, min_pages, want_pages) orelse pgr.Extent{ + .first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages), + .pages = want_pages, + }; + try self.insert_run(gpa, run.first, run.pages); + self.slab_tail = @as(u64, run.first) << pgr.page_shift; + self.slab_end = self.slab_tail + (@as(u64, run.pages) << pgr.page_shift); return skipped; } @@ -3670,6 +3687,71 @@ test "a reclaimed slab window is not reusable until two publishes later" { try testing.expect(in_extents(engine.pager.free_ready.items, gone)); } +test "a churning collection reuses its slab instead of growing the file" { + // The one that decides whether any of this was worth doing. Reclamation can + // be working perfectly -- windows counted, pages handed back, + // `reclaimed_bytes` climbing -- and the file still grow by the whole write + // volume, because nothing asks for the pages in the shape they come back + // in. That is what `alloc_slab_run` is for, and this is what says so. + // + // Delete-and-refill in rounds, with checkpoints per round so reclamation + // gets to run and what it frees becomes allocatable. The first round has to + // grow the file; the ones after it must not. + // + // Measured here: 2068 pages after the first round, 2083 after three more of + // the same volume -- 15 pages of growth against 4800 pages written. + // + // Mutation check: raise `slab_run_min_pages` to a whole extent, so no + // reclaimed run is ever long enough to qualify and every extent request + // bumps the tail. Red -- which is also the measurement that says the floor + // has to stay well below an extent to be any use. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // reuse, not rebuild + try engine.lock(); + defer engine.unlock(); + + // Documents a whole window wide, which is the case the design is built for + // -- see the note on small documents in the churn gate. + const doc_size = pgr.map_align; + const per_round = 400; + var round: i32 = 0; + var tail_after_first: u32 = 0; + while (round < 4) : (round += 1) { + var i: i32 = 0; + while (i < per_round) : (i += 1) { + var d = try make_padded(gpa, round * per_round + i, doc_size); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + i = 0; + while (i < per_round) : (i += 1) { + _ = try engine.remove_by_id("app", "c", .{ .int32 = round * per_round + i }); + } + try engine.commit(); + // Two, so what this round freed is allocatable in the next one. + try engine.checkpoint(); + try engine.checkpoint(); + if (round == 0) tail_after_first = engine.pager.alloc_tail; + } + + const coll = engine.get_collection("app", "c").?; + try testing.expect(coll.reclaimed_bytes > 0); + // Three more rounds of the same volume after the first. Anything left is + // fragmentation the windows could not cover, not the write volume. + const grew = engine.pager.alloc_tail - tail_after_first; + try testing.expect(grew < 3 * per_round * doc_size / pgr.page_size / 4); +} + test "a replace that changes nothing is not a write" { // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm. // Red on all three: the log grows, the document is superseded so the engine diff --git a/src/pager.zig b/src/pager.zig index 3642a35..aa960c6 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -704,6 +704,90 @@ pub const Pager = struct { return first; } + /// Take a run off the free list for a document slab: at least `min_pages`, + /// at most `max_pages`, starting and ending on a system-page boundary. + /// Returns null when nothing on the list qualifies, and the caller bumps the + /// tail instead. + /// + /// `take_free` cannot serve this, and that is the whole reason this exists. + /// It is deliberately best fit -- smallest sufficient run -- so that the + /// thousands of single-page copy-on-write requests per generation cannot + /// dismantle the large runs. A slab extent asks for 2048 pages, and window + /// reclamation gives back runs a few pages at a time, so with an exact-size + /// rule the free list could fill up with reclaimed slab that no slab request + /// would ever take: the pages come back, the file keeps growing, the ratio + /// does not move. That is the failure this whole milestone is measured + /// against. + /// + /// So this one takes a partial run when it cannot get a whole one, and is + /// allowed to trim a larger one. There is no cannibalisation to fear here: + /// the request is itself large (the caller's floor is 1 MiB), so what it + /// leaves behind is still a usable run rather than a hole. The one-page + /// requests still go through `take_free` unchanged, and its pinned mutation + /// test is untouched. + /// + /// The alignment is not cosmetic. `map_align` is the granularity writeback + /// works at and the granularity reclamation gives back at, so a run that + /// starts mid-system-page both wastes its first window and shares a kernel + /// page with whatever occupies the rest of it -- which for a page still in + /// the published image is the tearing `mark_appendable` refuses to risk. + pub fn alloc_slab_run(self: *Pager, hold: *Reservation, min_pages: u32, max_pages: u32) ?Extent { + assert(min_pages > 0); + assert(min_pages <= max_pages); + self.alloc_lock.lockUncancelable(self.io); + defer self.alloc_lock.unlock(self.io); + assert_msg(max_pages <= hold.pages, "a slab run request overran reserve_pages' promise"); + // A split can leave a piece at each end, so one entry may become two. + // Out of memory before anything is disturbed: the caller falls back to + // bumping the tail, which is what it would have done anyway. + self.free_ready.ensureUnusedCapacity(self.gpa, 1) catch return null; + + const spp: u32 = if (map_align >= page_size) @intCast(map_align / page_size) else 1; + var best: ?usize = null; + var best_first: u32 = 0; + var best_take: u32 = 0; + for (self.free_ready.items, 0..) |e, i| { + const from = std.mem.alignForward(u32, e.first, spp); + const to = std.mem.alignBackward(u32, e.first + e.pages, spp); + if (to <= from) continue; + const usable = to - from; + if (usable < min_pages) continue; + const take = @min(usable, max_pages); + // The longest run available, so the collection switches extents as + // rarely as possible -- every switch abandons what is left of the + // one before it. Ties go to the smallest source run, which leaves + // the big ones as whole as it can. + const better = if (best) |b| + take > best_take or + (take == best_take and e.pages < self.free_ready.items[b].pages) + else + true; + if (better) { + best = i; + best_first = from; + best_take = take; + } + } + const i = best orelse return null; + const e = self.free_ready.items[i]; + const head = best_first - e.first; + const tail_first = best_first + best_take; + const tail = (e.first + e.pages) - tail_first; + if (head > 0) { + self.free_ready.items[i] = .{ .first = e.first, .pages = head }; + if (tail > 0) self.free_ready.appendAssumeCapacity(.{ .first = tail_first, .pages = tail }); + } else if (tail > 0) { + self.free_ready.items[i] = .{ .first = tail_first, .pages = tail }; + } else { + _ = self.free_ready.swapRemove(i); + } + self.reserved_pages -= best_take; + hold.pages -= best_take; + self.unprotect(best_first, best_take); + self.mark_unpublished(best_first, best_take); + return .{ .first = best_first, .pages = best_take }; + } + /// Merge runs that touch, so the holes single-page frees leave behind can add /// up to an extent again. Without it the free list only ever fragments: every /// generation returns thousands of one-page copy-on-write victims, and an @@ -1971,6 +2055,65 @@ test "one-page requests do not carve up the runs the extents need" { try testing.expectEqual(tail_before, pg.alloc_tail); } +test "a slab run comes off the free list aligned, or not at all" { + // `alloc_slab_run` is a second allocation policy in the same allocator, so + // what it must not do is as important as what it must: + // + // - what it hands out starts and ends on a system-page boundary, because + // that is the granularity writeback tears at and the granularity + // reclamation gives back at; + // - a run too short to be worth an extent is left alone; + // - the pieces it trims off stay on the free list rather than leaking. + // + // Mutation checks: drop the `alignForward`/`alignBackward` and the first + // assertion goes red on the odd-page run; drop the `usable < min_pages` + // test and the short run is handed out. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + const spp: u32 = @intCast(map_align / page_size); + + // A long run deliberately starting one 4 KiB page past a boundary, and a + // short one, kept apart so coalescing cannot merge them. + _ = try pg.alloc_pages(std.mem.alignForward(u32, pg.alloc_tail, spp) - pg.alloc_tail + 1); + const long = try pg.alloc_pages(400); + _ = try pg.alloc_pages(1); // separator, never freed + const short = try pg.alloc_pages(8); + _ = try pg.alloc_pages(1); // separator, never freed + try testing.expect(long % spp != 0); + try pg.publish(.{ .seq = 1 }); + try pg.free_pages(long, 400); + try pg.free_pages(short, 8); + try pg.publish(.{ .seq = 2 }); + try pg.publish(.{ .seq = 3 }); + const ready_before = pg.free_ready_pages(); + + var hold: Reservation = .{}; + try pg.reserve_pages(&hold, 256); + const run = pg.alloc_slab_run(&hold, 64, 256) orelse return error.TestUnexpectedResult; + pg.release_reservation(&hold); + try testing.expectEqual(@as(u32, 0), run.first % spp); + try testing.expectEqual(@as(u32, 0), run.pages % spp); + try testing.expectEqual(@as(u32, 256), run.pages); + // It came out of the long run, not the short one, and past its unaligned + // first page. + try testing.expect(run.first > long); + try testing.expect(run.first < long + 400); + // Everything not handed out is still on the list. + try testing.expectEqual(ready_before - 256, pg.free_ready_pages()); + + // The short run is below the floor and stays where it is, whatever is asked + // of it; nothing else is left long enough either. + var hold2: Reservation = .{}; + try pg.reserve_pages(&hold2, 256); + try testing.expect(pg.alloc_slab_run(&hold2, 200, 256) == null); + pg.release_reservation(&hold2); + try testing.expectEqual(ready_before - 256, pg.free_ready_pages()); +} + test "a quiet checkpoint stops growing the file" { // Both streams a publish writes are allocated fresh every time, so that a // crash leaves the previous copy readable. Nothing gave them back, and a