dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

db/pager: a slab extent comes off the free list when one fits

Browse Source 
Without this the previous commit is decorative. Windows go back, the free list
fills up, and the file grows by the whole write volume anyway -- because
nothing asks for the pages in the shape they arrive in.

`take_free` cannot serve a slab extent from reclaimed windows, and that is on
purpose. It is best fit precisely so the thousands of single-page
copy-on-write requests per generation cannot dismantle the large runs; the
consequence is that a 2048-page extent request never matches anything smaller,
and reclamation hands back runs a few windows at a time.

So `alloc_slab_run` is a second policy in the same allocator: at least
`min_pages`, at most `max_pages`, longest available so the collection switches
extents as rarely as possible, ties to the smallest source run so the big ones
stay as whole as they can. It takes a partial run when it cannot have a whole
one and it is allowed to trim a larger one -- there is no cannibalisation to
fear when the request is itself at least 1 MiB, and what it leaves behind is a
run rather than a hole. `take_free` is untouched and its pinned first-fit
mutation test still passes.

What it hands out is aligned to `map_align`, which is not cosmetic. That is
the granularity writeback tears at and the granularity reclamation gives back
at, so a run starting mid-system-page both wastes its first window and shares
a kernel page with whatever holds the rest of it -- for a page still in the
published image, exactly the tearing `mark_appendable` refuses to risk. The
trimmed edges stay on the free list.

The caller's floor is 1 MiB: a shorter extent is exhausted after a handful of
documents and every exhaustion writes off what is left of the one before it.

Measured, in the new engine-level test: delete-and-refill of 400 16 KiB
documents per round, four rounds. The tail stands at 2068 pages after the
first round and 2083 after three more of the same volume -- 15 pages of growth
against 4800 pages written. That is the number the whole milestone is about,
and it is the one Risk 5 in the plan says to check directly rather than
inferring from a ratio.

Three tests. The churn one above. The pager's policy: what it hands out starts
and ends on a system-page boundary, comes out of the long run rather than the
short one and past the long one's unaligned first page, and everything not
handed out is still on the list; a run below the floor is left alone.

Mutations: raise `slab_run_min_pages` to a whole extent and the churn test goes
red (which is also the measurement saying the floor has to stay well under an
extent); drop the alignment and the pager test goes red on an odd-page run;
drop the `usable < min_pages` test and the short run is handed out.

184/184 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, e2e 49, e2e2
concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86,
crash-fuzz 60 cycles.
This commit is contained in:
A.Shakhmatov
2026-08-09 17:03:42 +03:00
parent afdb44fe90
commit 7fe1009243
2 changed files with 233 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
src/db.zig
View File

@@ -34,6 +34,16 @@ const assert_msg = @import("assert.zig").assert_msg;
/// Slack is bounded by one extent per collection.
const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size;
/// Shortest run worth taking off the free list for a slab, 1 MiB. Below this
/// the extent is exhausted after a few documents, and every exhaustion writes
/// off whatever is left of the one before it.
const slab_run_min_pages: u32 = slab_extent_pages / 8;
/// Pages a slab allocation of `len` bytes needs at minimum.
fn pages_for(len: usize) u32 {
return @intCast((len + pgr.page_size - 1) / pgr.page_size);
}
const LogKind = enum { upsert, delete, index_create, index_drop };
/// Dead bytes in one `map_align` window. The window is the unit of reclamation
@@ -533,15 +543,22 @@ pub const Collection = struct {
const skipped = self.note_skip(self.slab_end - self.slab_tail);
// A document larger than the standard extent gets one of its own; BSON
// reaches 16 MB and the extent is 8 MiB.
const want_pages: u32 = @intCast(@max(
slab_extent_pages,
(len + pgr.page_size - 1) / pgr.page_size,
));
const want_pages: u32 = @max(slab_extent_pages, pages_for(len));
try self.pager.reserve_pages(&self.hold, want_pages);
const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages);
try self.insert_run(gpa, first, want_pages);
self.slab_tail = @as(u64, first) << pgr.page_shift;
self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift);
// Off the free list first, or window reclamation is decorative: the
// pages come back, nothing asks for them in a shape they arrive in, and
// the file grows by the whole write volume anyway. A floor of 1 MiB,
// because a shorter extent is exhausted after a handful of documents
// and every exhaustion abandons what is left of it -- and because the
// floor is what makes trimming a larger run harmless.
const min_pages: u32 = @min(want_pages, @max(pages_for(len), slab_run_min_pages));
const run = self.pager.alloc_slab_run(&self.hold, min_pages, want_pages) orelse pgr.Extent{
.first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages),
.pages = want_pages,
};
try self.insert_run(gpa, run.first, run.pages);
self.slab_tail = @as(u64, run.first) << pgr.page_shift;
self.slab_end = self.slab_tail + (@as(u64, run.pages) << pgr.page_shift);
return skipped;
}
@@ -3670,6 +3687,71 @@ test "a reclaimed slab window is not reusable until two publishes later" {
try testing.expect(in_extents(engine.pager.free_ready.items, gone));
}
test "a churning collection reuses its slab instead of growing the file" {
// The one that decides whether any of this was worth doing. Reclamation can
// be working perfectly -- windows counted, pages handed back,
// `reclaimed_bytes` climbing -- and the file still grow by the whole write
// volume, because nothing asks for the pages in the shape they come back
// in. That is what `alloc_slab_run` is for, and this is what says so.
//
// Delete-and-refill in rounds, with checkpoints per round so reclamation
// gets to run and what it frees becomes allocatable. The first round has to
// grow the file; the ones after it must not.
//
// Measured here: 2068 pages after the first round, 2083 after three more of
// the same volume -- 15 pages of growth against 4800 pages written.
//
// Mutation check: raise `slab_run_min_pages` to a whole extent, so no
// reclaimed run is ever long enough to qualify and every extent request
// bumps the tail. Red -- which is also the measurement that says the floor
// has to stay well below an extent to be any use.
var threaded: std.Io.Threaded = .init_single_threaded;
defer threaded.deinit();
var env = test_env(&threaded);
const io = env.io;
const gpa = testing.allocator;
var tmp = try TmpLog.init(gpa);
defer tmp.deinit(gpa);
var engine = try Engine.open(gpa, io, tmp.path);
defer engine.deinit();
engine.compact_threshold = std.math.maxInt(u64); // reuse, not rebuild
try engine.lock();
defer engine.unlock();
// Documents a whole window wide, which is the case the design is built for
// -- see the note on small documents in the churn gate.
const doc_size = pgr.map_align;
const per_round = 400;
var round: i32 = 0;
var tail_after_first: u32 = 0;
while (round < 4) : (round += 1) {
var i: i32 = 0;
while (i < per_round) : (i += 1) {
var d = try make_padded(gpa, round * per_round + i, doc_size);
defer d.deinit();
try engine.insert("app", "c", &d, &env.gen);
}
try engine.commit();
i = 0;
while (i < per_round) : (i += 1) {
_ = try engine.remove_by_id("app", "c", .{ .int32 = round * per_round + i });
}
try engine.commit();
// Two, so what this round freed is allocatable in the next one.
try engine.checkpoint();
try engine.checkpoint();
if (round == 0) tail_after_first = engine.pager.alloc_tail;
}
const coll = engine.get_collection("app", "c").?;
try testing.expect(coll.reclaimed_bytes > 0);
// Three more rounds of the same volume after the first. Anything left is
// fragmentation the windows could not cover, not the write volume.
const grew = engine.pager.alloc_tail - tail_after_first;
try testing.expect(grew < 3 * per_round * doc_size / pgr.page_size / 4);
}
test "a replace that changes nothing is not a write" {
// Mutation check: delete the byte comparison in `upsert`'s `.replace` arm.
// Red on all three: the log grows, the document is superseded so the engine

143
src/pager.zig
View File

@@ -704,6 +704,90 @@ pub const Pager = struct {
return first;
}
/// Take a run off the free list for a document slab: at least `min_pages`,
/// at most `max_pages`, starting and ending on a system-page boundary.
/// Returns null when nothing on the list qualifies, and the caller bumps the
/// tail instead.
///
/// `take_free` cannot serve this, and that is the whole reason this exists.
/// It is deliberately best fit -- smallest sufficient run -- so that the
/// thousands of single-page copy-on-write requests per generation cannot
/// dismantle the large runs. A slab extent asks for 2048 pages, and window
/// reclamation gives back runs a few pages at a time, so with an exact-size
/// rule the free list could fill up with reclaimed slab that no slab request
/// would ever take: the pages come back, the file keeps growing, the ratio
/// does not move. That is the failure this whole milestone is measured
/// against.
///
/// So this one takes a partial run when it cannot get a whole one, and is
/// allowed to trim a larger one. There is no cannibalisation to fear here:
/// the request is itself large (the caller's floor is 1 MiB), so what it
/// leaves behind is still a usable run rather than a hole. The one-page
/// requests still go through `take_free` unchanged, and its pinned mutation
/// test is untouched.
///
/// The alignment is not cosmetic. `map_align` is the granularity writeback
/// works at and the granularity reclamation gives back at, so a run that
/// starts mid-system-page both wastes its first window and shares a kernel
/// page with whatever occupies the rest of it -- which for a page still in
/// the published image is the tearing `mark_appendable` refuses to risk.
pub fn alloc_slab_run(self: *Pager, hold: *Reservation, min_pages: u32, max_pages: u32) ?Extent {
assert(min_pages > 0);
assert(min_pages <= max_pages);
self.alloc_lock.lockUncancelable(self.io);
defer self.alloc_lock.unlock(self.io);
assert_msg(max_pages <= hold.pages, "a slab run request overran reserve_pages' promise");
// A split can leave a piece at each end, so one entry may become two.
// Out of memory before anything is disturbed: the caller falls back to
// bumping the tail, which is what it would have done anyway.
self.free_ready.ensureUnusedCapacity(self.gpa, 1) catch return null;
const spp: u32 = if (map_align >= page_size) @intCast(map_align / page_size) else 1;
var best: ?usize = null;
var best_first: u32 = 0;
var best_take: u32 = 0;
for (self.free_ready.items, 0..) |e, i| {
const from = std.mem.alignForward(u32, e.first, spp);
const to = std.mem.alignBackward(u32, e.first + e.pages, spp);
if (to <= from) continue;
const usable = to - from;
if (usable < min_pages) continue;
const take = @min(usable, max_pages);
// The longest run available, so the collection switches extents as
// rarely as possible -- every switch abandons what is left of the
// one before it. Ties go to the smallest source run, which leaves
// the big ones as whole as it can.
const better = if (best) |b|
take > best_take or
(take == best_take and e.pages < self.free_ready.items[b].pages)
else
true;
if (better) {
best = i;
best_first = from;
best_take = take;
}
}
const i = best orelse return null;
const e = self.free_ready.items[i];
const head = best_first - e.first;
const tail_first = best_first + best_take;
const tail = (e.first + e.pages) - tail_first;
if (head > 0) {
self.free_ready.items[i] = .{ .first = e.first, .pages = head };
if (tail > 0) self.free_ready.appendAssumeCapacity(.{ .first = tail_first, .pages = tail });
} else if (tail > 0) {
self.free_ready.items[i] = .{ .first = tail_first, .pages = tail };
} else {
_ = self.free_ready.swapRemove(i);
}
self.reserved_pages -= best_take;
hold.pages -= best_take;
self.unprotect(best_first, best_take);
self.mark_unpublished(best_first, best_take);
return .{ .first = best_first, .pages = best_take };
}
/// Merge runs that touch, so the holes single-page frees leave behind can add
/// up to an extent again. Without it the free list only ever fragments: every
/// generation returns thousands of one-page copy-on-write victims, and an
@@ -1971,6 +2055,65 @@ test "one-page requests do not carve up the runs the extents need" {
try testing.expectEqual(tail_before, pg.alloc_tail);
}
test "a slab run comes off the free list aligned, or not at all" {
// `alloc_slab_run` is a second allocation policy in the same allocator, so
// what it must not do is as important as what it must:
//
// - what it hands out starts and ends on a system-page boundary, because
// that is the granularity writeback tears at and the granularity
// reclamation gives back at;
// - a run too short to be worth an extent is left alone;
// - the pieces it trims off stay on the free list rather than leaking.
//
// Mutation checks: drop the `alignForward`/`alignBackward` and the first
// assertion goes red on the odd-page run; drop the `usable < min_pages`
// test and the short run is handed out.
var threaded: std.Io.Threaded = .init_single_threaded;
defer threaded.deinit();
const io = threaded.io();
var tp = try TmpPager.init(io, 64 << 20);
defer tp.deinit();
const pg = tp.pg();
const spp: u32 = @intCast(map_align / page_size);
// A long run deliberately starting one 4 KiB page past a boundary, and a
// short one, kept apart so coalescing cannot merge them.
_ = try pg.alloc_pages(std.mem.alignForward(u32, pg.alloc_tail, spp) - pg.alloc_tail + 1);
const long = try pg.alloc_pages(400);
_ = try pg.alloc_pages(1); // separator, never freed
const short = try pg.alloc_pages(8);
_ = try pg.alloc_pages(1); // separator, never freed
try testing.expect(long % spp != 0);
try pg.publish(.{ .seq = 1 });
try pg.free_pages(long, 400);
try pg.free_pages(short, 8);
try pg.publish(.{ .seq = 2 });
try pg.publish(.{ .seq = 3 });
const ready_before = pg.free_ready_pages();
var hold: Reservation = .{};
try pg.reserve_pages(&hold, 256);
const run = pg.alloc_slab_run(&hold, 64, 256) orelse return error.TestUnexpectedResult;
pg.release_reservation(&hold);
try testing.expectEqual(@as(u32, 0), run.first % spp);
try testing.expectEqual(@as(u32, 0), run.pages % spp);
try testing.expectEqual(@as(u32, 256), run.pages);
// It came out of the long run, not the short one, and past its unaligned
// first page.
try testing.expect(run.first > long);
try testing.expect(run.first < long + 400);
// Everything not handed out is still on the list.
try testing.expectEqual(ready_before - 256, pg.free_ready_pages());
// The short run is below the floor and stays where it is, whatever is asked
// of it; nothing else is left long enough either.
var hold2: Reservation = .{};
try pg.reserve_pages(&hold2, 256);
try testing.expect(pg.alloc_slab_run(&hold2, 200, 256) == null);
pg.release_reservation(&hold2);
try testing.expectEqual(ready_before - 256, pg.free_ready_pages());
}
test "a quiet checkpoint stops growing the file" {
// Both streams a publish writes are allocated fresh every time, so that a
// crash leaves the previous copy readable. Nothing gave them back, and a