From cd88e1a4d1f5dcb5591564874ed99466e249b919 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Tue, 4 Aug 2026 14:51:56 +0300 Subject: [PATCH 01/37] index/pager: place a split's new sibling positionally, and fix mmap growth alignment Two bugs, both of which the crash fuzzer surfaced and neither of which any existing test could see. **A split put the new sibling in the wrong slot when separators repeat.** `split_leaf` located the new right sibling with `separator_pos(node, key)`, a search for the promoted key. That agrees with "immediately after `left`" only while separators are distinct. When several children share one -- ten distinct values across thousands of documents, so each value spans dozens of leaves -- `separator_pos` returns the slot after the *whole* equal-key run, which puts the sibling at the end of that run while the leaf chain has it right after `left`. Parent child order then stops matching leaf chain order, and that is the one thing a lookup cannot survive: `descend_lower` picks the last child of the equal run, and `lookup_eq` walks forward from there over keys *smaller* than the one it wants, stops at the first mismatch, and reports nothing. Every entry is present, the chain is correctly ordered, `count()` is right -- and the query returns empty. `crash-fuzz.js` found it after ~700 heavy cycles as `find({k: 3})` returning 0 of 401 documents while every other key was exact. Fixed by `child_slot_after`, which is positional by construction. **mmap growth rounded with a non-power-of-two alignment.** Past 64 MiB the growth chunk becomes a proportion of the current size (`mapped_pages / 8`), which is not a power of two -- and `std.mem.alignForward` asserts that it is. In safe builds that panicked; in ReleaseFast, where the assert is compiled out, it computed `(addr + align - 1) & ~(align - 1)` with a non-power-of-two mask, which can round *down*. A mapping shorter than intended is survivable, but a mapping longer than the file is exactly what this function exists to prevent: a store into a mapped page past end-of-file raises SIGBUS, which no error path catches. `alignForwardAnyAlign` instead. Never noticed because no unit test grew a pager past 64 MiB. Also here, because both bugs were invisible rather than merely unfixed: - `assert_indexes_cover_every_document` (db.zig) checks the index invariant directly -- an index generates candidates and the full filter is re-applied to those, so a missing entry is a missing query result nothing else detects. - `Index.unreachable_key_count` counts keys present in the leaf chain but not reachable by descending from the root, which is precisely the state above: healthy by every other measure. - `Index.dbg_root` dumps parent/chain agreement. Marked TEMPORARY; drop it once the invariant checks have earned their keep. - `crash-fuzz.js` now asks the same question without the index, so a failure says whether the documents are wrong or only the index's answer about them, and reports per-key totals so one lost leaf is distinguishable from an empty index. Verified: `zig build test` in ReleaseFast and ReleaseSafe, and seeded fuzzer runs that previously reproduced the split bug. --- src/db.zig | 288 +++++++++++++++++++++++++++++++++++-- src/index.zig | 303 ++++++++++++++++++++++++++++++++++++++- src/pager.zig | 14 +- tests/fuzz/crash-fuzz.js | 35 ++++- 4 files changed, 624 insertions(+), 16 deletions(-) diff --git a/src/db.zig b/src/db.zig index 4ac2479..771a0e9 100644 --- a/src/db.zig +++ b/src/db.zig @@ -1,11 +1,23 @@ -//! In-memory database engine backed by the append-only log. Maps -//! db -> collection -> _id(serialized) -> owned Document. All mutations are -//! logged and synced before they become visible in memory, so a crash never -//! loses a committed write. Callers must hold the write lock (`lock`) around -//! any command that mutates state, and the read lock (`lock_read`) around -//! read-only commands so reads overlap with each other. +//! Database engine over an mmap'd data file with the append-only log in front +//! of it as the write-ahead log. Maps db -> collection -> `_id_` B+tree -> +//! absolute slab offset; documents, tree pages and overflow records all live in +//! the data file, so resident memory is the working set rather than the size of +//! the database. All mutations are logged and synced before they become visible, +//! so a crash never loses a committed write, and a checkpoint publishes the data +//! file and truncates the log so an open does not replay everything ever +//! written. Callers must hold the write lock (`lock`) around any command that +//! mutates state, and the read lock (`lock_read`) around read-only commands so +//! reads overlap with each other. +//! +//! Two invariants the rest of this file depends on. A checkpoint never renumbers +//! slab offsets, because index leaves hold them physically -- only `compact` +//! moves documents, and it rebuilds every index in the same pass. And an index +//! must never under-approximate: it generates candidates and the full filter is +//! re-applied to those, so a missing entry is a missing query result that +//! nothing else detects (see `assert_indexes_cover_every_document`). const std = @import("std"); +const builtin = @import("builtin"); const bson = @import("bson.zig"); const storage = @import("storage.zig"); const index = @import("index.zig"); @@ -374,6 +386,7 @@ pub const Engine = struct { // once replay completes (order-independent). A checkpointed open finds // them already populated, and the guard in rebuild_index skips them. try engine.build_all_indexes(); + engine.assert_indexes_cover_every_document(); // Everything replayed is durable by definition -- it was read back off // the log -- so the commit watermark starts level with the sequence. engine.committed_seq = engine.seq; @@ -1408,6 +1421,55 @@ pub const Engine = struct { /// as a candidate generator; future writes are still enforced) — the /// database always opens, leaving dropIndexes as an in-band recovery /// path. + /// Every document is reachable through every index that is supposed to cover + /// it, checked once at the end of an open. + /// + /// An index that is merely *incomplete* is the worst failure this engine can + /// have, because nothing reports it: the index only generates candidates and + /// the full filter is re-applied to those, so a missing entry is a missing + /// query result and every other check still passes. That is exactly how the + /// replay-time `createIndex` bug survived -- `countDocuments` was right, + /// `find({})` was right, and only `find({k: v})` was quietly short. + /// + /// `_id_` is exact: one entry per document, always. A secondary index is + /// checked only when its shape makes the count exact -- `sparse` omits + /// documents missing the key, and `multikey` contributes several entries for + /// one document -- so those are compared as a lower bound instead of an + /// equality. Debug and ReleaseSafe only; an open is not a hot path, but a + /// full index walk per collection is not free either. + fn assert_indexes_cover_every_document(self: *Engine) void { + if (builtin.mode == .ReleaseFast or builtin.mode == .ReleaseSmall) return; + var db_it = self.dbs.iterator(); + while (db_it.next()) |db_entry| { + var coll_it = db_entry.value_ptr.collections.iterator(); + while (coll_it.next()) |coll_entry| { + const coll = coll_entry.value_ptr.*; + assert_msg( + coll.id_index.count() == coll.doc_count, + "the _id_ index must hold exactly one entry per document after an open", + ); + assert_msg( + coll.id_index.unreachable_key_count() == 0, + "every _id_ entry must be findable by descent, not only by iteration", + ); + for (coll.indexes.items) |ix| { + // Reachability applies to every index whatever its shape: an + // entry in the leaf chain that a descent cannot find is a + // query result that silently goes missing. + if (ix.unreachable_key_count() != 0) { + ix.dbg_root(); + @panic("unreachable index entries"); + } + if (ix.sparse or ix.multikey) continue; + assert_msg( + ix.count() >= coll.doc_count, + "a non-sparse index must cover every document after an open", + ); + } + } + } + } + fn build_all_indexes(self: *Engine) !void { var db_it = self.dbs.iterator(); while (db_it.next()) |db_entry| { @@ -1812,11 +1874,15 @@ pub const Engine = struct { /// Register an (empty) index from a persisted spec document. A repeated /// create record for the same name is an idempotent no-op. + /// Register an index from a logged spec. Returns the new index, or null when + /// one of that name was already present (a re-registration is a no-op, not an + /// error). The caller needs the pointer because an index registered during + /// replay may have to be built over documents that replay will never see. fn register_index_from_spec( self: *Engine, coll: *Collection, spec_doc: *const bson.Document, - ) !void { + ) !?*index.Index { const parsed = try index.parse_spec(self.gpa, self.pager, spec_doc); const ix = self.gpa.create(index.Index) catch |err| { var dead = parsed; @@ -1829,9 +1895,10 @@ pub const Engine = struct { ix.deinit(self.gpa); self.gpa.destroy(ix); }; - if (coll.find_index(ix.name) != null) return; + if (coll.find_index(ix.name) != null) return null; try coll.indexes.append(self.gpa, ix); committed = true; + return ix; } }; @@ -1864,12 +1931,38 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an // after replay completes. switch (record.type) { storage.record_type_index_create => { - self.register_index_from_spec(coll, doc) catch |err| { + const registered = self.register_index_from_spec(coll, doc) catch |err| { std.debug.print("multiforadb: index create record failed to apply: {s}\n", .{ @errorName(err), }); return; }; + // A checkpointed open replays only what the watermark does not cover, + // so the documents already in the image never reach this index. Build + // it over them now, which is what the live `create_index` command + // does with pre-existing documents. + // + // Leaving it to `build_all_indexes` does not work and fails silently: + // the next upsert in the log puts one entry in, and a non-empty index + // is skipped by the `count() > 0` guard there -- so the index ends up + // holding the documents logged after its creation and none of the + // ones logged before, which is an index that under-approximates. + // + // Only for a maintaining replay. A full replay leaves every secondary + // index empty on purpose and `build_all_indexes` fills them in one + // pass at the end, which is cheaper than one pass per index here. + if (self.replay_maintains_indexes) { + if (registered) |ix| self.rebuild_index(coll, ix) catch |err| { + // The database must always open (ground rule 4). A failure + // here leaves the index short, so say so rather than leaving + // a query to be quietly wrong about it. + std.debug.print( + "multiforadb: WARNING: index '{s}' could not be built over existing " ++ + "documents during replay: {s}; drop and re-create it\n", + .{ ix.name, @errorName(err) }, + ); + }; + } return; }, storage.record_type_index_drop => { @@ -2106,6 +2199,174 @@ test "compaction reclaims garbage but leaves a garbage-free log alone" { try testing.expect(engine.log.data_bytes < after_insert * 2); } +test "a secondary index stays reachable across checkpoints, churn and a rebuild" { + // The one path the index unit tests cannot reach: copy-on-write. `test_pager` + // never publishes a watermark, so `stable_pages` is 0 there and every page is + // writable in place -- no node page is ever relocated. Through the engine a + // checkpoint makes the whole image stable, so the next tree mutation copies + // each node it touches to a fresh page and rewrites the id->page slot. + // + // Few distinct keys on purpose: ten values over thousands of documents means + // each value spans many leaves and most interior separators are duplicates, + // which is the shape the crash fuzzer fails on. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = 64 * 1024; // rebuild often, like --heavy + try engine.lock(); + defer engine.unlock(); + + var spec = try index_spec(gpa, "k", "k_1", false, false, null); + defer spec.deinit(); + _ = try engine.create_index("app", "c", &spec); + + const n_keys: i32 = 10; + const n: i32 = 1200; + var id: i32 = 0; + while (id < n) : (id += 1) { + var d = try make_keyed(gpa, id, @mod(id, n_keys)); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + + // Checkpoint, churn and rebuild interleaved with the writes, so tree + // mutations land on pages the last checkpoint froze. + if (@mod(id, 150) == 0) { + try engine.commit(); + try engine.checkpoint(); + } + if (@mod(id, 7) == 0 and id > 20) { + _ = try engine.remove_by_id("app", "c", .{ .int32 = id - 20 }); + } + if (engine.take_compact()) try engine.compact(); + } + try engine.commit(); + try engine.checkpoint(); + + const coll = engine.get_collection("app", "c").?; + const ix = coll.find_index("k_1").?; + + // Every entry the leaf chain holds must also be findable by descending from + // the root, which is the only way a query reaches it. + try testing.expectEqual(@as(u32, 0), ix.unreachable_key_count()); + try testing.expectEqual(@as(u32, 0), coll.id_index.unreachable_key_count()); + + // And per key, the index must agree with a scan of the documents. + var k: i32 = 0; + while (k < n_keys) : (k += 1) { + var want: usize = 0; + var scan = coll.id_index.iter(); + while (scan.next()) |e| { + const kv = try bson.get_at(gpa, coll.doc_bytes(e.off), "k"); + if (kv) |v| if (v.int32 == k) { + want += 1; + }; + } + var out: std.ArrayListUnmanaged(u64) = .empty; + defer out.deinit(gpa); + try ix.lookup_eq(gpa, &.{.{ .int32 = k }}, &out); + testing.expectEqual(want, out.items.len) catch |err| { + std.debug.print(" key {d}: index {d}, scan {d}, entry_count {d}\n", .{ + k, + out.items.len, + want, + ix.count(), + }); + return err; + }; + } +} + +test "an index created after the checkpoint indexes the documents that predate it" { + // Found by tests/fuzz/crash-fuzz.js in --heavy mode, roughly once per 700 + // crash/reopen cycles, as `find({k:v})` returning nothing for a key that has + // documents. `countDocuments` and `find({})` were right, so the documents + // were there and only the index's answer about them was wrong -- an index + // that under-approximates, which is silent by construction: the index only + // generates candidates and the full filter is re-applied to those, so a + // missing entry is a missing result and nothing complains. + // + // The sequence needs three things at once: a checkpoint, a `createIndex` + // logged after it, and a write after that. + // + // 1. documents exist and a checkpoint puts them in the durable image + // 2. createIndex is logged *after* the watermark + // 3. another document is written, also after the watermark + // + // On reopen the catalog restores step 1's documents but not the index, so + // replay starts at the watermark and never sees them. Replay registers the + // index empty and -- because a checkpointed open maintains indexes as it + // replays -- step 3's document goes in. The index is now non-empty and + // incomplete, so `rebuild_index`'s `count() > 0` guard skips it and step 1's + // documents are never indexed at all. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + { + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + try engine.lock(); + + // 1. Two documents, made durable in the data file. + var d1 = try make_user(gpa, 1, "a@x.io"); + defer d1.deinit(); + var d2 = try make_user(gpa, 2, "b@x.io"); + defer d2.deinit(); + try engine.insert("app", "users", &d1, &env.gen); + try engine.insert("app", "users", &d2, &env.gen); + try engine.commit(); + try engine.checkpoint(); + + // 2. The index arrives after the watermark. + var spec = try index_spec(gpa, "email", "email_1", false, false, null); + defer spec.deinit(); + _ = try engine.create_index("app", "users", &spec); + + // 3. And a write after that, which is what makes the index non-empty on + // replay and so hides the two documents behind the `count() > 0` guard. + var d3 = try make_user(gpa, 3, "c@x.io"); + defer d3.deinit(); + try engine.insert("app", "users", &d3, &env.gen); + try engine.commit(); + engine.unlock(); + // No second checkpoint: the watermark still predates the createIndex. + } + + var engine2 = try Engine.open(gpa, io, tmp.path); + defer engine2.deinit(); + const coll = engine2.get_collection("app", "users").?; + const ix = coll.find_index("email_1").?; + + // One entry per document. Under-approximation is the whole failure mode, so + // the count is the assertion that matters. + try testing.expectEqual(@as(u64, 3), coll.doc_count); + try testing.expectEqual(@as(usize, 3), ix.count()); + + // And every entry resolves to a document whose email re-encodes to its key, + // so the entries are the right ones and not merely the right number. + var seen: [3]bool = .{ false, false, false }; + var it = ix.iter(); + while (it.next()) |entry| { + const doc_id = (try bson.get_at(gpa, coll.doc_bytes(entry.off), "_id")).?; + const idx: usize = @intCast(doc_id.int32 - 1); + try testing.expect(idx < seen.len); + try testing.expect(!seen[idx]); + seen[idx] = true; + } + try testing.expect(seen[0] and seen[1] and seen[2]); +} + test "reopening without a checkpoint reuses the data file instead of appending to it" { // Mutation check: delete the `loaded.generation == 0` reset of `alloc_tail` // in `Pager.open`. Red -- each reopen starts allocating above the previous @@ -2831,6 +3092,15 @@ fn index_count( return 0; } +fn make_keyed(gpa: std.mem.Allocator, id: i32, k: i32) !bson.Document { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const pairs = try arena.allocator().alloc(bson.Pair, 2); + pairs[0] = .{ .key = try arena.allocator().dupe(u8, "_id"), .value = .{ .int32 = id } }; + pairs[1] = .{ .key = try arena.allocator().dupe(u8, "k"), .value = .{ .int32 = k } }; + return .{ .arena = arena, .pairs = pairs }; +} + fn make_user(gpa: std.mem.Allocator, id: i32, email: []const u8) !bson.Document { var arena = std.heap.ArenaAllocator.init(gpa); errdefer arena.deinit(); diff --git a/src/index.zig b/src/index.zig index 715b8d0..afda333 100644 --- a/src/index.zig +++ b/src/index.zig @@ -798,6 +798,70 @@ pub const Index = struct { return .{ .ix = self, .leaf = self.first_leaf, .slot = 0 }; } + /// TEMPORARY: does the parent's child order match the leaf chain, and is each + /// separator really its child's first key? + pub fn dbg_root(self: *const Index) void { + const rt = self.page(self.root); + std.debug.print(" root={d} count={d} first_child={d} depth={d}\n", .{ self.root, rt.count, rt.first_child, self.depth }); + // chain position of every leaf + var pos_of = std.mem.zeroes([512]i32); + for (&pos_of) |*v| v.* = -1; + var lf = self.first_leaf; + var pos: i32 = 0; + while (lf != 0) : (pos += 1) { + if (lf < pos_of.len) pos_of[lf] = pos; + lf = self.page(lf).next; + } + var prev_pos: i32 = pos_of[rt.first_child]; + var i: u32 = 0; + while (i < rt.count) : (i += 1) { + const child = get_slot(rt, i).extra; + const cp = if (child < pos_of.len) pos_of[child] else -2; + const sep = self.key_of(self.root, i); + const cfirst = if (self.page(child).count > 0) self.key_of(child, 0) else ""; + const sep_wrong = cfirst.len > 0 and !std.mem.eql(u8, sep, cfirst); + const order_wrong = cp != prev_pos + 1; + if (sep_wrong or order_wrong) { + std.debug.print(" slot[{d}] child={d} chainpos={d} (prev {d}){s}{s}\n sep ={x}\n first={x}\n", .{ + i, child, cp, prev_pos, + if (order_wrong) " ORDER" else "", if (sep_wrong) " SEP!=FIRST" else "", sep, cfirst, + }); + } + prev_pos = cp; + } + } + + /// Debug aid: how many distinct keys are present in the leaf chain but not + /// findable by descending from the root. + /// + /// Iteration and descent are two independent ways to reach an entry, and a + /// query only ever uses descent. `count()` cannot tell them apart -- it + /// returns a stored counter -- so an index whose leaves are intact but whose + /// interior nodes no longer route to them looks perfectly healthy by every + /// other measure, and silently answers a query with fewer documents than it + /// holds. That is what the crash fuzzer caught: a handful of key values + /// returning nothing while every other value was exact. + /// + /// O(distinct keys x depth). For assertions and tests, not for the hot path. + pub fn unreachable_key_count(self: *const Index) u32 { + var bad: u32 = 0; + var it = self.iter(); + var prev: ?[]const u8 = null; + while (it.next()) |e| { + if (prev) |p| { + if (std.mem.eql(u8, p, e.key)) continue; + } + prev = e.key; + var probe = self.seek(e.key); + const first = probe.next() orelse { + bad += 1; + continue; + }; + if (cmp_prefix(e.key, first.key) != .eq) bad += 1; + } + return bad; + } + /// Reverse ordered iteration. Leaves are doubly linked and `prev` has /// always been maintained -- nothing walked it until now, so a descending /// scan had to materialize every candidate and reverse the list. This turns @@ -1242,6 +1306,36 @@ pub const Index = struct { /// Separator position in an internal node: after any equal keys, so the /// "last separator <= key" descent lands on the newest right child. + /// The slot at which a new right sibling of `left` belongs: immediately after + /// `left`'s own position among this node's children. + /// + /// Deliberately positional, not a search for the promoted key. The two agree + /// only while separators are distinct. When several children share a + /// separator -- ten distinct values across thousands of documents, so each + /// value spans dozens of leaves -- `separator_pos` returns the slot after the + /// *whole* equal-key run, which puts the new sibling at the end of that run + /// while the leaf chain has it immediately after `left`. + /// + /// Parent child order then no longer matches leaf chain order, and that is + /// the one thing a lookup cannot survive: `descend_lower` picks the last + /// child of the equal run, and `lookup_eq` walks forward from there over keys + /// that are *smaller* than the one it wants, so it stops at the first + /// mismatch and reports nothing. The entries are all present, the chain is + /// correctly ordered, `count()` is right -- and a query returns an empty + /// result. Found by tests/fuzz/crash-fuzz.js after ~700 heavy cycles as + /// `find({k: 3})` returning 0 of 401 documents while every other key was + /// exact. + fn child_slot_after(self: *const Index, node_id: u32, left: u32) u32 { + const node = self.page(node_id); + if (node.first_child == left) return 0; + var i: u32 = 0; + while (i < node.count) : (i += 1) { + if (get_slot(node, i).extra == left) return i + 1; + } + assert_msg(false, "a split's left sibling must be a child of the node taking its separator"); + return node.count; + } + fn separator_pos(self: *const Index, node_id: u32, key: []const u8) u32 { const node = self.page(node_id); var lo: u32 = 0; @@ -1352,7 +1446,7 @@ pub const Index = struct { } const child = self.descend_insert(node_id, key); const res = self.insert_rec(child, key, off) orelse return null; - return self.insert_separator(node_id, res); + return self.insert_separator(node_id, child, res); } /// Split a full leaf around the record being inserted. The new record @@ -1426,7 +1520,7 @@ pub const Index = struct { /// Insert a promoted separator into an internal node, splitting it when /// full. Returns the next promotion, or null. - fn insert_separator(self: *Index, node_id: u32, split: Split) ?Split { + fn insert_separator(self: *Index, node_id: u32, left: u32, split: Split) ?Split { // The incoming key may live in the promo buffer, which a nested // split_internal (below) would overwrite with its own promoted key; // spilled keys already live in the immutable slab. Copy inline keys @@ -1443,7 +1537,7 @@ pub const Index = struct { self.repack_keep_prefix(node_id, self.page(node_id).count); } if (self.fits(node_id, key.len)) { - self.store_record(node_id, self.separator_pos(node_id, key), .{ + self.store_record(node_id, self.child_slot_after(node_id, left), .{ .key = key, .child = split.right, .spill_off = split.spill_off, @@ -1451,7 +1545,7 @@ pub const Index = struct { self.page_mut(split.right).parent = node_id; return null; } - return self.split_internal(node_id, key, split.spill_off, split.right); + return self.split_internal(node_id, left, key, split.spill_off, split.right); } /// Split a full internal node around the separator being inserted: the @@ -1462,13 +1556,15 @@ pub const Index = struct { fn split_internal( self: *Index, node_id: u32, + left: u32, key: []const u8, spill_off: ?u64, child: u32, ) Split { const old_count = self.page(node_id).count; std.debug.assert(old_count >= 2); - const pos = self.separator_pos(node_id, key); + // Positional, for the reason `child_slot_after` documents. + const pos = self.child_slot_after(node_id, left); const n = old_count + 1; var costs: [max_slots + 1]u32 = undefined; @@ -2345,6 +2441,50 @@ fn simple_index( return Index.init(gpa, pager, "test", keys[0..paths.len], unique, sparse, null); } +test "a bulk build with many duplicate keys stays reachable for every key" { + // The shape the crash fuzzer failed on: ~4000 entries over 10 distinct key + // values, so each value spans several leaves and the interior separators + // repeat. `find({k:v})` came back empty for a few values and exactly right + // for the rest, with `count()` still reporting every entry -- entries that + // exist and cannot be reached. + const gpa = testing.allocator; + var ix = try simple_index(gpa, test_pager(), &.{"k"}, false, false); + defer ix.deinit(gpa); + + const n_docs: usize = 4000; + const n_keys: i32 = 10; + var docs: std.ArrayListUnmanaged([]u8) = .empty; + defer { + for (docs.items) |d| gpa.free(d); + docs.deinit(gpa); + } + for (0..n_docs) |i| { + const k: i32 = @intCast(@mod(@as(i32, @intCast(i)), n_keys)); + const pairs = [_]bson.Pair{.{ .key = "k", .value = .{ .int32 = k } }}; + const bytes = try bytes_of(gpa, &pairs); + try docs.append(gpa, bytes); + try ix.append_doc_entries(gpa, bytes, @intCast(i + 1)); + } + _ = try ix.finish_bulk(gpa, false); + try testing.expectEqual(n_docs, ix.count()); + + // Iteration must see every entry: that separates "never inserted" from + // "inserted and unreachable from the root". + var walked: usize = 0; + var wit = ix.iter(); + while (wit.next()) |_| walked += 1; + try testing.expectEqual(n_docs, walked); + + // And every key must be reachable by descent, which is what a query does. + var k: i32 = 0; + while (k < n_keys) : (k += 1) { + var out: std.ArrayListUnmanaged(u64) = .empty; + defer out.deinit(gpa); + try ix.lookup_eq(gpa, &.{.{ .int32 = k }}, &out); + try testing.expectEqual(n_docs / @as(usize, @intCast(n_keys)), out.items.len); + } +} + /// Look up the documents under `key` and compare with the expected offsets. /// A leaf record's payload is a slab offset now, so tests identify documents by /// small distinct numbers rather than by byte-string ids. @@ -2884,6 +3024,159 @@ test "incremental inserts and removals stay identical to a brute-force model" { } } +test "splitting inside a run of equal separators keeps every key reachable" { + // The crash fuzzer's exact shape, and the reason the two differentials above + // miss it: bulk-pack first, *then* keep inserting. + // + // A packed tree has full leaves, so the next inserts split leaves in the + // middle of a run of equal separators -- and a new right sibling placed by + // key rather than by position lands at the end of that run, so the parent's + // child order stops matching the leaf chain. A purely incremental build + // leaves leaves half full and rarely produces the geometry. + const gpa = testing.allocator; + var ix = try simple_index(gpa, test_pager(), &.{"k"}, false, false); + defer ix.deinit(gpa); + + const n_keys: i32 = 10; + const packed_docs: usize = 4000; + const grown_docs: usize = 2000; + + // Phase 1: bulk pack, which fills every leaf. + for (0..packed_docs) |i| { + const k: i32 = @intCast(@mod(@as(i32, @intCast(i)), n_keys)); + const d = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = k } }}); + defer gpa.free(d); + try ix.append_doc_entries(gpa, d, @intCast(i + 1)); + } + _ = try ix.finish_bulk(gpa, false); + ix.pager.release_reservation(&ix.hold); + + // Phase 2: grow it in random key order, which is what puts a split on the + // leaf *before* an equal-key run -- the case where the new right sibling's + // promoted key equals the run's key while its chain position is at the run's + // start. Round-robin never produces it: every insert routes to the last + // child of its run, and a split there belongs at the end of the run anyway. + var prng = std.Random.DefaultPrng.init(0xbad_5eed); + const rand = prng.random(); + for (0..grown_docs) |j| { + const k = rand.intRangeLessThan(i32, 0, n_keys); + const d = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = k } }}); + defer gpa.free(d); + _ = try ix.add_doc(gpa, d, @intCast(packed_docs + j + 1), false); + ix.pager.release_reservation(&ix.hold); + if (j % 25 == 0) { + testing.expectEqual(@as(u32, 0), ix.unreachable_key_count()) catch |err| { + std.debug.print(" went unreachable after {d} grown inserts\n", .{j}); + return err; + }; + } + } + + try testing.expectEqual(packed_docs + grown_docs, ix.count()); + // Every entry the chain holds must be findable by descent too. + try testing.expectEqual(@as(u32, 0), ix.unreachable_key_count()); + + // And a lookup must find as many entries as iteration holds for that key. + var k: i32 = 0; + while (k < n_keys) : (k += 1) { + var enc: std.ArrayListUnmanaged(u8) = .empty; + defer enc.deinit(gpa); + try bson.encode_key(bson.Value{ .int32 = k }, gpa, &enc); + var want: usize = 0; + var wit = ix.iter(); + while (wit.next()) |e| { + if (cmp_prefix(enc.items, e.key) == .eq) want += 1; + } + var out: std.ArrayListUnmanaged(u64) = .empty; + defer out.deinit(gpa); + try ix.lookup_eq(gpa, &.{.{ .int32 = k }}, &out); + testing.expectEqual(want, out.items.len) catch |err| { + std.debug.print(" key {d}: reachable {d}, iteration holds {d} (entry_count {d})\n", .{ + k, out.items.len, want, ix.count(), + }); + return err; + }; + } +} + +test "an incrementally built index with few distinct keys stays reachable" { + // The crash fuzzer's shape, which the existing differentials miss: a single + // key with only ten distinct values over thousands of documents, so each + // value spans dozens of leaves and most interior separators are duplicates. + // The compound-key differential above uses ~961 combinations over 600 + // inserts, which is almost no duplication at all. + // + // Symptom being hunted: `lookup_eq` returning nothing for a few values while + // every other value is exactly right, and `count()` still reporting every + // entry -- entries that exist and cannot be reached by descent. + const gpa = testing.allocator; + var prng = std.Random.DefaultPrng.init(0xd0_0d_1e); + const rand = prng.random(); + var ix = try simple_index(gpa, test_pager(), &.{"k"}, false, false); + defer ix.deinit(gpa); + + const n_keys: i32 = 10; + const n: usize = 3000; + var keys: std.ArrayListUnmanaged(i32) = .empty; + defer keys.deinit(gpa); + var live: std.ArrayListUnmanaged(bool) = .empty; + defer live.deinit(gpa); + + // Interleave inserts and removals, which is what a real workload does and + // what leaves half-empty leaves and one-child internal nodes behind. + for (0..n) |i| { + const k = rand.intRangeAtMost(i32, 0, n_keys - 1); + const off: u64 = @intCast(i + 1); + const d = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = k } }}); + defer gpa.free(d); + _ = try ix.add_doc(gpa, d, off, false); + // A bare index test has no engine to do this at a write boundary, and + // without it the promise accumulates and grows the shared test file + // without bound. + ix.pager.release_reservation(&ix.hold); + try keys.append(gpa, k); + try live.append(gpa, true); + + // Remove an earlier document every few inserts. + if (i > 20 and i % 3 == 0) { + const victim = rand.intRangeLessThan(usize, 0, keys.items.len); + if (live.items[victim]) { + const vd = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = keys.items[victim] } }}); + defer gpa.free(vd); + ix.remove_doc(gpa, vd, @intCast(victim + 1)); + live.items[victim] = false; + } + } + + // Every value must be reachable by descent, and the counts must match a + // brute-force pass over the model. Checked periodically rather than every + // step: this is 10 descents over a tree of thousands of entries. + if (i % 250 != 0 and i != n - 1) continue; + var k_check: i32 = 0; + while (k_check < n_keys) : (k_check += 1) { + var want: usize = 0; + for (keys.items, live.items) |kk, is_live| { + if (is_live and kk == k_check) want += 1; + } + var out: std.ArrayListUnmanaged(u64) = .empty; + defer out.deinit(gpa); + try ix.lookup_eq(gpa, &.{.{ .int32 = k_check }}, &out); + testing.expectEqual(want, out.items.len) catch |err| { + std.debug.print( + " at insert {d}: key {d} reachable {d}, expected {d} (entry_count {d})\n", + .{ i, k_check, out.items.len, want, ix.count() }, + ); + return err; + }; + } + // And iteration must see exactly as many entries as the tree claims. + var walked: usize = 0; + var wit = ix.iter(); + while (wit.next()) |_| walked += 1; + try testing.expectEqual(ix.count(), walked); + } +} + /// One document's facts in the incremental-mutation differential. const ModelFact = struct { a: i32, b: i32, off: u64 }; diff --git a/src/pager.zig b/src/pager.zig index f2f67b1..0dc1036 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -771,8 +771,20 @@ pub const Pager = struct { // Round up to a growth chunk, and to the system page size, so a // 16 KiB-page host never gets a partial mapping request. + // + // `alignForwardAnyAlign`, not `alignForward`: the chunk is a *proportion* + // of the current size once the file passes 64 MiB, and `mapped_pages / 8` + // is not a power of two. `alignForward` asserts that it is -- so this + // panicked in safe builds and, worse, in ReleaseFast (where the assert is + // compiled out) computed `(addr + align - 1) & ~(align - 1)` with a + // non-power-of-two mask, which can round *down*. A mapping longer than the + // file is the one thing this function exists to prevent: a store into a + // mapped page past end-of-file raises SIGBUS, which no error path catches. + // + // Never noticed because no unit test grew a pager past 64 MiB, which is + // where the chunk stops being `grow_chunk_pages`. const chunk = @max(grow_chunk_pages, self.mapped_pages / 8); - var new_pages = std.mem.alignForward(u32, want_pages, chunk); + var new_pages = std.mem.alignForwardAnyAlign(u32, want_pages, chunk); const sys_pages: u32 = @intCast(map_align / page_size); if (sys_pages > 1) new_pages = std.mem.alignForward(u32, new_pages, sys_pages); diff --git a/tests/fuzz/crash-fuzz.js b/tests/fuzz/crash-fuzz.js index 50cfdca..e673502 100644 --- a/tests/fuzz/crash-fuzz.js +++ b/tests/fuzz/crash-fuzz.js @@ -561,8 +561,41 @@ async function verify(client, base, r, cycleNo) { .map((d) => canon(d)) .sort(); if (got.length !== want.length || got.some((c, i) => c !== want[i])) { + // Decisive diagnostic: the same question asked without the index. `dbDocs` + // came from find({}) on this same reopened server, so filtering it here + // says whether the *documents* are wrong or only the index's answer about + // them. An index that returns fewer documents than a scan is the canonical + // under-approximation -- candidates are generated from the index and the + // full filter is only re-applied to those, so a missing entry is a + // silently missing result. + const scanGot = dbDocs.filter((d) => d.k === v).map((d) => canon(d)).sort(); throw new Fail(`cycle ${cycleNo}: find({k:${v}}) mismatch at prefix ${matched}`, { - cycleNo, v, matched, got, want, + cycleNo, + v, + matched, + verdict: + scanGot.length === want.length && scanGot.every((c, i) => c === want[i]) + ? 'INDEX under-approximates: a scan of the same server returns the expected documents' + : 'DOCUMENTS differ too: the scan does not match the model either', + indexReturned: got.length, + scanReturned: scanGot.length, + modelExpected: want.length, + // Per-key totals, so a single lost leaf is distinguishable from an + // index that came back empty. + perKeyIndexVsScan: await (async () => { + const rows = []; + for (let u = 0; u < 10; u++) { + const idx = (await coll.find({ k: u }).toArray()).length; + const scan = dbDocs.filter((d) => d.k === u).length; + rows.push({ k: u, index: idx, scan }); + } + return rows; + })(), + indexes: dbIndexes.map((i) => i.name), + totalDocs: dbDocs.length, + serverLog: serverLog.slice(-2000), + got, + want, }); } } -- 2.39.5 From f2844e78944e3c4a7a154b32a5b00ad818421590 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Tue, 4 Aug 2026 14:54:27 +0300 Subject: [PATCH 02/37] cursors: server-side cursors for find, aggregate and the listing commands Every reply came back in a single batch with `cursor.id = 0`, `getMore` was a stub answering an empty `nextBatch` on the literal namespace `test.$cmd`, and nothing read `batchSize`. That caps the useful collection size at what fits in one 48 MiB message, which is the opposite of the tens-of-GB target and the reason M0 made whole-index scans stream: the streaming candidate generator existed with no consumer that could suspend. ## What a cursor is allowed to remember A cursor holds no lock between requests, so everything it saves has to survive arbitrary concurrent mutation. Nothing here is a pointer, and the two things that look like stable addresses are not: `reset_tree` re-creates node ids 0 and 1 as different nodes, and `rebuild_collection` moves every document. Three sources, chosen by query shape, each with a different memory contract: - **stream** -- an index-ordered walk resumed from a `(key, off)` anchor plus a `(leaf, slot)` hint. O(key) state, so this is what lets a cursor walk a collection larger than memory. Survives a rebuild, because a repack changes no key. - **offsets** -- the matched slab offsets a narrowed plan already materialized, 8 bytes each. Killed by a rebuild with `QueryPlanKilled`, because those offsets now name unrelated bytes. - **buffered** -- canonical BSON copies, for a sort no index provides and for aggregate/listing output. Depends on nothing, which is what lets a listing hold a cursor over a `$cmd.*` namespace no collection backs. `Collection.layout_epoch` and `Index.epoch` are the invalidation tokens, both checked as error returns rather than assertions since a client reaches them by keeping a cursor open across maintenance. ## Resume `resume_forward`/`resume_reverse` are O(1) while the hint holds and fall back to an exact-order band walk bounded by `resume_walk_max`. Without the hint, `seek` lands at the *start* of an equal-key band, so `sort({status: 1})` over three distinct values across 10M documents would cost ~5e10 comparisons to drain. Two hazards found by draining a collection while writing to it, neither predictable from reading the code: - A deleted anchor must resume at its *band position*, or the rest of an equal-key band is silently dropped -- most of the collection on a low-cardinality index. Hence `band_index`. - On a **unique** index a same-key entry can only be the anchor rewritten, so resuming at it returned updated documents twice. Observed as duplicate `_id`s while updating underneath a drain. ## Protocol Measured against mongod 8.3.7 rather than recalled, which corrected three assumptions: a bare `getMore` does *not* inherit the find's `batchSize` (4998 of 5000 documents come back), a namespace mismatch is `Unauthorized` (13) not `CursorNotFound`, and `CursorInUse` is 143 not 12051. `internalQueryFindCommandBatchSize` is 101, `cursorTimeoutMillis` 600000, `clientCursorMonitorFrequencySecs` 4. The rule everything follows is **never look ahead**: a batch that met its target leaves the cursor open even when the source is in fact exhausted, so four documents at `batchSize: 2` take three commands. `limit` acts as an EOF source, which is what makes `batchSize == limit` close in one round trip. `skip` is consumed once. `batchSize: 0` returns an empty batch with a live cursor. Cursor ids are `(nonce << 20) | slot`, always positive. The nonce is not decoration: without it a recycled slot serves one client another's documents. Cursors are not connection-pinned, since the driver spec allows a `getMore` on any connection to the same server; they end at exhaustion, `killCursors`, or the idle sweep (a second monitor fiber, separate from the TTL one because the cadences differ by an order of magnitude and a TTL failure must not stop reclamation). The registry is fixed-capacity and evicts the least recently used cursor, whose client sees the same 43 an idle timeout gives. Fixed alongside, because cursors are what expose them: - `listCollections` reported `"."` with an *empty* collection part, which makes the driver throw client-side -- so it would have broken the moment its cursor stopped being id 0. Now `.$cmd.listCollections`, as mongod uses. - `count` ignored `skip` and `limit` entirely. - `wire.end_message` now bounds a reply by the 48 MiB we advertise rather than by `maxInt(u32)`; a reply past what we told the client to expect is not a large reply, it is a desynchronized connection. - Two `codeName` strings were wrong: 72 is `InvalidOptions` (MongoDB has no `InvalidArgument`), and 40324 reports as `Location40324`. ## Verification Unit 160/160 in ReleaseFast and ReleaseSafe; `tests/e2e/e2e7.js` adds 86 cursor checks across five phases (batching/lifecycle/errors, streaming across churn, aggregate+listings+count, expiry+capacity, restart) and is self-contained because cursor behaviour is only observable with non-default flags. No regressions: e2e 49, e2e3 16, e2e4 17, e2e2 2, e2e6 72. Spec 168 pass / 124 fail, +5 against the previous scorecard. Mutation-checked, per the repo's second ground rule: `hint_slot + 1`, the `band_index` off-by-one, both epoch bumps, the id nonce, the at-least-one- document rule, and `stream_shape` returning null each turn the intended test red. One claim was withdrawn rather than kept -- swapping `std.mem.order` for `cmp_prefix` in the band walk changes nothing observable, so the comment now says so instead of asserting a check that does not hold. --- AGENTS.md | 21 +- PLAN.md | 44 +- README.md | 40 +- src/commands.zig | 1254 ++++++++++++++++++++++++++++++++++---- src/cursor.zig | 979 +++++++++++++++++++++++++++++ src/db.zig | 127 +++- src/index.zig | 463 +++++++++++++- src/lib.zig | 2 + src/main.zig | 142 +++-- src/server.zig | 23 + src/wire.zig | 41 +- tests/e2e/README.md | 13 + tests/e2e/e2e7.js | 474 ++++++++++++++ tests/spec/run.js | 29 +- tests/spec/scorecard.txt | 13 +- 15 files changed, 3489 insertions(+), 176 deletions(-) create mode 100644 src/cursor.zig create mode 100644 tests/e2e/e2e7.js diff --git a/AGENTS.md b/AGENTS.md index 79b386e..fac2755 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,9 +13,11 @@ indexes, and TTL/unique/sparse/compound index support. **Forward plan**: the project's direction — a full-fledged embedded, tens-of-GB, maximally MongoDB-compatible database — is decided and written -in **[PLAN.md](PLAN.md)**. The current milestone is **M0 (mmap + WAL -storage foundation)**. Before starting any work, read PLAN.md; its decision -record (D1-D9) and ground rules are binding. +in **[PLAN.md](PLAN.md)**. M0 (mmap + WAL storage foundation) has landed; +the current milestone is **M1 (cursors + wire polish)**, whose cursor work is +done — see `src/cursor.zig` and `tests/e2e/e2e7.js`. Before starting any +work, read PLAN.md; its decision record (D1-D9) and ground rules are +binding. ## Read first, in order @@ -71,6 +73,7 @@ node tests/e2e/e2e2.js crash-b # restart, verify all 50 survived node tests/e2e/e2e3.js # secondary indexes node tests/e2e/e2e4.js # TTL indexes (server must run --ttl-sweep-secs 1) node tests/e2e/e2e6.js # self-contained full lifecycle (spawns its own server, incl. kill -9) +node tests/e2e/e2e7.js # self-contained cursors (spawns its own servers; needs no server running) ``` Which suites to run for a given change: @@ -78,6 +81,7 @@ Which suites to run for a given change: - anything touching the write path or log format → the crash pair (e2e2 crash-a/b) and e2e6 - anything touching indexes → e2e3.js and e2e4.js +- anything touching cursors, batching or the reply size → e2e7.js - everything → all of the above `tests/e2e/README.md` has the full matrix, ports, and harness docs @@ -180,7 +184,8 @@ src/server.zig TCP accept loop, per-connection handlers, TTL sweep monitor src/db.zig engine: db → collection → _id → document maps, slab storage src/storage.zig append-only log: blocks, LZ4, XxHash3, replay, compaction src/query.zig filter matcher, regex engine, sort, projection -src/index.zig B+tree indexes: entries, search, query planner +src/index.zig B+tree indexes: entries, search, query planner, scan resume +src/cursor.zig server-side cursor state: registry, batch policy, expiry src/update.zig update operators with dot-path navigation src/main.zig CLI: --port, --bind, --db, --ttl-sweep-secs, --compact-threshold ``` @@ -193,6 +198,8 @@ src/main.zig CLI: --port, --bind, --db, --ttl-sweep-secs, --compact-threshol baseline. 3. Commit scorecard and benchmark results with each milestone (PLAN D9) so progress stays verifiable across sessions. -4. Deferred designs (cursors, aggregation, transactions, change streams, - C API) are deliberately *not* specified yet — grill the design with the - human before implementing (PLAN section 6). +4. Deferred designs (aggregation, transactions, change streams, C API) are + deliberately *not* specified yet — grill the design with the human before + implementing (PLAN section 6). Cursors are no longer among them: the + design was settled and implemented in M1, and `src/cursor.zig`'s module + comment is where it is written down. diff --git a/PLAN.md b/PLAN.md index 561b99e..e0f2270 100644 --- a/PLAN.md +++ b/PLAN.md @@ -635,9 +635,47 @@ it. ## 6. Deferred designs (grill each at its milestone) -- **M1 cursors**: cursor id allocation, idle expiration, batchSize - semantics, getMore against a lagging/compactable engine, cursor state - lifecycle across compaction. +- **M1 cursors** — *settled and implemented.* The design lives in + `src/cursor.zig`'s module comment; the decisions it records, and how each + was reached: + - **Cursor ids** are `(nonce << 20) | slot`, always positive, never 0. The + nonce is not decoration: without it a recycled slot serves one client + another's documents, which is the worst failure this feature could have. + - **batchSize semantics** were *measured against mongod 8.3.7*, not + recalled, and three assumptions were wrong: a bare `getMore` does **not** + inherit the find's batchSize (4998 of 5000 documents come back), a + namespace mismatch is `Unauthorized` (13) rather than `CursorNotFound`, + and `CursorInUse` is 143 rather than the 12051 an earlier note claimed. + `internalQueryFindCommandBatchSize` is 101, `cursorTimeoutMillis` + 600000, `clientCursorMonitorFrequencySecs` 4. + - **Never look ahead**: a batch that met its target leaves the cursor open + even when the source is in fact exhausted, so four documents at + `batchSize: 2` take three commands. The pinned suites assert that count. + - **Idle expiration** is a second monitor fiber, separate from the TTL one: + the cadences differ by an order of magnitude, and a TTL sweep failure + must not stop cursors being reclaimed. The registry is fixed-capacity and + evicts the least recently used cursor, whose client sees the same + `CursorNotFound` an idle timeout gives. + - **Against a lagging/compactable engine**, what survives depends on what + the cursor remembers, so the check is per-source: a repack changes no + key, so a streaming cursor resumes; slab offsets all move, so an offsets + cursor is killed with `QueryPlanKilled`; a snapshot needs no collection at + all. `Collection.layout_epoch` and `Index.epoch` are the tokens. + - **Resume** anchors on `(key, off)` plus a position hint gated on + `Index.epoch`, with an exact-order band walk bounded by + `resume_walk_max`. Two hazards found while implementing: a deleted anchor + must resume at its band position or the rest of an equal-key band is + silently dropped, and on a *unique* index a same-key entry can only be + the anchor rewritten — resuming at it returned updated documents twice, + caught by draining a collection being updated underneath. + + Still open in M1: the doc-level free list, sessions plumbing (`lsid` + accepted), and command-monitoring (`expectEvents`) in the spec runner. + **A prerequisite the free list must honour**, recorded here while it is + still being designed: *an offset that was ever a record start must remain a + record start.* `doc_bytes` reads a `u32` length prefix in place, so an + offset landing mid-record after a re-split is a garbage-length read rather + than a wrong answer — and an offsets cursor holds exactly such offsets. - **M2 aggregation**: stage/expression tiers, which spec-test files are the gate, whether $lookup/$unwind/facet make the first cut. - **M4 transactions**: snapshot isolation over mmap (COW vs undo), read diff --git a/README.md b/README.md index e8eaf07..980fe9e 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ maximally MongoDB-compatible database — its decision record, milestones and gates live in [PLAN.md](PLAN.md). Milestone 0 (mmap + WAL storage foundation) has landed; its measured gate results are in [`tests/e2e/results/m0-gates.txt`](tests/e2e/results/m0-gates.txt). -Milestone 1 (cursors, and the doc-level free list the churn gate showed is -needed) is next. +Milestone 1 is in progress: server-side cursors have landed (see +**Cursors** below); the doc-level free list the churn gate showed is needed +is still open. ## Quick start @@ -145,10 +146,43 @@ whole pass, so the interval is the tuning knob: expiry is never more precise than `--ttl-sweep-secs`, and a very large TTL index wants a longer one. +## Cursors + +`find`, `aggregate`, `listCollections` and `listIndexes` return real cursor +ids, and `getMore`/`killCursors` work. Batching follows MongoDB: a first +batch of 101 documents unless `batchSize` says otherwise, a `getMore` with +no `batchSize` bounded only by the 16 MiB batch cap, `batchSize: 0` as an +empty batch with a live cursor, and `limit` honoured across batches. Every +default here was measured against a real `mongod` rather than assumed. + +A cursor holds no lock between requests, so what it remembers has to survive +arbitrary concurrent writes. Three shapes, picked by the query: + +| query | what the cursor keeps | +| --- | --- | +| a whole-index walk (`find({})`, or a sort an index provides) | the last key and offset it yielded — O(key), whatever the collection size | +| a narrowed index plan | the matching offsets, 8 bytes each | +| a sort no index provides, or aggregate/listing output | a snapshot of the remaining documents | + +The first is what lets a cursor walk a collection larger than memory. It +also survives a compaction, because a repack changes no key; the offsets +form cannot, and says so with `QueryPlanKilled` rather than returning +documents from the wrong place. + +Cursors are not pinned to the connection that created them, so a `getMore` +may arrive on any connection — which is what the driver specification +allows. They are reclaimed when exhausted, when killed, or after +`--cursor-timeout-ms` idle (default 600000, MongoDB's own +`cursorTimeoutMillis`); `--max-open-cursors` bounds the registry and evicts +the least recently used cursor at capacity, whose client then sees the same +`CursorNotFound` an idle timeout gives. + ## Not (yet) implemented - Authentication (SCRAM) — run without credentials -- Real cursors (all results are returned in one batch, cursor id 0) +- Tailable/awaitData cursors, which need capped collections; a tailable + `find` is rejected, exactly as MongoDB rejects one on a non-capped + collection - Transactions, change streams, replicasets - Compression (OP_COMPRESSED) - `collMod`, so an index's `expireAfterSeconds` cannot be changed in diff --git a/src/commands.zig b/src/commands.zig index f2d1b0a..4f92d5b 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -10,6 +10,7 @@ const Collection = db.Collection; const query = @import("query.zig"); const update = @import("update.zig"); const index = @import("index.zig"); +const cursor = @import("cursor.zig"); // Always active, including in the default ReleaseFast build -- see assert.zig. const assert = @import("assert.zig").assert; const assert_msg = @import("assert.zig").assert_msg; @@ -27,17 +28,30 @@ pub const Context = struct { pub const ErrorCode = enum(i32) { command_not_found = 59, bad_value = 2, - invalid_argument = 72, + /// 72 is MongoDB's `InvalidOptions`. There is no `InvalidArgument` in its + /// table at all, so that is the name this used to send. + invalid_options = 72, namespace_not_found = 26, index_not_found = 27, duplicate_key = 11000, namespace_exists = 48, failed_to_parse = 9, internal_error = 1, - invalid_pipeline_operator = 40324, + /// "Unrecognized pipeline stage name". A `Location` code, so mongod names it + /// `Location40324` rather than after any symbol. + location_unrecognized_stage = 40324, index_options_conflict = 85, cannot_create_index = 67, invalid_index_specification_option = 197, + // Cursor codes. Taken from MongoDB's own error_codes.js rather than + // recalled -- CursorInUse in particular is 143, not the 12051 that turns up + // in older notes. + cursor_not_found = 43, + cursor_in_use = 143, + query_plan_killed = 175, + unauthorized = 13, + type_mismatch = 14, + operation_failed = 96, }; /// Which lock (if any) a command needs on the engine. Contract: only @@ -45,13 +59,18 @@ pub const ErrorCode = enum(i32) { /// remove, drop*, get_or_create_collection, compact); `.read` commands may /// only read (`get_collection`, `database_names`, `collection_names`); /// `.none` commands must not touch the engine at all. +/// +/// This describes engine *data* only. The cursor store is a separate resource +/// with its own leaf mutex, and any kind may mutate it: `getMore` is `.read` +/// because it only reads documents, even though it advances cursor state, and +/// `killCursors` is `.none` because the store is all it touches. const CommandKind = enum { none, read, write }; /// Lock shape for one command, acquired by dispatch: the catalog lock mode /// and whether the command's target collection is locked (shared for reads, /// exclusive for writes/DDL). The target collection is the message field /// named after the command (find/count/insert/...), which every -/// collection-targeting command uses. +/// collection-targeting command uses except `getMore` -- see `Command.coll_field`. const LockShape = struct { catalog: enum { none, shared, exclusive } = .none, coll: enum { none, shared, exclusive } = .none, @@ -61,6 +80,9 @@ const Command = struct { name: []const u8, kind: CommandKind, locks: LockShape = .{}, + /// Body field naming the target collection, when it is not the command's + /// own field. Only `getMore` needs it: its own value is an int64 cursor id. + coll_field: ?[]const u8 = null, handler: *const fn (*Context, *wire.Message, *wire.Reply) anyerror!void, }; @@ -80,10 +102,20 @@ const command_table = [_]Command{ .{ .name = "serverStatus", .kind = .none, .handler = cmd_server_status }, .{ .name = "endSessions", .kind = .none, .handler = cmd_end_sessions }, .{ .name = "connectionStatus", .kind = .none, .handler = cmd_connection_status }, - .{ .name = "getMore", .kind = .none, .handler = cmd_get_more }, + // killCursors touches only the cursor store, so it needs no lock -- and its + // own field really is the collection name, unlike getMore's. .{ .name = "killCursors", .kind = .none, .handler = cmd_kill_cursors }, // Read-only: scan the engine without mutating it. .{ .name = "find", .kind = .read, .locks = .{ .catalog = .shared, .coll = .shared }, .handler = cmd_find }, + // getMore is the one command whose collection is not its own field: its + // value is an int64 cursor id, so dispatch reads `collection` instead. + .{ + .name = "getMore", + .kind = .read, + .locks = .{ .catalog = .shared, .coll = .shared }, + .coll_field = "collection", + .handler = cmd_get_more, + }, .{ .name = "count", .kind = .read, .locks = .{ .catalog = .shared, .coll = .shared }, .handler = cmd_count }, .{ .name = "aggregate", .kind = .read, .locks = .{ .catalog = .shared, .coll = .shared }, .handler = cmd_aggregate }, .{ .name = "listDatabases", .kind = .read, .locks = .{ .catalog = .shared }, .handler = cmd_list_databases }, @@ -140,7 +172,8 @@ pub fn dispatch(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { if (cmd.locks.coll != .none) { const db_name = msg.db_name() orelse return bad_value(reply, "command requires a $db"); - const coll_name = str_arg(msg.body.get(name)) orelse + const field = cmd.coll_field orelse name; + const coll_name = str_arg(msg.body.get(field)) orelse return bad_value(reply, "command requires a collection name"); ns = .{ .db = db_name, .coll = coll_name }; } @@ -368,21 +401,30 @@ fn cmd_list_databases(ctx: *Context, _: *wire.Message, reply: *wire.Reply) !void try reply.put_ok(); } +/// The collection part of the namespace a `listCollections` cursor reports. +/// mongod uses this pseudo-collection, and the exact string matters: the previous +/// `"."` had an *empty* collection part, and the driver throws client-side +/// when it tries to build a getMore or killCursors from a namespace like that -- +/// so the moment such a cursor stopped being id 0 it would have broken. +const list_collections_ns: []const u8 = "$cmd.listCollections"; + fn cmd_list_collections(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { const db_name = msg.db_name() orelse return invalid_arg(reply, "listCollections requires $db"); + const batch_size = (try aggregate_batch_size(reply, msg) orelse return).value; var names: std.ArrayListUnmanaged([]const u8) = .empty; defer names.deinit(ctx.gpa); try ctx.engine.collection_names(db_name, &names); - const values = try reply.arena_alloc().alloc(bson.Value, names.items.len); + const arena = reply.arena_alloc(); + const docs = try arena.alloc(*const bson.Document, names.items.len); for (names.items, 0..) |n, i| { - const entry = try reply.arena_alloc().alloc(bson.Pair, 3); - entry[0] = .{ .key = "name", .value = .{ .string = try reply.arena_alloc().dupe(u8, n) } }; + const entry = try arena.alloc(bson.Pair, 3); + entry[0] = .{ .key = "name", .value = .{ .string = try arena.dupe(u8, n) } }; entry[1] = .{ .key = "type", .value = .{ .string = "collection" } }; entry[2] = .{ .key = "options", .value = .{ .doc = &.{} } }; - values[i] = .{ .doc = entry }; + docs[i] = try doc_from_pairs(arena, entry); } - try reply.put("cursor", .{ .doc = try cursor_doc(reply, 0, try format_namespace(reply, db_name, ""), "firstBatch", values) }); + try emit_first_batch(ctx, reply, db_name, list_collections_ns, null, docs, batch_size); try reply.put_ok(); } @@ -506,21 +548,22 @@ fn cmd_list_indexes(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void const coll = ctx.engine.get_collection(db_name, coll_name) orelse return reply.put_error(@intFromEnum(ErrorCode.namespace_not_found), "NamespaceNotFound", "ns not found"); + const batch_size = (try aggregate_batch_size(reply, msg) orelse return).value; // The _id_ index first, then the secondaries. - const n = coll.indexes.items.len + 1; - const values = try reply.arena_alloc().alloc(bson.Value, n); - const id_pairs = try reply.arena_alloc().alloc(bson.Pair, 2); + const arena = reply.arena_alloc(); + const values = try arena.alloc(*const bson.Document, coll.indexes.items.len + 1); + const id_pairs = try arena.alloc(bson.Pair, 2); id_pairs[0] = .{ .key = "v", .value = .{ .int32 = 2 } }; id_pairs[1] = .{ .key = "key", .value = .{ .doc = &.{.{ .key = "_id", .value = .{ .int32 = 1 } }} } }; - values[0] = .{ .doc = try index_pairs_append(reply, id_pairs, "_id_") }; + values[0] = try doc_from_pairs(arena, try index_pairs_append(reply, id_pairs, "_id_")); for (coll.indexes.items, 0..) |ix, i| { - // The pairs live in the reply arena (freed with it); the values - // array below references them. + // The pairs live in the reply arena (freed with it); the docs array + // below references them. var pairs: std.ArrayListUnmanaged(bson.Pair) = .empty; - try ix.spec_pairs(reply.arena_alloc(), &pairs); - values[1 + i] = .{ .doc = pairs.items }; + try ix.spec_pairs(arena, &pairs); + values[1 + i] = try doc_from_pairs(arena, pairs.items); } - try reply.put("cursor", .{ .doc = try cursor_doc(reply, 0, try format_namespace(reply, db_name, coll_name), "firstBatch", values) }); + try emit_first_batch(ctx, reply, db_name, coll_name, null, values, batch_size); try reply.put_ok(); } @@ -656,68 +699,307 @@ fn cmd_insert(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { try reply.put_ok(); } +/// Parse a `batchSize`-shaped option. Null means the error reply is already +/// written. `zero_is_default` distinguishes `find`, where 0 is a real request for +/// an empty batch, from `getMore`, where mongod reads it as "no document target" +/// -- which is also what an absent field means. +fn batch_size_arg( + reply: *wire.Reply, + v: bson.Value, + label: []const u8, + zero_is_default: bool, +) !??u32 { + const n = int_value(v) orelse { + const text = try std.fmt.allocPrint(reply.arena_alloc(), "{s} must be a number", .{label}); + try bad_value(reply, text); + return null; + }; + if (n < 0) { + const text = try std.fmt.allocPrint( + reply.arena_alloc(), + "{s} value must be non-negative", + .{label}, + ); + try bad_value(reply, text); + return null; + } + if (n == 0 and zero_is_default) return @as(?u32, null); + return @as(?u32, std.math.cast(u32, n) orelse std.math.maxInt(u32)); +} + +/// The `find` options that shape the cursor rather than the query. +const CursorOpts = struct { + batch_size: ?u32 = null, + /// One batch and no cursor. Set explicitly, and also by a negative `limit`. + single_batch: bool = false, + no_timeout: bool = false, +}; + +/// Parse the cursor-shaping options, or write an error reply and return null. +fn parse_cursor_opts(reply: *wire.Reply, msg: *wire.Message) !?CursorOpts { + // Every tailable form is refused, and that is parity rather than a gap: + // mongod rejects a tailable cursor on a non-capped collection, and this + // engine has no capped collections at all. Silently ignoring the flag would + // be worse than erroring -- the cursor would report EOF and a driver's tail + // loop would exit, which reads to the application as data loss. + if (bool_arg(msg.body.get("tailable")) orelse false) { + try bad_value(reply, "tailable cursor requested on non capped collection"); + return null; + } + if (bool_arg(msg.body.get("awaitData")) orelse false) { + try bad_value(reply, "Cannot set 'awaitData' without also setting 'tailable'"); + return null; + } + + var opts = CursorOpts{}; + if (msg.body.get("batchSize")) |v| { + opts.batch_size = try batch_size_arg(reply, v, "batchSize", false) orelse return null; + } + opts.single_batch = bool_arg(msg.body.get("singleBatch")) orelse false; + opts.no_timeout = bool_arg(msg.body.get("noCursorTimeout")) orelse false; + return opts; +} + +/// Whether a source may outlive the request that built it. +/// +/// Only a snapshot can fail this: it is the one arm that pins bytes proportional +/// to the result set, and holding a large one for the idle timeout times the +/// number of live cursors is exactly what `cursor_buffer_max` exists to prevent. +/// Over the bound the caller emits everything in one batch instead -- which is +/// what this server did before cursors existed. +fn source_keepable(source: cursor.Source) bool { + const buffered = switch (source) { + .buffered => |b| b, + else => return true, + }; + var total: u64 = 0; + for (buffered.docs) |d| total += d.len; + return total <= cursor_buffer_max; +} + fn cmd_find(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { const db_name = msg.db_name() orelse return invalid_arg(reply, "find requires $db"); const coll_name = str_arg(msg.body.get("find")) orelse return bad_value(reply, "find requires a collection name"); - const filter = doc_arg(msg.body.get("filter")) orelse return bad_value(reply, "filter must be a document"); - const sort_keys = try parse_sort_keys(reply, msg.body.get("sort")); const proj_pairs = doc_arg(msg.body.get("projection")); const skip: u64 = int_arg(msg.body.get("skip")) orelse 0; - // A negative limit means "return this many in a single batch"; we always - // reply with one batch, so only the magnitude matters. - const limit: usize = @abs(int_value(msg.body.get("limit")) orelse 0); + const limit_raw = int_value(msg.body.get("limit")) orelse 0; + const opts = try parse_cursor_opts(reply, msg) orelse return; + // A negative limit is the historical `ntoreturn` shape: this many documents + // in exactly one batch. Drivers translate it before sending, but the wire + // form is still legal. + const single_batch = opts.single_batch or limit_raw < 0; + const limit: u64 = @abs(limit_raw); + const target = cursor.batch_target(opts.batch_size, true); + + // A find on a namespace that does not exist is an empty cursor, not an + // error, and the ordinary path below says exactly that. + const coll = ctx.engine.get_collection(db_name, coll_name); + + var arena = std.heap.ArenaAllocator.init(ctx.gpa); + var arena_owned = false; + defer if (!arena_owned) arena.deinit(); + + // Whether this reply is the whole answer, which is what makes the top-k + // sort shortcut legal -- it leaves everything past k unspecified, and an + // open cursor would later need those documents. + const closes_here = single_batch or (limit > 0 and limit <= (target orelse 0)); + + var feed = try find_feed(ctx, reply, &arena, coll, .{ + .db_name = db_name, + .coll_name = coll_name, + .filter = filter, + .sort_keys = sort_keys, + .skip = skip, + .limit = limit, + .closes_here = closes_here, + }); + + var values: std.ArrayListUnmanaged(bson.Value) = .empty; + const exhausted = try fill_batch(ctx, reply, coll, &feed, filter, proj_pairs, target, &values); + + var cursor_id: i64 = 0; + if (!exhausted and !single_batch and source_keepable(feed.source)) { + cursor_id = keep_find_cursor(ctx, &arena, coll, &feed, .{ + .db = db_name, + .coll = coll_name, + }, filter, proj_pairs, opts); + arena_owned = cursor_id != 0; + } + // No cursor, but documents still to come: one batch holding the rest. + if (cursor_id == 0 and !exhausted and !single_batch) { + _ = try fill_batch(ctx, reply, coll, &feed, filter, proj_pairs, null, &values); + } + + const ns = try format_namespace(reply, db_name, coll_name); + const batch = try cursor_doc(reply, cursor_id, ns, "firstBatch", values.items); + try reply.put("cursor", .{ .doc = batch }); + try reply.put_ok(); +} + +/// Register the remainder of a `find` as a cursor, returning its id or 0. +fn keep_find_cursor( + ctx: *Context, + arena: *std.heap.ArenaAllocator, + coll: ?*Collection, + feed: *const Feed, + ns: cursor.Ns, + filter: []const bson.Pair, + proj_pairs: ?[]const bson.Pair, + opts: CursorOpts, +) i64 { + // Serialized before the arena is handed over: `open_cursor` takes it by + // value, so anything allocated after that call would be invisible to the copy + // the cursor keeps. + const a = arena.allocator(); + const filter_bytes = serialize_pairs(a, filter) catch return 0; + const proj_bytes = if (proj_pairs) |pp| (serialize_pairs(a, pp) catch return 0) else ""; + return open_cursor(ctx, arena.*, .{ + .ns = ns, + .layout_epoch = if (coll) |c| c.layout_epoch else 0, + .filter_bytes = filter_bytes, + .proj_bytes = proj_bytes, + .remaining_limit = feed.remaining_limit, + .batch_size = opts.batch_size, + .no_timeout = opts.no_timeout, + // feed.source, not the source it started as: the first batch advanced it, + // and for a stream that advance *is* the resume point. + .source = feed.source, + }); +} + +/// Everything `find_feed` needs that is not a lock or an arena. +const FindRequest = struct { + db_name: []const u8, + coll_name: []const u8, + filter: []const bson.Pair, + sort_keys: []const query.SortKey, + skip: u64, + limit: u64, + closes_here: bool, +}; + +/// Choose the source for a `find` and open the feed that fills its first batch. +/// +/// A whole-index walk is served without materializing anything: the cursor +/// remembers a key and an offset, so this is what makes `find({})` over a +/// collection larger than memory possible at all. Every other shape collects its +/// matches first, exactly as before cursors existed. +fn find_feed( + ctx: *Context, + reply: *wire.Reply, + arena: *std.heap.ArenaAllocator, + coll: ?*Collection, + req: FindRequest, +) !Feed { + const remaining: ?u64 = if (req.limit == 0) null else req.limit; + + // One plan for the whole request: `stream_shape` reads it to decide whether a + // resumable walk is possible, and the fallback scan reuses it instead of + // planning the same query a second time. + var plan_opt = if (coll) |c| + try index.plan(ctx.gpa, &c.id_index, c.indexes.items, req.filter, req.sort_keys) + else + null; + defer if (plan_opt) |*p| p.deinit(ctx.gpa); + const plan: ?*const index.Plan = if (plan_opt) |*p| p else null; + + if (coll) |c| { + if (stream_shape(c, plan, req.sort_keys)) |st| { + var feed = Feed{ .source = .{ .stream = st }, .remaining_limit = remaining }; + feed.scan = open_scan(c, &feed.source.stream); + // `skip` is consumed once, here, through the same anchor the batch + // uses -- which is what lets a cursor whose entire first batch falls + // inside the skipped prefix still resume from the right place. + try stream_skip(ctx, c, &feed, req.filter, req.skip); + return feed; + } + } + + // Documents needed to fill the page, counting the skipped prefix; 0 means + // unbounded. Only an index-ordered scan may stop there. + const page_end: usize = if (req.limit == 0) 0 else blk: { + const skip_usize = std.math.cast(usize, req.skip) orelse break :blk 0; + break :blk skip_usize +| req.limit; + }; + // Lives only until `find_source` copies what it needs into `arena`. var matched: std.ArrayListUnmanaged(u64) = .empty; defer matched.deinit(ctx.gpa); - // Documents needed to fill the page, counting the skipped prefix; 0 - // means unbounded. - const page_end: usize = if (limit == 0) 0 else blk: { - const skip_usize = std.math.cast(usize, skip) orelse break :blk 0; - break :blk skip_usize +| limit; - }; - // An index whose order already is the requested one lets the scan stop - // at the page boundary and skip sorting entirely. Otherwise a sort has - // to see every match before it can tell which ones the page contains. var index_sorted = false; - _ = try scan_sorted(ctx, db_name, coll_name, filter, page_end, &matched, sort_keys, &index_sorted); + _ = try scan_planned( + ctx, + req.db_name, + req.coll_name, + req.filter, + page_end, + &matched, + req.sort_keys, + &index_sorted, + plan, + ); + if (coll == null) { + assert_msg(matched.items.len == 0, "find matched in a collection that does not exist"); + } + const source = try find_source(reply, arena, coll, matched.items, .{ + .sort_keys = req.sort_keys, + .index_sorted = index_sorted, + .skip = req.skip, + .page_end = page_end, + .closes_here = req.closes_here, + }); + return .{ .source = source, .remaining_limit = remaining }; +} - // Sorting and emitting need the documents as trees; materialize the - // matched page into the reply arena (the slab itself is never copied). - // A find on a namespace that does not exist is an empty cursor, not an - // error and not a reply missing `ok`: the scan above matched nothing, so - // falling through to the emit at the end of this function says exactly - // that without a second exit path to keep in step with it. - const coll = ctx.engine.get_collection(db_name, coll_name); - // The scan above ran against this same collection with the catalog lock - // held, so a missing collection means nothing matched. Asserted rather than - // left implicit: if that ever stops holding, the loop below silently emits - // an empty page for a query that did match, which is the hardest kind of - // wrong answer to notice. - if (coll == null) assert_msg(matched.items.len == 0, "find matched documents in a collection that does not exist"); - // Lives in the reply arena; freed with it. +/// How `find_source` should turn a match list into a source. +const SourceShape = struct { + sort_keys: []const query.SortKey, + index_sorted: bool, + skip: u64, + page_end: usize, + closes_here: bool, +}; + +/// Build the source `find` pulls from, with `skip` already consumed. +/// +/// The query shape decides which source is possible. An index-ordered scan +/// yields offsets -- 8 bytes apiece, and valid until the collection is rebuilt. +/// A sort no index provides had to materialize and order every match, so there +/// is no ordered offset list to point at and the remainder is snapshotted. +fn find_source( + reply: *wire.Reply, + arena: *std.heap.ArenaAllocator, + coll: ?*Collection, + matched: []const u64, + shape: SourceShape, +) !cursor.Source { + const c = coll orelse return .{ .offsets = .{ .items = &.{} } }; + if (shape.sort_keys.len == 0 or shape.index_sorted) { + // Already in the order the client asked for, so skip is a slice. + const rest = if (shape.skip < matched.len) matched[shape.skip..] else &.{}; + return .{ .offsets = .{ .items = try arena.allocator().dupe(u64, rest) } }; + } + + // Ordering needs the values: materialize into the reply arena and sort. + const reply_arena = reply.arena_alloc(); var tree_docs: std.ArrayListUnmanaged(*const bson.Document) = .empty; - const arena = reply.arena_alloc(); - if (coll) |c| { - for (matched.items) |off| { - try tree_docs.append(arena, try doc_tree(arena, c, off)); - } + for (matched) |off| try tree_docs.append(reply_arena, try doc_tree(reply_arena, c, off)); + if (shape.closes_here and shape.page_end > 0 and shape.page_end *| 4 <= tree_docs.items.len) { + try query.sort_docs_top_k(reply_arena, tree_docs.items, shape.sort_keys, shape.page_end); + } else { + try query.sort_docs(reply_arena, tree_docs.items, shape.sort_keys); } - if (sort_keys.len > 0 and !index_sorted) { - // Selecting the page is much cheaper than ordering everything when - // the page is a small fraction of the matches. Above that fraction - // the heap's bookkeeping stops paying for itself. - if (page_end > 0 and page_end *| 4 <= tree_docs.items.len) { - try query.sort_docs_top_k(arena, tree_docs.items, sort_keys, page_end); - } else { - try query.sort_docs(arena, tree_docs.items, sort_keys); - } - } - const rest = if (skip < tree_docs.items.len) tree_docs.items[skip..] else &.{}; - const page = if (limit > 0 and limit < rest.len) rest[0..limit] else rest; - try emit_docs_tree(reply, db_name, coll_name, proj_pairs, page); - try reply.put_ok(); + const rest = if (shape.skip < tree_docs.items.len) tree_docs.items[shape.skip..] else &.{}; + return buffered_source(arena.allocator(), rest); +} + +/// Serialize `pairs` into `arena`. A cursor cannot keep the parsed form: those +/// pairs point into the per-request message arena. +fn serialize_pairs(arena: std.mem.Allocator, pairs: []const bson.Pair) ![]const u8 { + var buf: std.ArrayListUnmanaged(u8) = .empty; + try bson.write_doc(pairs, arena, &buf); + return buf.items; } /// Collect the documents in `db_name.coll_name` matching `filter`, stopping @@ -754,6 +1036,27 @@ fn scan_sorted( out: ?*std.ArrayListUnmanaged(u64), sort: []const query.SortKey, sorted: ?*bool, +) !usize { + return scan_planned(ctx, db_name, coll_name, filter, limit, out, sort, sorted, null); +} + +/// `scan_sorted` with the plan supplied. `find` decides between a resumable walk +/// and a materializing scan by looking at the plan, so without this it would plan +/// once to choose and `scan_sorted` would plan the identical query again -- +/// `index.plan` flattens the filter's clauses and evaluates every index, both +/// allocating, on the hot read path. +/// +/// `prebuilt` is borrowed: the caller keeps ownership, including its `deinit`. +fn scan_planned( + ctx: *Context, + db_name: []const u8, + coll_name: []const u8, + filter: []const bson.Pair, + limit: usize, + out: ?*std.ArrayListUnmanaged(u64), + sort: []const query.SortKey, + sorted: ?*bool, + prebuilt: ?*const index.Plan, ) !usize { if (sorted) |flag| flag.* = false; // Stopping early is only meaningful when the candidates come out in the @@ -781,10 +1084,16 @@ fn scan_sorted( var offs: std.ArrayListUnmanaged(u64) = .empty; defer offs.deinit(ctx.gpa); var cands: index.Candidates = undefined; - var plan_opt = try index.plan(ctx.gpa, &coll.id_index, coll.indexes.items, filter, sort); - defer if (plan_opt) |*p| p.deinit(ctx.gpa); + // Only plan here when the caller did not; theirs is borrowed, so only ours + // is freed. + var owned_plan = if (prebuilt != null) + null + else + try index.plan(ctx.gpa, &coll.id_index, coll.indexes.items, filter, sort); + defer if (owned_plan) |*p| p.deinit(ctx.gpa); + const plan_opt: ?*const index.Plan = prebuilt orelse if (owned_plan) |*p| p else null; - if (plan_opt) |*plan| { + if (plan_opt) |plan| { if (sorted) |flag| flag.* = plan.provides_sort; if (plan.provides_sort) lim = limit; if (plan.full_scan()) { @@ -818,26 +1127,384 @@ fn scan_sorted( /// the pair/value skeleton is allocated. The arena owns the skeleton, so /// the result is never deinit'd — the reply arena frees it with the reply. fn doc_tree(arena: std.mem.Allocator, coll: *const Collection, off: u64) !*const bson.Document { + return doc_tree_bytes(arena, coll.doc_bytes(off)); +} + +/// The same borrowed spine over bytes that are already in hand -- the slab for a +/// live scan, or a cursor's own snapshot for a buffered one. +fn doc_tree_bytes(arena: std.mem.Allocator, bytes: []const u8) !*const bson.Document { const doc = try arena.create(bson.Document); - doc.* = bson.Document{ .arena = undefined, .pairs = try bson.spine(arena, coll.doc_bytes(off)) }; + doc.* = bson.Document{ .arena = undefined, .pairs = try bson.spine(arena, bytes) }; return doc; } -fn emit_docs_tree( - reply: *wire.Reply, - db_name: []const u8, - coll_name: []const u8, - proj_pairs: ?[]const bson.Pair, - docs: []const *const bson.Document, -) !void { - const values = try reply.arena_alloc().alloc(bson.Value, docs.len); - for (docs, 0..) |d, i| { - values[i] = try project_doc(reply, d, proj_pairs); +// --------------------------------------------------------------------------- +// Cursors: filling a batch, and the sources a batch pulls from +// --------------------------------------------------------------------------- + +/// Largest snapshot a cursor will copy into its own arena. Mirrors MongoDB's +/// 32 MiB in-memory sort limit, and only the shapes that had to materialize +/// anyway can reach it. +/// +/// Over the bound the cursor is *declined* and the whole result goes out in one +/// batch -- exactly what this server did before cursors existed. Declining is +/// the right direction to fail: the alternative, holding the snapshot anyway, +/// pins it for the idle timeout times the number of live cursors. +const cursor_buffer_max: u64 = 32 * 1024 * 1024; + +/// The part of a cursor that a batch consumes. Kept separate from +/// `cursor.Cursor` so `find` can fill its first batch and only then decide +/// whether a cursor needs to exist at all. +const Feed = struct { + source: cursor.Source, + /// Documents still owed across every remaining batch; null is unbounded. + remaining_limit: ?u64, + /// A `.stream` source's live position in the tree, valid for **this request + /// only**. It is deliberately not part of `cursor.Source`: a tree position + /// must never outlive the collection lock that made it safe to hold, which is + /// the whole reason the stored form is a value-typed anchor instead. + scan: ?Scan = null, + + fn limit_exhausted(self: *const Feed) bool { + const rem = self.remaining_limit orelse return false; + return rem == 0; + } +}; + +/// A walk over one index, in one direction, for the duration of one request. +const Scan = struct { + walk: union(enum) { + fwd: index.Index.Iter, + rev: index.Index.RevIter, + }, + /// The entry `peek_bytes` has produced but the batch has not yet accepted. + /// Held because a batch that turns out to be full must not consume it. + pending: ?index.Index.Positioned = null, + + fn next(self: *Scan) ?index.Index.Positioned { + return switch (self.walk) { + .fwd => |*it| it.positioned(), + .rev => |*it| it.positioned(), + }; + } +}; + +/// The next document the source will yield, *without* consuming it -- a batch +/// that turns out to be full must not swallow a document it cannot carry. +/// +/// Candidates whose document no longer matches are consumed and skipped here. +/// That re-check is the index invariant (`src/index.zig`) applied per batch, and +/// it is also what makes a saved offset safe once documents start being +/// recycled: a reused offset either fails the filter or resolves to a document +/// that genuinely matches it. +fn peek_bytes( + ctx: *Context, + coll: ?*Collection, + filter: []const bson.Pair, + feed: *Feed, +) !?[]const u8 { + switch (feed.source) { + // A snapshot of documents that already matched, and that nothing can + // mutate underneath us -- so it is not re-filtered. + .buffered => |*b| { + if (b.next >= b.docs.len) return null; + return b.docs[b.next]; + }, + .offsets => |*o| { + const c = coll orelse return null; + while (o.next < o.items.len) { + const bytes = c.doc_bytes(o.items[o.next]); + if (try query.matches_bytes(ctx.gpa, filter, bytes)) return bytes; + o.next += 1; + } + return null; + }, + .stream => { + const c = coll orelse return null; + const sc = &(feed.scan orelse return null); + if (sc.pending) |p| return c.doc_bytes(p.off); + while (sc.next()) |p| { + const bytes = c.doc_bytes(p.off); + if (!try query.matches_bytes(ctx.gpa, filter, bytes)) { + // A candidate the filter rejects is still progress: the + // anchor must move past it, or a resume would walk it again. + feed.source.stream.advance(p.key, p.off, p.leaf, p.slot); + continue; + } + sc.pending = p; + return bytes; + } + return null; + }, } - try reply.put("cursor", .{ .doc = try cursor_doc(reply, 0, try format_namespace(reply, db_name, coll_name), "firstBatch", values) }); } -/// Project a stored doc (or deep-copy it) into the reply arena. +fn consume_one(feed: *Feed) void { + switch (feed.source) { + .buffered => |*b| b.next += 1, + .offsets => |*o| o.next += 1, + .stream => |*st| { + const sc = &(feed.scan orelse return); + const p = sc.pending orelse return; + st.advance(p.key, p.off, p.leaf, p.slot); + sc.pending = null; + }, + } +} + +/// Resolve the index a `.stream` cursor names. Empty is the implicit `_id_`, +/// which is deliberately kept out of `Collection.indexes` and so is not findable +/// by name. +fn stream_index(coll: *Collection, name: []const u8) ?*const index.Index { + if (name.len == 0) return &coll.id_index; + return coll.find_index(name); +} + +/// Open this request's walk over the index a `.stream` cursor is following. +/// +/// Null means the cursor can never produce another document: the index it was +/// following is gone. The caller turns that into `QueryPlanKilled` rather than +/// an empty batch, because an empty batch would claim the result set ended. +fn open_scan(coll: *Collection, st: *cursor.Stream) ?Scan { + const ix = stream_index(coll, st.index_name()) orelse return null; + // A hint is a node id plus a slot, and `reset_tree` recycles ids 0 and 1 as + // different nodes -- so the hint is only meaningful at the epoch it was + // taken. The *anchor* is unaffected: it is key bytes and an offset, both + // values, so an epoch change costs a band walk rather than correctness. + const trusted = st.index_epoch == ix.epoch; + st.index_epoch = ix.epoch; + + if (!st.started()) { + // `batchSize: 0` leaves a cursor with no anchor yet, so a first getMore + // starts the walk from the beginning rather than resuming. + return .{ .walk = if (st.backward) + .{ .rev = ix.iter_reverse() } + else + .{ .fwd = ix.iter() } }; + } + if (st.backward) { + const r = ix.resume_reverse( + st.anchor_key(), + st.anchor_off, + st.hint_leaf, + st.hint_slot, + trusted, + ); + if (r.capped) return null; + return .{ .walk = .{ .rev = r.it } }; + } + const r = ix.resume_forward( + st.anchor_key(), + st.anchor_off, + st.band_index, + st.hint_leaf, + st.hint_slot, + trusted, + ); + if (r.capped) return null; + return .{ .walk = .{ .fwd = r.it } }; +} + +/// Whether this query can be served by walking one index end to end, and if so +/// which index and in which direction. +/// +/// This is the shape that lets a cursor outlive its result set: it holds a key +/// and an offset instead of a list, so `find({})` over a collection larger than +/// memory costs O(key) of cursor state rather than 8 bytes per matching +/// document. Everything else keeps the materialized sources. +/// +/// Two shapes qualify. A query the planner declines outright (`{}` with no sort) +/// walks the `_id_` index forward -- every document has an `_id` and the index is +/// not sparse, so a full walk cannot miss one. And a plan whose `full_scan()` +/// holds is by construction a whole-index walk in the requested order; note that +/// `full_scan()` implies `provides_sort`, since a plan with no run, no range and +/// no sort direction is declined before it is built. +/// +/// A narrowed plan is excluded on purpose: it dedupes `$in` and multikey +/// candidates across the whole set, which a stream cannot do without remembering +/// what it has already emitted. +fn stream_shape( + coll: *Collection, + plan_opt: ?*const index.Plan, + sort: []const query.SortKey, +) ?cursor.Stream { + var st = cursor.Stream{}; + if (plan_opt) |plan| { + if (!plan.full_scan()) return null; + st.backward = plan.backward; + if (plan.index != &coll.id_index) { + if (plan.index.name.len > cursor.index_name_max) return null; + @memcpy(st.index_name_buf[0..plan.index.name.len], plan.index.name); + st.index_name_len = @intCast(plan.index.name.len); + } + st.index_epoch = plan.index.epoch; + return st; + } + // No usable predicate and no ordering to honour: every document in _id + // order, which is what the old materializing fallback did too. + if (sort.len != 0) return null; + st.index_epoch = coll.id_index.epoch; + return st; +} + +/// Consume `skip` matching documents without emitting them, advancing the anchor +/// as it goes so a resume does not walk them again. +fn stream_skip( + ctx: *Context, + coll: *Collection, + feed: *Feed, + filter: []const bson.Pair, + skip: u64, +) !void { + var left = skip; + while (left > 0) { + _ = try peek_bytes(ctx, coll, filter, feed) orelse return; + consume_one(feed); + left -= 1; + } +} + +/// Fill one batch from `feed`, projecting into the reply arena. Returns whether +/// the source is **exhausted**. +/// +/// Exhaustion is observed, never predicted: a batch that reached its document +/// target returns false even when the source happens to have nothing left, so +/// the cursor stays open and the client gets one more (possibly empty) batch. +/// Predicting it here would close the cursor a round trip early and break the +/// command-count assertions in the pinned spec suites. +fn fill_batch( + ctx: *Context, + reply: *wire.Reply, + coll: ?*Collection, + feed: *Feed, + filter: []const bson.Pair, + proj_pairs: ?[]const bson.Pair, + target: ?u32, + out: *std.ArrayListUnmanaged(bson.Value), +) !bool { + var builder = cursor.BatchBuilder.init(target); + const arena = reply.arena_alloc(); + while (true) { + // Checked before the target so the batch that takes the last document + // the limit allows is itself the one that closes the cursor. This is + // what lets `batchSize == limit` finish in a single round trip. + if (feed.limit_exhausted()) return true; + if (builder.full()) return false; + const bytes = try peek_bytes(ctx, coll, filter, feed) orelse return true; + // The stored length is exact with no projection and an upper bound with + // one, since `query.project` only ever drops fields. + if (builder.offer(bytes.len) == .batch_full) return false; + const doc = try doc_tree_bytes(arena, bytes); + try out.append(arena, try project_doc(reply, doc, proj_pairs)); + consume_one(feed); + if (feed.remaining_limit) |rem| feed.remaining_limit = rem - 1; + } +} + +/// Register `feed`'s remainder as a cursor and return its id, or 0 when no +/// cursor is needed or one could not be had. +/// +/// Every "could not" path degrades to `id: 0` rather than to an error: the +/// client then has the batch it was given and no cursor, which is precisely how +/// this server behaved before cursors existed. +fn open_cursor(ctx: *Context, arena: std.heap.ArenaAllocator, spec: cursor.OpenSpec) i64 { + var owned = arena; + return ctx.engine.cursors.open(ctx.io, now_ms(ctx), owned, spec) catch |err| switch (err) { + // A namespace too long for the fixed buffers, a store whose every slot + // is pinned, or an allocation failure. None is worth failing a query + // whose documents are already in the reply. + error.NameTooLong, error.TooManyCursors, error.OutOfMemory, error.Canceled => { + owned.deinit(); + return 0; + }, + }; +} + +fn now_ms(ctx: *Context) i64 { + return std.Io.Timestamp.now(ctx.io, .real).toMilliseconds(); +} + +/// Copy `docs` into `arena` as canonical BSON, for a result with no stable +/// backing store to point at. +fn buffered_source( + arena: std.mem.Allocator, + docs: []const *const bson.Document, +) !cursor.Source { + const out = try arena.alloc([]const u8, docs.len); + for (docs, 0..) |d, i| out[i] = try serialize_pairs(arena, d.pairs); + return .{ .buffered = .{ .docs = out } }; +} + +/// Wrap pairs already living in an arena as a borrowed document, so generated +/// results can go through the same batch path as stored ones. +fn doc_from_pairs(arena: std.mem.Allocator, pairs: []const bson.Pair) !*const bson.Document { + const doc = try arena.create(bson.Document); + doc.* = bson.Document{ .arena = undefined, .pairs = pairs }; + return doc; +} + +/// Emit a first batch from documents already materialized in the reply arena, +/// registering a cursor for whatever does not fit in it. +/// +/// The snapshot source is the only one available here: these documents are either +/// generated (`$group`, a listing) or the output of a pipeline that has already +/// materialized its window, so there is no stable structure to point back into. +/// That also makes the resulting cursor independent of its collection, which is +/// what lets a listing hold a cursor over a `$cmd.*` namespace. +/// +/// `ns_coll` is the collection part of the reported namespace and the one a +/// `getMore` must name -- for a listing that is `$cmd.listCollections`, not the +/// empty string that used to be reported. An empty collection part makes the +/// driver throw client-side before it even sends the getMore. +fn emit_first_batch( + ctx: *Context, + reply: *wire.Reply, + ns_db: []const u8, + ns_coll: []const u8, + proj_pairs: ?[]const bson.Pair, + docs: []const *const bson.Document, + batch_size: ?u32, +) !void { + var arena = std.heap.ArenaAllocator.init(ctx.gpa); + var arena_owned = false; + defer if (!arena_owned) arena.deinit(); + + const source = try buffered_source(arena.allocator(), docs); + var feed = Feed{ .source = source, .remaining_limit = null }; + var values: std.ArrayListUnmanaged(bson.Value) = .empty; + const target = cursor.batch_target(batch_size, true); + const exhausted = try fill_batch(ctx, reply, null, &feed, &.{}, proj_pairs, target, &values); + + var cursor_id: i64 = 0; + if (!exhausted and source_keepable(feed.source)) { + cursor_id = open_cursor(ctx, arena, .{ + .ns = .{ .db = ns_db, .coll = ns_coll }, + // Nothing about this cursor depends on the collection's layout. + .layout_epoch = 0, + .batch_size = batch_size, + .source = feed.source, + }); + arena_owned = cursor_id != 0; + } + if (cursor_id == 0 and !exhausted) { + _ = try fill_batch(ctx, reply, null, &feed, &.{}, proj_pairs, null, &values); + } + + const ns = try format_namespace(reply, ns_db, ns_coll); + const batch = try cursor_doc(reply, cursor_id, ns, "firstBatch", values.items); + try reply.put("cursor", .{ .doc = batch }); +} + +/// `batchSize` out of an `aggregate`'s `cursor` option. A bare `cursor: {}` means +/// the default; a missing `cursor` is accepted as the same thing, which is looser +/// than mongod (it requires the field) but cannot surprise any driver. +fn aggregate_batch_size(reply: *wire.Reply, msg: *wire.Message) !?struct { value: ?u32 } { + const spec = doc_arg(msg.body.get("cursor")) orelse return .{ .value = null }; + const v = bson.get_pair(spec, "batchSize") orelse return .{ .value = null }; + const parsed = try batch_size_arg(reply, v, "cursor.batchSize", false) orelse return null; + return .{ .value = parsed }; +} + fn project_doc( reply: *wire.Reply, doc: *const bson.Document, @@ -1065,8 +1732,22 @@ fn cmd_count(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { const db_name = msg.db_name() orelse return invalid_arg(reply, "count requires $db"); const coll_name = str_arg(msg.body.get("count")) orelse return bad_value(reply, "count requires a collection name"); const q = doc_arg(msg.body.get("query")) orelse &.{}; + // `count` takes skip and limit like `find` does, and ignoring them was a + // silent wrong answer for `countDocuments(f, {limit})`. + const skip: u64 = int_arg(msg.body.get("skip")) orelse 0; + const limit: u64 = @abs(int_value(msg.body.get("limit")) orelse 0); - const n = try scan_matching(ctx, db_name, coll_name, q, 0, null); + // Counting only needs to know whether the matches reach skip + limit, so the + // scan may stop there. Unlike `find` this needs no index to be an early stop: + // the *count* of a window does not depend on which documents fall in it. + const ceiling: usize = if (limit == 0) 0 else blk: { + const s = std.math.cast(usize, skip) orelse break :blk 0; + const l = std.math.cast(usize, limit) orelse break :blk 0; + break :blk s +| l; + }; + const matched = try scan_matching(ctx, db_name, coll_name, q, ceiling, null); + const after_skip = matched -| (std.math.cast(usize, skip) orelse matched); + const n = if (limit == 0) after_skip else @min(after_skip, limit); try reply.put("n", .{ .int32 = @intCast(n) }); try reply.put_ok(); } @@ -1075,6 +1756,7 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { const db_name = msg.db_name() orelse return invalid_arg(reply, "aggregate requires $db"); const coll_name = str_arg(msg.body.get("aggregate")) orelse return bad_value(reply, "aggregate requires a collection name"); + const batch_size = (try aggregate_batch_size(reply, msg) orelse return).value; const pipeline_value = msg.body.get("pipeline") orelse return bad_value(reply, "aggregate requires pipeline"); var stages = switch (pipeline_value) { .array => |a| a, @@ -1106,13 +1788,12 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { .{ .double = sum }, }; } - const doc = try arena.create(bson.Document); - doc.* = bson.Document{ .arena = undefined, .pairs = pairs }; + const doc = try doc_from_pairs(arena, pairs); const one = try arena.alloc(*const bson.Document, 1); one[0] = doc; docs = one; } - try emit_docs_tree(reply, db_name, coll_name, null, docs); + try emit_first_batch(ctx, reply, db_name, coll_name, null, docs, batch_size); return reply.put_ok(); } @@ -1132,7 +1813,7 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { // collection. MongoDB answers an aggregate over a missing collection with // an empty cursor. const coll = ctx.engine.get_collection(db_name, coll_name) orelse { - try emit_docs_tree(reply, db_name, coll_name, null, &.{}); + try emit_first_batch(ctx, reply, db_name, coll_name, null, &.{}, batch_size); return reply.put_ok(); }; // A leading $match is pushed down into an indexed candidate scan; the @@ -1249,7 +1930,12 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { }; } else { const msg_text = try std.fmt.allocPrint(reply.arena_alloc(), "Unrecognized pipeline stage name: '{s}'", .{stage_name}); - return reply.put_error(@intFromEnum(ErrorCode.invalid_pipeline_operator), "InvalidPipelineOperator", msg_text); + // 40324 is right for "unrecognized stage" but its name is not + // `InvalidPipelineOperator` (that is 168). mongod reports numeric + // Location codes under a `Location` name -- verified by asking a + // real mongod for an unknown stage. + const code = @intFromEnum(ErrorCode.location_unrecognized_stage); + return reply.put_error(code, "Location40324", msg_text); } } @@ -1257,17 +1943,19 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { const len = if (in_trees) trees.items[start..end].len else offs.items[start..end].len; const c = try reply.arena_alloc().alloc(bson.Pair, 1); c[0] = .{ .key = try reply.arena_alloc().dupe(u8, name), .value = .{ .int32 = @intCast(len) } }; - const values = try reply.arena_alloc().alloc(bson.Value, 1); - values[0] = .{ .doc = c }; - try reply.put("cursor", .{ .doc = try cursor_doc(reply, 0, try format_namespace(reply, db_name, coll_name), "firstBatch", values) }); + const one = try reply.arena_alloc().alloc(*const bson.Document, 1); + one[0] = try doc_from_pairs(reply.arena_alloc(), c); + try emit_first_batch(ctx, reply, db_name, coll_name, null, one, batch_size); } else { const arena = reply.arena_alloc(); if (in_trees) { - try emit_docs_tree(reply, db_name, coll_name, proj_pairs, trees.items[start..end]); + const window = trees.items[start..end]; + try emit_first_batch(ctx, reply, db_name, coll_name, proj_pairs, window, batch_size); } else { var page: std.ArrayListUnmanaged(*const bson.Document) = .empty; for (offs.items[start..end]) |off| try page.append(arena, try doc_tree(arena, coll, off)); - try emit_docs_tree(reply, db_name, coll_name, proj_pairs, page.items); + const window = page.items; + try emit_first_batch(ctx, reply, db_name, coll_name, proj_pairs, window, batch_size); } } try reply.put_ok(); @@ -1471,16 +2159,227 @@ fn query_path_value_bytes( return cur; } -fn cmd_get_more(_: *Context, _: *wire.Message, reply: *wire.Reply) !void { - // Cursors are never left open, so getMore always yields an empty batch. - try reply.put("cursor", .{ .doc = try cursor_doc(reply, 0, "test.$cmd", "nextBatch", &.{}) }); +/// A cursor id from the wire. Accepts int32 as well as int64, so a hand-written +/// `runCommand` is not rejected on a technicality; the driver always sends a +/// long. +fn cursor_id_arg(v: ?bson.Value) ?i64 { + return switch (v orelse return null) { + .int64 => |i| i, + .int32 => |i| i, + else => null, + }; +} + +fn cursor_not_found(reply: *wire.Reply, id: i64) !void { + const text = try std.fmt.allocPrint(reply.arena_alloc(), "cursor id {d} not found", .{id}); + return reply.put_error(@intFromEnum(ErrorCode.cursor_not_found), "CursorNotFound", text); +} + +fn query_plan_killed(reply: *wire.Reply, why: []const u8) !void { + const text = try std.fmt.allocPrint( + reply.arena_alloc(), + "query plan killed :: caused by :: {s}", + .{why}, + ); + const code = @intFromEnum(ErrorCode.query_plan_killed); + return reply.put_error(code, "QueryPlanKilled", text); +} + +/// Whether this cursor can still be answered from `coll`, or why not. +/// +/// What a rebuild since the cursor was created costs depends entirely on what the +/// cursor remembers. `.offsets` holds slab offsets and a rebuild moved every +/// document, so those offsets now name unrelated bytes. `.stream` holds key +/// bytes, and a repack changes no key -- only the anchor's offset and position +/// hint go stale, and both are checked before they are believed, so the walk +/// resumes at the right key and the offsets it yields come fresh out of the tree. +/// `.buffered` holds copies of the documents and needs no collection at all, +/// which is what lets a listing hold a cursor over a `$cmd.*` namespace. +fn cursor_still_valid(c: *cursor.Cursor, coll: ?*Collection) ?[]const u8 { + const live = coll orelse { + // A snapshot needs no collection at all, which is what lets a listing + // hold a cursor over a `$cmd.*` namespace nothing backs. + return if (c.source == .buffered) null else "collection dropped"; + }; + if (live.layout_epoch == c.layout_epoch) return null; + if (c.source == .offsets) return "collection rebuilt"; + // Survived the rebuild: adopt the new layout so the next getMore does not + // re-examine it. + c.layout_epoch = live.layout_epoch; + return null; +} + +/// The stored filter and projection, reparsed for this request. They were +/// serialized into the cursor's own arena because the parsed forms pointed into +/// the request that created them; `spine` parses the structure and borrows the +/// leaf bytes, so this costs no copy of the filter's strings. +fn cursor_query(reply: *wire.Reply, c: *const cursor.Cursor) !struct { + filter: []const bson.Pair, + proj: ?[]const bson.Pair, +} { + const arena = reply.arena_alloc(); + return .{ + .filter = if (c.filter_bytes.len == 0) &.{} else try bson.spine(arena, c.filter_bytes), + .proj = if (c.proj_bytes.len == 0) null else try bson.spine(arena, c.proj_bytes), + }; +} + +/// The shape of a `getMore` request, or null once the error reply is written. +fn parse_get_more(reply: *wire.Reply, msg: *wire.Message) !?struct { id: i64, batch_size: ?u32 } { + const id = cursor_id_arg(msg.body.get("getMore")) orelse { + const text = "BSON field 'getMore.getMore' is the wrong type"; + try reply.put_error(@intFromEnum(ErrorCode.type_mismatch), "TypeMismatch", text); + return null; + }; + var batch_size: ?u32 = null; + if (msg.body.get("batchSize")) |v| { + batch_size = try batch_size_arg(reply, v, "batchSize", true) orelse return null; + } + return .{ .id = id, .batch_size = batch_size }; +} + +/// Claim the cursor for this request, or write the error reply and return null. +/// +/// The namespace check inside `pin` is load-bearing rather than cosmetic: +/// dispatch locks the collection the *message* names, so a getMore quoting one +/// cursor's id and another collection's name would otherwise iterate the first +/// collection's index while holding the second collection's lock. +fn pin_cursor(ctx: *Context, reply: *wire.Reply, id: i64, ns: cursor.Ns) !?*cursor.Cursor { + return ctx.engine.cursors.pin(ctx.io, id, ns, now_ms(ctx)) catch |err| switch (err) { + error.CursorNotFound => { + try cursor_not_found(reply, id); + return null; + }, + error.CursorNamespaceMismatch => { + var found: cursor.NsBuf = .{}; + const arena = reply.arena_alloc(); + const text = if (ctx.engine.cursors.ns_of(ctx.io, id, &found)) + try std.fmt.allocPrint( + arena, + "Requested getMore on namespace '{s}.{s}', but cursor belongs to" ++ + " a different namespace {s}.{s}", + .{ ns.db, ns.coll, found.ns().db, found.ns().coll }, + ) + else + try std.fmt.allocPrint( + arena, + "Requested getMore on namespace '{s}.{s}', but cursor belongs to" ++ + " a different namespace", + .{ ns.db, ns.coll }, + ); + try reply.put_error(@intFromEnum(ErrorCode.unauthorized), "Unauthorized", text); + return null; + }, + error.CursorInUse => { + const text = try std.fmt.allocPrint( + reply.arena_alloc(), + "cursor id {d} is already in use", + .{id}, + ); + try reply.put_error(@intFromEnum(ErrorCode.cursor_in_use), "CursorInUse", text); + return null; + }, + error.Canceled => return err, + }; +} + +fn cmd_get_more(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { + const db_name = msg.db_name() orelse return invalid_arg(reply, "getMore requires $db"); + const coll_name = str_arg(msg.body.get("collection")) orelse + return bad_value(reply, "Field 'collection' must be of type string"); + if (coll_name.len == 0) return bad_value(reply, "Collection names cannot be empty"); + const req = try parse_get_more(reply, msg) orelse return; + const id = req.id; + + const ns = cursor.Ns{ .db = db_name, .coll = coll_name }; + const c = try pin_cursor(ctx, reply, id, ns) orelse return; + // Set to true by every path that must not leave the cursor behind, including + // the error paths below: a cursor whose collection is gone can never answer + // again, so keeping it would only occupy a slot until the idle sweep. + var done = false; + defer ctx.engine.cursors.release(ctx.io, c, now_ms(ctx), done); + + const coll = ctx.engine.get_collection(db_name, coll_name); + if (cursor_still_valid(c, coll)) |why| { + done = true; + return query_plan_killed(reply, why); + } + const q = try cursor_query(reply, c); + + var feed = Feed{ .source = c.source, .remaining_limit = c.remaining_limit }; + // Persist on every exit, not just the success path. For a stream the advance + // *is* the resume anchor, so an early return that skipped this would report a + // batch and then hand the same documents out again on the next getMore. + // Registered after `release`'s defer, so it runs before it. + defer { + c.source = feed.source; + c.remaining_limit = feed.remaining_limit; + } + if (feed.source == .stream) { + // Only a stream needs the collection here; `open_scan` walks its index. + assert_msg(coll != null, "a streaming cursor reached getMore with no collection"); + // Reopening the walk is where a resume actually happens. Null means the + // index the cursor was following is gone, or its anchor could not be + // located within the walk bound -- either way there is no honest way to + // continue, and an empty batch would falsely claim the result ended. + feed.scan = open_scan(coll.?, &feed.source.stream); + if (feed.scan == null) { + done = true; + return query_plan_killed(reply, "the index this cursor was following is gone"); + } + } + var values: std.ArrayListUnmanaged(bson.Value) = .empty; + // Deliberately not `batch_size orelse c.batch_size`: mongod does not carry + // the find's batchSize into a bare getMore. Measured -- find with + // batchSize 2 then a bare getMore returns 4998 of 5000 documents. + const target = cursor.batch_target(req.batch_size, false); + done = try fill_batch(ctx, reply, coll, &feed, q.filter, q.proj, target, &values); + + const ns_str = try format_namespace(reply, db_name, coll_name); + const live_id: i64 = if (done) 0 else id; + const batch = try cursor_doc(reply, live_id, ns_str, "nextBatch", values.items); + try reply.put("cursor", .{ .doc = batch }); try reply.put_ok(); } -fn cmd_kill_cursors(_: *Context, _: *wire.Message, reply: *wire.Reply) !void { - try reply.put("cursorsKilled", .{ .array = &.{} }); - try reply.put("cursorsNotFound", .{ .array = &.{} }); +fn cmd_kill_cursors(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { + const db_name = msg.db_name() orelse return invalid_arg(reply, "killCursors requires $db"); + // Unlike getMore, killCursors' own field really is the collection name. + const coll_name = str_arg(msg.body.get("killCursors")) orelse + return bad_value(reply, "killCursors requires a collection name"); + const cursors_arg = msg.body.get("cursors") orelse + return bad_value(reply, "killCursors requires a cursors array"); + const ids = switch (cursors_arg) { + .array => |a| a, + else => return bad_value(reply, "cursors must be an array"), + }; + + const arena = reply.arena_alloc(); + var killed: std.ArrayListUnmanaged(bson.Value) = .empty; + var not_found: std.ArrayListUnmanaged(bson.Value) = .empty; + const ns = cursor.Ns{ .db = db_name, .coll = coll_name }; + for (ids) |v| { + const id = cursor_id_arg(v) orelse { + try not_found.append(arena, v); + continue; + }; + // A namespace mismatch reports not-found rather than erroring: + // killCursors is best-effort by design, and the driver ignores its reply + // entirely. + switch (ctx.engine.cursors.kill(ctx.io, id, ns)) { + .killed => try killed.append(arena, .{ .int64 = id }), + .not_found => try not_found.append(arena, .{ .int64 = id }), + } + } + + try reply.put("cursorsKilled", .{ .array = killed.items }); + try reply.put("cursorsNotFound", .{ .array = not_found.items }); + // Empty by construction: a cursor pinned by an in-flight getMore is marked + // and reported killed, since the client's intent is satisfied and the + // request frees it on release. `cursorsUnknown` exists for shape -- every + // outcome here is classified. try reply.put("cursorsAlive", .{ .array = &.{} }); + try reply.put("cursorsUnknown", .{ .array = &.{} }); try reply.put_ok(); } @@ -1618,7 +2517,7 @@ fn batch_arg( } fn invalid_arg(reply: *wire.Reply, msg: []const u8) !void { - return reply.put_error(@intFromEnum(ErrorCode.invalid_argument), "InvalidArgument", msg); + return reply.put_error(@intFromEnum(ErrorCode.invalid_options), "InvalidOptions", msg); } fn bad_value(reply: *wire.Reply, msg: []const u8) !void { @@ -2075,8 +2974,8 @@ fn dispatch_find_ids( defer reply.deinit(); try dispatch(&ctx, &msg, &reply); try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double); - const cursor = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; - const batch = switch (bson.get_pair(cursor.doc, "firstBatch") orelse return error.TestUnexpectedResult) { + const cur = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; + const batch = switch (bson.get_pair(cur.doc, "firstBatch") orelse return error.TestUnexpectedResult) { .array => |a| a, else => return error.TestUnexpectedResult, }; @@ -2132,8 +3031,8 @@ test "createIndexes, listIndexes, dropIndexes, and idempotent re-create" { defer reply.deinit(); try dispatch(&ctx, &msg, &reply); try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double); - const cursor = bson.get_pair(reply.pairs.items, "cursor").?; - const batch = bson.get_pair(cursor.doc, "firstBatch").?.array; + const cur = bson.get_pair(reply.pairs.items, "cursor").?; + const batch = bson.get_pair(cur.doc, "firstBatch").?.array; try testing.expectEqual(@as(usize, 2), batch.len); try testing.expectEqualStrings("_id_", bson.get_pair(batch[0].doc, "name").?.string); try testing.expectEqualStrings("email_1", bson.get_pair(batch[1].doc, "name").?.string); @@ -2210,8 +3109,8 @@ test "TTL index round-trips through createIndexes/listIndexes; bad specs give 67 var reply = wire.Reply.init(testing.allocator); defer reply.deinit(); try dispatch(&ctx, &msg, &reply); - const cursor = bson.get_pair(reply.pairs.items, "cursor").?; - const batch = bson.get_pair(cursor.doc, "firstBatch").?.array; + const cur = bson.get_pair(reply.pairs.items, "cursor").?; + const batch = bson.get_pair(cur.doc, "firstBatch").?.array; try testing.expectEqual(@as(usize, 2), batch.len); try testing.expectEqual(@as(i32, 60), bson.get_pair(batch[1].doc, "expireAfterSeconds").?.int32); // The _id_ entry never carries one. @@ -2380,8 +3279,8 @@ test "aggregate $sort without a preceding $group sorts and frees correctly" { try dispatch(&ctx, &msg, &reply); try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double); - const cursor = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; - const batch = switch (bson.get_pair(cursor.doc, "firstBatch") orelse return error.TestUnexpectedResult) { + const cur = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; + const batch = switch (bson.get_pair(cur.doc, "firstBatch") orelse return error.TestUnexpectedResult) { .array => |a| a, else => return error.TestUnexpectedResult, }; @@ -2489,8 +3388,8 @@ test "a sorted full scan over a multikey index returns each document once" { defer reply.deinit(); try dispatch(&ctx, &msg, &reply); try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double); - const cursor = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; - const batch = switch (bson.get_pair(cursor.doc, "firstBatch") orelse return error.TestUnexpectedResult) { + const cur = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; + const batch = switch (bson.get_pair(cur.doc, "firstBatch") orelse return error.TestUnexpectedResult) { .array => |arr| arr, else => return error.TestUnexpectedResult, }; @@ -2801,3 +3700,136 @@ test "indexed queries are equivalent to scans over a mixed corpus" { for (scanned.items, indexed.items) |a, b| try testing.expectEqualSlices(u8, a, b); } } + +/// Insert `n` documents `{_id: 1..n, a: i % 5, pad}` in one batch. +/// +/// Goes through `dispatch_insert` rather than dispatching itself, because that +/// helper checks `ok`, `writeErrors` *and* `n` -- and its comment records what +/// omitting those cost last time: a corpus silently lost a document and every +/// test over it still passed. A cursor test whose premise is "60 documents" must +/// not be able to become "0 documents" quietly. +const seed_pad = "0123456789012345678901234567890123456789"; + +fn seed_docs(tdb: *TestDb, io: std.Io, coll: []const u8, n: i32) !void { + const arena = testing.allocator; + const docs = try arena.alloc(bson.Value, @intCast(n)); + defer { + for (docs) |d| arena.free(d.doc); + arena.free(docs); + } + for (docs, 0..) |*d, i| { + const pairs = try arena.alloc(bson.Pair, 3); + pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i + 1) } }; + pairs[1] = .{ .key = "a", .value = .{ .int32 = @intCast(@mod(i + 1, 5)) } }; + pairs[2] = .{ .key = "pad", .value = .{ .string = seed_pad } }; + d.* = .{ .doc = pairs }; + } + try dispatch_insert(tdb, io, coll, docs); +} + +/// Run `find` and return (cursor id, first-batch length). +fn dispatch_find( + ctx: *Context, + coll: []const u8, + filter: []const bson.Pair, + batch_size: i32, +) !struct { id: i64, n: usize } { + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg("find", .{ .string = coll }, &.{ + .{ .key = "filter", .value = .{ .doc = filter } }, + .{ .key = "batchSize", .value = .{ .int32 = batch_size } }, + }); + defer msg.deinit(); + try dispatch(ctx, &msg, &reply); + const cur = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult; + const id = bson.get_pair(cur.doc, "id").?.int64; + const batch = bson.get_pair(cur.doc, "firstBatch").?.array; + return .{ .id = id, .n = batch.len }; +} + +/// Run `getMore` and return the error code, or 0 on success. +fn dispatch_get_more(ctx: *Context, coll: []const u8, id: i64) !i32 { + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg("getMore", .{ .int64 = id }, &.{ + .{ .key = "collection", .value = .{ .string = coll } }, + .{ .key = "batchSize", .value = .{ .int32 = 5 } }, + }); + defer msg.deinit(); + try dispatch(ctx, &msg, &reply); + if (bson.get_pair(reply.pairs.items, "code")) |c| return c.int32; + return 0; +} + +test "a rebuild kills an offsets cursor and spares a streaming one" { + // This is also the test that proves the streaming source is *selected*: a + // whole-index walk and an indexed-predicate scan are given different sources, + // and a rebuild is exactly what tells them apart. If `find({})` quietly fell + // back to materializing offsets, both would die here. + // + // A rebuild is triggered directly rather than through churn, because whether + // churn crosses the compaction threshold is not something a test should have + // to guess at. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + try seed_docs(&tdb, io, "c", 60); + + // A whole-index walk: no predicate, no sort. + const walk = try dispatch_find(&ctx, "c", &.{}, 5); + try testing.expectEqual(@as(usize, 5), walk.n); + try testing.expect(walk.id != 0); + // A narrowed plan over the _id_ index, which materializes its candidates. + const narrowed = try dispatch_find(&ctx, "c", &.{ + .{ .key = "_id", .value = .{ .doc = &.{.{ .key = "$gte", .value = .{ .int32 = 1 } }} } }, + }, 5); + try testing.expect(narrowed.id != 0); + + // Both still work before the rebuild, so the difference below is the rebuild. + try testing.expectEqual(@as(i32, 0), try dispatch_get_more(&ctx, "c", walk.id)); + try testing.expectEqual(@as(i32, 0), try dispatch_get_more(&ctx, "c", narrowed.id)); + + try ctx.engine.compact(); + + // The stream remembers key bytes, which a repack does not change. + try testing.expectEqual(@as(i32, 0), try dispatch_get_more(&ctx, "c", walk.id)); + // The offsets name bytes that have moved, so continuing would be a wrong + // answer; QueryPlanKilled says so instead. + try testing.expectEqual( + @intFromEnum(ErrorCode.query_plan_killed), + try dispatch_get_more(&ctx, "c", narrowed.id), + ); + // And a second getMore on the killed cursor no longer knows it at all. + try testing.expectEqual( + @intFromEnum(ErrorCode.cursor_not_found), + try dispatch_get_more(&ctx, "c", narrowed.id), + ); + + // The surviving stream drains to exactly the 60 documents, once each. + // Three batches of 5 are already out: the first batch, the getMore before the + // rebuild, and the one just after it. + var seen: u32 = 5 + 5 + 5; + var id = walk.id; + var rounds: u32 = 0; + while (rounds < 100) : (rounds += 1) { + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg("getMore", .{ .int64 = id }, &.{ + .{ .key = "collection", .value = .{ .string = "c" } }, + .{ .key = "batchSize", .value = .{ .int32 = 7 } }, + }); + defer msg.deinit(); + try dispatch(&ctx, &msg, &reply); + const cur = bson.get_pair(reply.pairs.items, "cursor").?; + seen += @intCast(bson.get_pair(cur.doc, "nextBatch").?.array.len); + id = bson.get_pair(cur.doc, "id").?.int64; + if (id == 0) break; + } + try testing.expectEqual(@as(i64, 0), id); + try testing.expectEqual(@as(u32, 60), seen); +} diff --git a/src/cursor.zig b/src/cursor.zig new file mode 100644 index 0000000..039319f --- /dev/null +++ b/src/cursor.zig @@ -0,0 +1,979 @@ +//! Server-side cursor state: what a `find`/`aggregate` leaves behind so a later +//! `getMore` can carry on, and the fixed-capacity registry that holds it. +//! +//! This module is deliberately *pure*: it owns state and policy, never +//! execution. It does not import `db.zig` or `commands.zig`, so `db.Engine` can +//! own a `Store` with no import cycle, and the batch policy below is testable +//! with no engine, no socket and no allocator. Filling a batch stays in +//! `commands.zig`, which already owns orchestration the way `index.zig` owns +//! planning. +//! +//! **The one rule the whole batching protocol follows: never look ahead.** A +//! batch ends either because it reached its target -- and the cursor stays open +//! -- or because the source reported EOF, and then the cursor closes with +//! `id: 0` in that same reply. A batch that reached its target leaves the cursor +//! open *even when the source happens to be exhausted*. So four documents at +//! `batchSize: 2` need a third command answering `nextBatch: []` with `id: 0`; +//! that empty terminal batch is correct, not a bug, and the pinned spec suites +//! assert exactly that command count. +//! +//! ## What a cursor is allowed to remember +//! +//! A cursor holds no lock between requests, so everything it saves must survive +//! arbitrary concurrent mutation. Nothing here is a pointer, and the two things +//! that look like stable addresses are not: +//! +//! - A tree position `(leaf, slot)` is invalidated by `Index.reset_tree`, +//! which clears the node table so ids 0 and 1 become a live but *unrelated* +//! root and leaf. Guarded by `Stream.index_epoch`. +//! - A slab offset is invalidated by `rebuild_collection`, which moves every +//! document. Guarded by `layout_epoch`. +//! +//! Both are checked as error returns rather than assertions, because a client +//! can reach either one by keeping a cursor open across maintenance. + +const std = @import("std"); +const bson = @import("bson.zig"); +const index = @import("index.zig"); +// Always active, including in the default ReleaseFast build -- see assert.zig. +const assert = @import("assert.zig").assert; +const assert_msg = @import("assert.zig").assert_msg; + +// --------------------------------------------------------------------------- +// Bounds +// --------------------------------------------------------------------------- + +/// Longest index key a `.stream` cursor will anchor on. Tied to the B+tree's +/// own "this record is normal" threshold rather than picked: a key past it has +/// already spilled to the overflow slab, so the tree itself considers it +/// exceptional. Also what makes the anchor a fixed inline array instead of an +/// allocation. +/// +/// Unbounded, this is a memory denial of service and not a subtle one: +/// `bson.encode_key` escapes NULs, so a 16 MB string doubles, and a compound +/// index may carry 32 of them. +pub const anchor_key_max: usize = 1024; + +comptime { + // The bound is only defensible if it really is the tree's spill threshold. + // If the page size or the spill fraction ever changes, this fails to + // compile rather than silently becoming an arbitrary number. + std.debug.assert(anchor_key_max == index.inline_limit); +} + +pub const ns_db_max: usize = 64; +pub const ns_coll_max: usize = 192; +pub const index_name_max: usize = 128; + +/// Documents in a first batch when the client named no `batchSize`. MongoDB's +/// own default (`internalQueryFindCommandBatchSize`). +pub const default_first_batch: u32 = 101; + +/// Cap on a batch's document payload: `maxBsonObjectSize`, which is also what +/// leaves room for the reply envelope inside the 48 MiB message limit. +pub const batch_bytes_max: u64 = 16 * 1024 * 1024; + +/// Idle milliseconds before the sweep reaps a cursor. MongoDB's +/// `cursorTimeoutMillis`. +pub const default_idle_timeout_ms: i64 = 10 * 60 * 1000; + +/// Slots in the registry unless configured otherwise. +pub const default_capacity: u32 = 4096; + +/// Low bits of a cursor id that address its slot; the rest is the nonce. +const slot_bits: u6 = 20; +const slot_mask: u64 = (@as(u64, 1) << slot_bits) - 1; +/// Nonce width, leaving the sign bit clear so every id is a positive i64. +const nonce_mask: u64 = (@as(u64, 1) << (63 - slot_bits)) - 1; + +pub const max_capacity: u32 = @intCast(slot_mask); + +// --------------------------------------------------------------------------- +// Cursor state +// --------------------------------------------------------------------------- + +/// A namespace, by value. A cursor cannot hold a `*Collection`: `drop` frees +/// it, and the pointer would dangle exactly the way the M0 notes on +/// heap-allocating collections describe. +pub const Ns = struct { + db: []const u8, + coll: []const u8, +}; + +/// Where the remaining documents come from. +pub const Source = union(enum) { + /// An index-ordered scan, resumed from a value-typed anchor. O(key) + /// memory, so this is the shape that lets a cursor walk a collection far + /// larger than memory -- the reason M0 made whole-index scans stream. + stream: Stream, + /// Matched slab offsets, 8 bytes each, which `scan_sorted` has already + /// materialized for a narrowed plan. + /// + /// Safe against an offset the coming document free list has recycled, + /// because every batch re-applies the full filter -- the index invariant. + /// A recycled offset is therefore either rejected or resolves to a + /// document that genuinely matches. It needs one guarantee from the free + /// list, recorded in PLAN: an offset that was ever a record start must + /// stay a record start, since `doc_bytes` reads a length prefix in place. + offsets: struct { items: []u64, next: u32 = 0 }, + /// Canonical BSON bytes owned by the cursor's arena, for results with no + /// stable backing store to point at: a sort no index provides, and + /// aggregate/listCollections/listIndexes output. + buffered: struct { docs: []const []const u8, next: u32 = 0 }, +}; + +/// A resumable index scan. Every field is a value; nothing here is a pointer +/// into the tree, the slab or the request that created it. +pub const Stream = struct { + /// Empty means the implicit `_id_` index. Re-resolved by name on every + /// `getMore`, so a `dropIndexes` cannot leave a dangling `*Index`. + index_name_buf: [index_name_max]u8 = undefined, + index_name_len: u8 = 0, + /// Bumped by `reset_tree`/`replace_root_with_leaf`; if it moved, the hint + /// below addresses a different tree and must not be trusted. + index_epoch: u64 = 0, + backward: bool = false, + anchor_buf: [anchor_key_max]u8 = undefined, + anchor_len: u16 = 0, + anchor_off: u64 = 0, + /// Entries sharing the anchor's key that this cursor has already yielded. + /// Without it, an anchor whose document was deleted between batches would + /// resume past the entire equal-key band -- on a three-value index that is + /// millions of documents silently missing. + band_index: u64 = 0, + /// Last known position of the anchor. A hint, never trusted without + /// re-reading the entry there: it turns resume from a walk down the + /// equal-key band into O(1), which is what keeps a low-cardinality + /// `sort({status: 1})` from costing O(band) per batch. + hint_leaf: u32 = 0, + hint_slot: u32 = 0, + + pub fn index_name(self: *const Stream) []const u8 { + return self.index_name_buf[0..self.index_name_len]; + } + + pub fn anchor_key(self: *const Stream) []const u8 { + return self.anchor_buf[0..self.anchor_len]; + } + + /// Whether anything has been yielded yet. Derived rather than stored: an + /// encoded index key always begins with `bson.encode_key`'s rank byte, so it + /// is never empty, and a separate `started` flag would be a second field that + /// has to agree with this one. + /// + /// It can legitimately be false on a live cursor: `batchSize: 0` returns an + /// empty first batch without consuming anything, and such a cursor starts at + /// `iter()`/`iter_reverse()` rather than resuming. + pub fn started(self: *const Stream) bool { + return self.anchor_len > 0; + } + + /// Record the entry just yielded as the point to resume after. + /// + /// Asserts the anchor advances in scan order. This is the single check most + /// likely to catch a resume bug: going backwards duplicates documents, + /// standing still makes `getMore` loop forever, and both are far easier to + /// see here than in a client's result set. Equal keys are legal (a + /// duplicate band), which is exactly why `band_index` also has to move. + pub fn advance(self: *Stream, key: []const u8, off: u64, leaf: u32, slot: u32) void { + assert(key.len <= anchor_key_max); + // What makes `started()` derivable, so pin it here rather than trust it. + assert_msg(key.len > 0, "an encoded index key is never empty"); + if (self.started()) { + const order = std.mem.order(u8, key, self.anchor_key()); + if (self.backward) { + assert_msg(order != .gt, "a reverse cursor's anchor moved forward"); + } else { + assert_msg(order != .lt, "a forward cursor's anchor moved backward"); + } + if (order == .eq) { + assert_msg( + off != self.anchor_off or self.band_index > 0, + "a cursor re-anchored on the entry it just yielded", + ); + // Still inside the anchor's band, so the position within it has + // to move or a resume could not tell the two entries apart. + self.band_index += 1; + } else { + self.band_index = 0; + } + } + @memcpy(self.anchor_buf[0..key.len], key); + self.anchor_len = @intCast(key.len); + self.anchor_off = off; + self.hint_leaf = leaf; + self.hint_slot = slot; + } +}; + +pub const Cursor = struct { + /// Positive and never 0: `id: 0` is "no cursor" on the wire. + id: i64, + ns_db_buf: [ns_db_max]u8 = undefined, + ns_db_len: u8 = 0, + ns_coll_buf: [ns_coll_max]u8 = undefined, + ns_coll_len: u8 = 0, + /// Bumped when a rebuild moves documents, so a saved offset or anchor + /// offset is stale. Also the drop detector. + layout_epoch: u64 = 0, + /// Serialized so they outlive the request that parsed them: a parsed + /// `[]bson.Pair` points into the per-request message arena, and the reply + /// arena is reset on every request. + filter_bytes: []const u8 = &.{}, + proj_bytes: []const u8 = &.{}, + /// Documents still owed across all remaining batches; null is unbounded. + /// Reaching 0 is an EOF *source*, which is what closes the cursor in the + /// very batch that exhausts the limit rather than one round trip later. + /// Optional rather than "0 means unbounded" precisely because 0 has to keep + /// its literal meaning here. + remaining_limit: ?u64 = null, + /// The client's `batchSize`, reused when a `getMore` names none. + batch_size: ?u32 = null, + /// Exempt from the idle sweep. Still killable by `killCursors` and by + /// eviction -- a fixed-capacity registry cannot promise "never expires". + no_timeout: bool = false, + /// A request is using this cursor right now. Concurrent use is rejected + /// rather than queued: queueing lets one client turn a single cursor into a + /// connection-count denial of service. + pinned: bool = false, + /// `killCursors` arrived while pinned; the in-flight request frees it. + kill_requested: bool = false, + last_use_ms: i64 = 0, + arena: std.heap.ArenaAllocator, + source: Source, + + pub fn ns(self: *const Cursor) Ns { + return .{ + .db = self.ns_db_buf[0..self.ns_db_len], + .coll = self.ns_coll_buf[0..self.ns_coll_len], + }; + } + + pub fn ns_matches(self: *const Cursor, other: Ns) bool { + const own = self.ns(); + return std.mem.eql(u8, own.db, other.db) and std.mem.eql(u8, own.coll, other.coll); + } +}; + +/// Everything a caller must decide before a cursor can exist. Grouped so +/// `open` cannot be called with an argument silently in the wrong position. +pub const OpenSpec = struct { + ns: Ns, + layout_epoch: u64, + filter_bytes: []const u8 = &.{}, + proj_bytes: []const u8 = &.{}, + remaining_limit: ?u64 = null, + batch_size: ?u32 = null, + no_timeout: bool = false, + source: Source, +}; + +pub const OpenError = error{ + /// The namespace does not fit the fixed buffers. Callers degrade to a + /// single batch rather than failing the query. + NameTooLong, + /// Every slot is pinned by an in-flight request. + TooManyCursors, + OutOfMemory, + /// Taking the store mutex was cancelled (shutdown). + Canceled, +}; + +pub const PinError = error{ + CursorNotFound, + /// The id exists but belongs to another namespace. Distinct from + /// `CursorNotFound` because mongod answers this with `Unauthorized` (13), + /// not 43, and leaves the cursor alive -- the request is wrong, not the + /// cursor. + CursorNamespaceMismatch, + CursorInUse, + Canceled, +}; + +/// Owned storage for a namespace copied out of the store, so an error message +/// can name a cursor's namespace without holding the store's mutex or a pointer +/// into its slots. +pub const NsBuf = struct { + db_buf: [ns_db_max]u8 = undefined, + db_len: u8 = 0, + coll_buf: [ns_coll_max]u8 = undefined, + coll_len: u8 = 0, + + pub fn ns(self: *const NsBuf) Ns { + return .{ .db = self.db_buf[0..self.db_len], .coll = self.coll_buf[0..self.coll_len] }; + } + + fn set(self: *NsBuf, from: Ns) void { + @memcpy(self.db_buf[0..from.db.len], from.db); + self.db_len = @intCast(from.db.len); + @memcpy(self.coll_buf[0..from.coll.len], from.coll); + self.coll_len = @intCast(from.coll.len); + } +}; + +pub const KillOutcome = enum { killed, not_found }; + +// --------------------------------------------------------------------------- +// The registry +// --------------------------------------------------------------------------- + +pub const Store = struct { + /// Guards every field below. A **leaf** lock: no other lock -- catalog, + /// collection, log -- is ever acquired while it is held, so it cannot + /// participate in a cycle. In particular a `getMore` copies what it needs + /// out, releases this, and only then iterates under the collection lock; + /// otherwise the reaper would block behind a full scan. + mutex: std.Io.Mutex = .init, + /// Boxed, not inline. A `Cursor` inlines its anchor and namespace buffers and + /// so is ~1.5 KiB; at the default capacity an inline table would be 6.2 MiB + /// allocated and zeroed at *every* `Engine.open` -- paid by every embedded + /// user and by all ~47 engine opens in the unit suite, to hold zero cursors. + /// A pointer table is 32 KiB and the cursor itself is allocated when one + /// actually exists, which is also when its arena is created anyway. + slots: []?*Cursor, + /// Mixed into every id so ids are not guessable across processes, and so a + /// reused slot rejects the previous id exactly. Without this a stale + /// `getMore` can address a recycled slot and read another client's cursor. + nonce: u64, + live: u32 = 0, + idle_timeout_ms: i64 = default_idle_timeout_ms, + gpa: std.mem.Allocator, + + pub fn init( + gpa: std.mem.Allocator, + io: std.Io, + capacity: u32, + idle_timeout_ms: i64, + ) !Store { + assert(capacity > 0 and capacity <= max_capacity); + var seed: [8]u8 = undefined; + io.random(&seed); + const slots = try gpa.alloc(?*Cursor, capacity); + @memset(slots, null); + return .{ + .slots = slots, + // A zero nonce would make the first slot's id equal to its index, + // and slot 0's id would be 0 -- which means "no cursor". + .nonce = std.mem.readInt(u64, &seed, .little) | 1, + .idle_timeout_ms = idle_timeout_ms, + .gpa = gpa, + }; + } + + pub fn deinit(self: *Store) void { + for (self.slots) |maybe| { + if (maybe) |c| destroy_cursor(self.gpa, c); + } + self.gpa.free(self.slots); + self.slots = &.{}; + } + + /// Free a cursor: its arena first, then the box the slot pointed at. + fn destroy_cursor(gpa: std.mem.Allocator, c: *Cursor) void { + c.arena.deinit(); + gpa.destroy(c); + } + + fn slot_of(id: i64) usize { + return @intCast(@as(u64, @bitCast(id)) & slot_mask); + } + + /// Build the id for `slot` at the store's current nonce, then advance the + /// nonce so the next cursor in this slot gets a different id. + fn mint(self: *Store, slot: usize) i64 { + const n = self.nonce & nonce_mask; + self.nonce +%= 1; + const raw = (n << slot_bits) | @as(u64, @intCast(slot)); + const id: i64 = @intCast(raw & ~(@as(u64, 1) << 63)); + // Both properties are load-bearing on the wire and in lookup. + assert_msg(id > 0, "a cursor id must be a positive int64"); + assert_msg(slot_of(id) == slot, "a cursor id must address its own slot"); + return id; + } + + /// Register a cursor and return its id, or null when the caller should + /// answer in a single batch instead (`NameTooLong` is not worth failing a + /// query over -- the degradation is exactly today's behaviour). + /// + /// Takes ownership of `spec.source` and of the arena backing it. + pub fn open( + self: *Store, + io: std.Io, + now_ms: i64, + arena: std.heap.ArenaAllocator, + spec: OpenSpec, + ) OpenError!i64 { + if (spec.ns.db.len > ns_db_max or spec.ns.coll.len > ns_coll_max) { + return error.NameTooLong; + } + try self.mutex.lock(io); + defer self.mutex.unlock(io); + + const slot = self.free_slot(now_ms) orelse return error.TooManyCursors; + assert(self.slots[slot] == null); + + const c = try self.gpa.create(Cursor); + errdefer self.gpa.destroy(c); + c.* = .{ + .id = self.mint(slot), + .layout_epoch = spec.layout_epoch, + .filter_bytes = spec.filter_bytes, + .proj_bytes = spec.proj_bytes, + .remaining_limit = spec.remaining_limit, + .batch_size = spec.batch_size, + .no_timeout = spec.no_timeout, + .last_use_ms = now_ms, + .arena = arena, + .source = spec.source, + }; + @memcpy(c.ns_db_buf[0..spec.ns.db.len], spec.ns.db); + c.ns_db_len = @intCast(spec.ns.db.len); + @memcpy(c.ns_coll_buf[0..spec.ns.coll.len], spec.ns.coll); + c.ns_coll_len = @intCast(spec.ns.coll.len); + + self.slots[slot] = c; + self.live += 1; + return c.id; + } + + /// An empty slot: a genuinely free one, else the least-recently-used + /// unpinned cursor. Evicting is legal and cheap to reason about, because + /// the victim's client gets `CursorNotFound` on its next `getMore` -- the + /// same answer an idle timeout gives, which every driver already handles. + /// Caller holds the mutex. + fn free_slot(self: *Store, now_ms: i64) ?usize { + var lru: ?usize = null; + var lru_ms: i64 = std.math.maxInt(i64); + for (self.slots, 0..) |maybe, i| { + const c = maybe orelse return i; + // Reap on the way past, so a store that has gone quiet does not + // wait for the sweep tick to reclaim what already expired. + if (self.expired(c, now_ms)) { + self.destroy(i); + return i; + } + if (c.pinned) continue; + if (c.last_use_ms < lru_ms) { + lru_ms = c.last_use_ms; + lru = i; + } + } + if (lru) |i| { + self.destroy(i); + return i; + } + return null; + } + + /// Caller holds the mutex. + fn expired(self: *const Store, c: *const Cursor, now_ms: i64) bool { + if (c.pinned or c.no_timeout or self.idle_timeout_ms <= 0) return false; + return now_ms -| c.last_use_ms >= self.idle_timeout_ms; + } + + /// Caller holds the mutex. + fn destroy(self: *Store, slot: usize) void { + const c = self.slots[slot] orelse return; + assert_msg(!c.pinned, "a pinned cursor must not be destroyed under its user"); + destroy_cursor(self.gpa, c); + self.slots[slot] = null; + self.live -= 1; + } + + /// Claim a cursor for one request. The returned pointer is stable only + /// until `release`, and only the pinning request may touch it. + /// + /// The namespace check is not cosmetic: `dispatch` locks the collection + /// named in the *message*, so a `getMore` quoting one cursor's id and + /// another collection's name would otherwise iterate the first collection's + /// index while holding the second collection's lock. A mismatch leaves the + /// cursor alive -- it is the request that is wrong, not the cursor. + pub fn pin(self: *Store, io: std.Io, id: i64, ns: Ns, now_ms: i64) PinError!*Cursor { + try self.mutex.lock(io); + defer self.mutex.unlock(io); + if (id <= 0) return error.CursorNotFound; + const slot = slot_of(id); + if (slot >= self.slots.len) return error.CursorNotFound; + const c = self.slots[slot] orelse return error.CursorNotFound; + // Compare the whole id, not just the slot: this is what makes a + // recycled slot reject its predecessor's id. + if (c.id != id) return error.CursorNotFound; + if (self.expired(c, now_ms)) { + self.destroy(slot); + return error.CursorNotFound; + } + if (!c.ns_matches(ns)) return error.CursorNamespaceMismatch; + if (c.pinned) return error.CursorInUse; + c.pinned = true; + c.last_use_ms = now_ms; + return c; + } + + /// The namespace a live cursor belongs to, copied out. Only used to build + /// the namespace-mismatch error message, so a second lock acquisition on an + /// error path is the right trade for not threading an out-parameter through + /// the success path. + pub fn ns_of(self: *Store, io: std.Io, id: i64, out: *NsBuf) bool { + self.mutex.lock(io) catch return false; + defer self.mutex.unlock(io); + if (id <= 0) return false; + const slot = slot_of(id); + if (slot >= self.slots.len) return false; + const c = self.slots[slot] orelse return false; + if (c.id != id) return false; + out.set(c.ns()); + return true; + } + + /// Hand a pinned cursor back. `exhausted` destroys it, and so does a + /// `killCursors` that arrived while it was pinned. + pub fn release(self: *Store, io: std.Io, c: *Cursor, now_ms: i64, exhausted: bool) void { + self.mutex.lock(io) catch { + // Cancellation while returning a cursor would otherwise leave it + // pinned forever, unreachable and un-reapable. Unpinning without + // the lock is the lesser evil: the field is only ever written by + // the one request that owns the pin. + c.pinned = false; + return; + }; + defer self.mutex.unlock(io); + assert_msg(c.pinned, "released a cursor that was not pinned"); + c.pinned = false; + c.last_use_ms = now_ms; + if (exhausted or c.kill_requested) { + const slot = slot_of(c.id); + assert(self.slots[slot].? == c); + self.destroy(slot); + } + } + + /// `killCursors` for one id. A pinned cursor is marked and reported killed: + /// the client's intent is satisfied, and the in-flight request frees it on + /// release. Storage is never freed under a running request. + pub fn kill(self: *Store, io: std.Io, id: i64, ns: Ns) KillOutcome { + self.mutex.lock(io) catch return .not_found; + defer self.mutex.unlock(io); + if (id <= 0) return .not_found; + const slot = slot_of(id); + if (slot >= self.slots.len) return .not_found; + const c = self.slots[slot] orelse return .not_found; + if (c.id != id or !c.ns_matches(ns)) return .not_found; + if (c.pinned) { + c.kill_requested = true; + return .killed; + } + self.destroy(slot); + return .killed; + } + + /// Kill every cursor on a namespace. Called when the collection or its + /// database is dropped: a later `getMore` would fail anyway, since the + /// cursor holds names rather than a pointer, but reaping here frees the + /// slots at once and keeps the open-cursor metric honest. + pub fn kill_namespace( + self: *Store, + io: std.Io, + db_name: []const u8, + coll_name: ?[]const u8, + ) u32 { + self.mutex.lock(io) catch return 0; + defer self.mutex.unlock(io); + var n: u32 = 0; + for (self.slots, 0..) |maybe, i| { + const c = maybe orelse continue; + const own = c.ns(); + if (!std.mem.eql(u8, own.db, db_name)) continue; + if (coll_name) |name| { + if (!std.mem.eql(u8, own.coll, name)) continue; + } + if (c.pinned) { + c.kill_requested = true; + } else { + self.destroy(i); + } + n += 1; + } + return n; + } + + /// Reap idle cursors. Returns how many went. + pub fn sweep(self: *Store, io: std.Io, now_ms: i64) u32 { + self.mutex.lock(io) catch return 0; + defer self.mutex.unlock(io); + if (self.live == 0) return 0; + var n: u32 = 0; + for (self.slots, 0..) |maybe, i| { + const c = maybe orelse continue; + if (!self.expired(c, now_ms)) continue; + self.destroy(i); + n += 1; + } + return n; + } +}; + +// --------------------------------------------------------------------------- +// Batch policy +// --------------------------------------------------------------------------- + +/// What `offer` decided about one document. +pub const Offered = enum { + appended, + /// The batch is full. **Not** EOF: the cursor stays open, and this document + /// has not been consumed -- the caller must hand it to the next batch. + batch_full, +}; + +/// Accumulates one batch and owns the two limits that end it. +/// +/// Split out from the emit path so the whole policy is a pure function of +/// `(target, emitted, bytes, size)` and can be unit-tested without an engine, +/// a socket or an allocator. The subtle parts are all here: a target of 0 is +/// unbounded (a `getMore` naming no `batchSize`), a `batchSize: 0` first batch +/// is a target that is *reached immediately*, and the byte cap must still let +/// the first document through or an oversized document would wedge the cursor +/// forever, returning empty batches with no progress. +pub const BatchBuilder = struct { + /// Documents wanted; null means no document target, fill to the byte cap. + /// Optional rather than "0 means unbounded" because `batchSize: 0` is a real + /// request for an empty batch, and conflating the two returned the whole + /// collection where mongod returns nothing. + target: ?u32, + bytes_max: u64 = batch_bytes_max, + emitted: u32 = 0, + bytes: u64 = 0, + + pub fn init(target: ?u32) BatchBuilder { + return .{ .target = target }; + } + + /// Whether the batch has already met its document target, checked before + /// pulling from the source so a full batch never consumes a document it + /// cannot carry. + pub fn full(self: *const BatchBuilder) bool { + const t = self.target orelse return false; + return self.emitted >= t; + } + + /// Account for a document of `size` serialized bytes. + pub fn offer(self: *BatchBuilder, size: u64) Offered { + assert_msg(!self.full(), "offered a document to a batch that was already full"); + // The at-least-one rule: an empty batch takes the document whatever it + // measures. Stored documents cannot exceed the cap (inserts enforce + // 16 MiB), so this only arises for a generated one. + if (self.emitted > 0 and self.bytes + size > self.bytes_max) return .batch_full; + self.emitted += 1; + self.bytes += size; + return .appended; + } +}; + +/// The document target for a batch: the client's `batchSize` if it named one, +/// otherwise 101 for a first batch and *no* document target for a `getMore`. +/// +/// Both defaults are measured against mongod 8.3.7 rather than assumed. +/// `internalQueryFindCommandBatchSize` reports 101, and a `getMore` carrying no +/// `batchSize` after a `find` with `batchSize: 2` returns 4998 of 5000 +/// documents -- so a bare `getMore` is bounded by bytes alone and does *not* +/// inherit the `batchSize` the cursor was created with. +pub fn batch_target(batch_size: ?u32, first: bool) ?u32 { + if (batch_size) |n| return n; + return if (first) default_first_batch else null; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +const testing = std.testing; + +/// A Store for tests, with a threaded Io so the mutex is real. +const TestStore = struct { + threaded: std.Io.Threaded, + store: Store, + + fn init(capacity: u32, idle_timeout_ms: i64) !TestStore { + var self: TestStore = undefined; + self.threaded = std.Io.Threaded.init(testing.allocator, .{}); + self.store = try Store.init( + testing.allocator, + self.threaded.io(), + capacity, + idle_timeout_ms, + ); + return self; + } + + fn io(self: *TestStore) std.Io { + return self.threaded.io(); + } + + fn deinit(self: *TestStore) void { + self.store.deinit(); + self.threaded.deinit(); + } + + fn open_one(self: *TestStore, coll: []const u8, now_ms: i64) !i64 { + const arena = std.heap.ArenaAllocator.init(testing.allocator); + return self.store.open(self.io(), now_ms, arena, .{ + .ns = .{ .db = "t", .coll = coll }, + .layout_epoch = 0, + .source = .{ .buffered = .{ .docs = &.{} } }, + }); + } +}; + +test "cursor ids are positive, address their slot, and never repeat" { + var ts = try TestStore.init(4, default_idle_timeout_ms); + defer ts.deinit(); + + var seen: [16]i64 = undefined; + for (0..16) |i| { + const id = try ts.open_one("c", 0); + try testing.expect(id > 0); + // Freeing the slot immediately means the next open reuses it, which is + // exactly the case the nonce has to survive. + const killed = ts.store.kill(ts.io(), id, .{ .db = "t", .coll = "c" }); + try testing.expectEqual(KillOutcome.killed, killed); + seen[i] = id; + } + for (seen, 0..) |a, i| { + for (seen[i + 1 ..]) |b| try testing.expect(a != b); + } +} + +test "a recycled slot rejects the id it used to hold" { + // The guard that keeps one client from reading another's cursor. Without + // the nonce in the id, `slot_of(stale) == slot_of(fresh)` and the stale + // getMore would be served the new cursor's documents. + var ts = try TestStore.init(1, default_idle_timeout_ms); + defer ts.deinit(); + const ns = Ns{ .db = "t", .coll = "c" }; + + const stale = try ts.open_one("c", 0); + try testing.expectEqual(KillOutcome.killed, ts.store.kill(ts.io(), stale, ns)); + const fresh = try ts.open_one("c", 0); + + try testing.expectEqual(Store.slot_of(stale), Store.slot_of(fresh)); + try testing.expect(stale != fresh); + try testing.expectError(error.CursorNotFound, ts.store.pin(ts.io(), stale, ns, 0)); + _ = try ts.store.pin(ts.io(), fresh, ns, 0); +} + +test "pin rejects a wrong namespace and a second holder, and leaves the cursor alive" { + var ts = try TestStore.init(4, default_idle_timeout_ms); + defer ts.deinit(); + const ns = Ns{ .db = "t", .coll = "c" }; + const id = try ts.open_one("c", 0); + + // A wrong namespace must not kill the cursor: the request is wrong, not + // the cursor, and the client is allowed to retry correctly. Reported apart + // from CursorNotFound because mongod answers it with Unauthorized (13). + const wrong_coll = Ns{ .db = "t", .coll = "other" }; + const wrong_db = Ns{ .db = "other", .coll = "c" }; + const mismatch = error.CursorNamespaceMismatch; + try testing.expectError(mismatch, ts.store.pin(ts.io(), id, wrong_coll, 0)); + try testing.expectError(mismatch, ts.store.pin(ts.io(), id, wrong_db, 0)); + + var found: NsBuf = .{}; + try testing.expect(ts.store.ns_of(ts.io(), id, &found)); + try testing.expectEqualStrings("t", found.ns().db); + try testing.expectEqualStrings("c", found.ns().coll); + + const c = try ts.store.pin(ts.io(), id, ns, 0); + try testing.expectError(error.CursorInUse, ts.store.pin(ts.io(), id, ns, 0)); + ts.store.release(ts.io(), c, 1, false); + // Released, so it can be pinned again. + const again = try ts.store.pin(ts.io(), id, ns, 2); + ts.store.release(ts.io(), again, 3, true); + try testing.expectError(error.CursorNotFound, ts.store.pin(ts.io(), id, ns, 4)); +} + +test "a full store evicts the least recently used unpinned cursor" { + var ts = try TestStore.init(3, default_idle_timeout_ms); + defer ts.deinit(); + const ns = Ns{ .db = "t", .coll = "c" }; + + const a = try ts.open_one("c", 100); + const b = try ts.open_one("c", 200); + const c = try ts.open_one("c", 300); + // Touch `a` so `b` becomes the least recently used. + const pinned_a = try ts.store.pin(ts.io(), a, ns, 400); + ts.store.release(ts.io(), pinned_a, 400, false); + + const d = try ts.open_one("c", 500); + try testing.expectError(error.CursorNotFound, ts.store.pin(ts.io(), b, ns, 500)); + for ([_]i64{ a, c, d }) |id| { + const live = try ts.store.pin(ts.io(), id, ns, 500); + ts.store.release(ts.io(), live, 500, false); + } +} + +test "a store whose every slot is pinned refuses rather than evicting" { + var ts = try TestStore.init(2, default_idle_timeout_ms); + defer ts.deinit(); + const ns = Ns{ .db = "t", .coll = "c" }; + const a = try ts.open_one("c", 0); + const b = try ts.open_one("c", 0); + _ = try ts.store.pin(ts.io(), a, ns, 0); + _ = try ts.store.pin(ts.io(), b, ns, 0); + try testing.expectError(error.TooManyCursors, ts.open_one("c", 0)); +} + +test "the sweep reaps idle cursors and spares noCursorTimeout" { + var ts = try TestStore.init(4, 1000); + defer ts.deinit(); + const ns = Ns{ .db = "t", .coll = "c" }; + + const perishable = try ts.open_one("c", 0); + const arena = std.heap.ArenaAllocator.init(testing.allocator); + const immortal = try ts.store.open(ts.io(), 0, arena, .{ + .ns = ns, + .layout_epoch = 0, + .no_timeout = true, + .source = .{ .buffered = .{ .docs = &.{} } }, + }); + + // Just short of the timeout: nothing goes. + try testing.expectEqual(@as(u32, 0), ts.store.sweep(ts.io(), 999)); + try testing.expectEqual(@as(u32, 1), ts.store.sweep(ts.io(), 1000)); + try testing.expectError(error.CursorNotFound, ts.store.pin(ts.io(), perishable, ns, 1000)); + + // The exempt one survives an interval it would otherwise have died in... + try testing.expectEqual(@as(u32, 0), ts.store.sweep(ts.io(), 100_000)); + const live = try ts.store.pin(ts.io(), immortal, ns, 100_000); + ts.store.release(ts.io(), live, 100_000, false); + // ...but is still killable explicitly. + try testing.expectEqual(KillOutcome.killed, ts.store.kill(ts.io(), immortal, ns)); +} + +test "killCursors reports a pinned cursor killed and frees it on release" { + var ts = try TestStore.init(4, default_idle_timeout_ms); + defer ts.deinit(); + const ns = Ns{ .db = "t", .coll = "c" }; + const id = try ts.open_one("c", 0); + const c = try ts.store.pin(ts.io(), id, ns, 0); + + try testing.expectEqual(KillOutcome.killed, ts.store.kill(ts.io(), id, ns)); + // Still pinned, so its storage must not have been freed under the request. + try testing.expect(c.kill_requested); + // Not exhausted, but the pending kill wins. + ts.store.release(ts.io(), c, 1, false); + try testing.expectError(error.CursorNotFound, ts.store.pin(ts.io(), id, ns, 2)); + try testing.expectEqual(KillOutcome.not_found, ts.store.kill(ts.io(), id, ns)); +} + +test "kill_namespace reaps a collection's cursors and leaves the rest" { + var ts = try TestStore.init(8, default_idle_timeout_ms); + defer ts.deinit(); + const doomed = try ts.open_one("doomed", 0); + const spared = try ts.open_one("spared", 0); + + try testing.expectEqual(@as(u32, 1), ts.store.kill_namespace(ts.io(), "t", "doomed")); + const doomed_ns = Ns{ .db = "t", .coll = "doomed" }; + try testing.expectError(error.CursorNotFound, ts.store.pin(ts.io(), doomed, doomed_ns, 0)); + const live = try ts.store.pin(ts.io(), spared, .{ .db = "t", .coll = "spared" }, 0); + ts.store.release(ts.io(), live, 0, false); + + // Whole-database form. + try testing.expectEqual(@as(u32, 1), ts.store.kill_namespace(ts.io(), "t", null)); + try testing.expectEqual(@as(u32, 0), ts.store.live); +} + +test "a namespace too long for the fixed buffers declines rather than failing" { + var ts = try TestStore.init(2, default_idle_timeout_ms); + defer ts.deinit(); + const long = "c" ** (ns_coll_max + 1); + try testing.expectError(error.NameTooLong, ts.open_one(long, 0)); +} + +test "batch_target: 101 for a first batch, unbounded for a getMore, honoured when given" { + try testing.expectEqual(@as(?u32, default_first_batch), batch_target(null, true)); + // A bare getMore has no document target at all. Measured against mongod: + // it does not inherit the batchSize the cursor was created with. + try testing.expectEqual(@as(?u32, null), batch_target(null, false)); + try testing.expectEqual(@as(?u32, 7), batch_target(7, true)); + + // batchSize: 0 is a real target of zero -- an empty first batch with a live + // cursor, which drivers use to obtain a cursor cheaply. It must NOT read as + // "unbounded": conflating the two returns the whole collection where mongod + // returns nothing, which is exactly the bug this optional prevents. + try testing.expectEqual(@as(?u32, 0), batch_target(0, true)); + var zero = BatchBuilder.init(batch_target(0, true)); + try testing.expect(zero.full()); + var bare = BatchBuilder.init(batch_target(null, false)); + try testing.expect(!bare.full()); +} + +test "BatchBuilder stops at its document target" { + var b = BatchBuilder.init(2); + try testing.expect(!b.full()); + try testing.expectEqual(Offered.appended, b.offer(10)); + try testing.expect(!b.full()); + try testing.expectEqual(Offered.appended, b.offer(10)); + try testing.expect(b.full()); + try testing.expectEqual(@as(u32, 2), b.emitted); +} + +test "BatchBuilder: a null target is unbounded by documents" { + var b = BatchBuilder.init(null); + for (0..5000) |_| { + try testing.expect(!b.full()); + try testing.expectEqual(Offered.appended, b.offer(1)); + } + try testing.expect(!b.full()); +} + +test "BatchBuilder stops on bytes, but always takes at least one document" { + // Hitting the byte cap must not read as EOF, or the cursor would close and + // silently drop the rest of the result. + var b = BatchBuilder.init(null); + b.bytes_max = 100; + try testing.expectEqual(Offered.appended, b.offer(60)); + try testing.expectEqual(Offered.batch_full, b.offer(60)); + // The refused document was not accounted for, so the caller can hand it to + // the next batch. + try testing.expectEqual(@as(u32, 1), b.emitted); + try testing.expectEqual(@as(u64, 60), b.bytes); + + // An oversized document on an empty batch goes through anyway: refusing it + // would wedge the cursor, returning empty batches and never progressing. + var solo = BatchBuilder.init(null); + solo.bytes_max = 100; + try testing.expectEqual(Offered.appended, solo.offer(1_000_000)); + try testing.expectEqual(@as(u32, 1), solo.emitted); + try testing.expectEqual(Offered.batch_full, solo.offer(1)); +} + +test "Stream.advance records the anchor and counts an equal-key band" { + var s = Stream{}; + try testing.expect(!s.started()); + + s.advance("aaa", 10, 3, 4); + try testing.expect(s.started()); + try testing.expectEqualStrings("aaa", s.anchor_key()); + try testing.expectEqual(@as(u64, 10), s.anchor_off); + try testing.expectEqual(@as(u32, 3), s.hint_leaf); + try testing.expectEqual(@as(u32, 4), s.hint_slot); + try testing.expectEqual(@as(u64, 0), s.band_index); + + // Same key, different document: still inside the band, so the position + // within it has to advance or a resume could not tell them apart. + s.advance("aaa", 11, 3, 5); + try testing.expectEqual(@as(u64, 1), s.band_index); + s.advance("aaa", 12, 3, 6); + try testing.expectEqual(@as(u64, 2), s.band_index); + + // A new key ends the band. + s.advance("bbb", 13, 3, 7); + try testing.expectEqual(@as(u64, 0), s.band_index); + try testing.expectEqualStrings("bbb", s.anchor_key()); +} + +test "Stream.advance accepts a reverse cursor moving down" { + var s = Stream{ .backward = true }; + s.advance("ccc", 1, 1, 5); + s.advance("bbb", 2, 1, 4); + s.advance("aaa", 3, 1, 3); + try testing.expectEqualStrings("aaa", s.anchor_key()); +} diff --git a/src/db.zig b/src/db.zig index 771a0e9..57ba93e 100644 --- a/src/db.zig +++ b/src/db.zig @@ -22,6 +22,7 @@ const bson = @import("bson.zig"); const storage = @import("storage.zig"); const index = @import("index.zig"); const pgr = @import("pager.zig"); +const cursor = @import("cursor.zig"); // Always active, including in the default ReleaseFast build -- see assert.zig // for why std.debug.assert is the wrong tool for these invariants. const assert = @import("assert.zig").assert; @@ -97,8 +98,21 @@ pub const Collection = struct { /// replaces the old serialization-guarded docs-map fast path for /// integer/string/etc. _id lookups. id_index: index.Index, + /// Identity-and-layout token for open cursors. Drawn from + /// `Engine.layout_epoch_seq`, so it is unique across the engine's life and + /// bumped again by every rebuild. + /// + /// It answers two questions a cursor cannot answer any other way. A rebuild + /// moves every document, so a saved slab offset (or a saved index anchor's + /// offset) is stale -- and the keys surviving unchanged makes that *worse*, + /// because a lookup then succeeds and quietly resolves to the wrong bytes. + /// And a cursor holds namespace *strings*, not a `*Collection`, so a + /// drop-and-recreate under the same name would otherwise be invisible to it; + /// drawing from an engine-wide sequence rather than starting each collection + /// at zero is what makes the recreated one compare unequal. + layout_epoch: u64, - fn init(gpa: std.mem.Allocator, pager: *pgr.Pager) !Collection { + fn init(gpa: std.mem.Allocator, pager: *pgr.Pager, layout_epoch: u64) !Collection { var self: Collection = .{ .doc_count = 0, .pager = pager, @@ -110,6 +124,7 @@ pub const Collection = struct { .hold = .{}, .indexes = .empty, .id_index = undefined, + .layout_epoch = layout_epoch, }; const keys = [_]index.IndexKey{.{ .path = "_id", .descending = false }}; // unique: the tree, not the docs map, is what enforces _id uniqueness @@ -290,6 +305,15 @@ pub const Engine = struct { /// rewrite is worth doing — see `note_compact`. live_docs: u64 = 0, dead_docs: u64 = 0, + /// Hands out `Collection.layout_epoch` values. Monotonic and never reset, so + /// no two collection instances -- including a drop followed by a recreate + /// under the same name -- ever share one. + layout_epoch_seq: u64 = 0, + /// Open cursors. Lives on the engine rather than the server because the C + /// API seam (PLAN D1) lists cursor iteration, and because the unit tests + /// build an Engine with no server at all. Its mutex is a leaf: see + /// `cursor.Store`. + cursors: cursor.Store, /// The same question in bytes, about the *data file* rather than the log. /// Once a checkpoint truncates the log, the log no longer holds the garbage /// -- the doc slab does, and only a rebuild reclaims it. These are what @@ -324,6 +348,12 @@ pub const Engine = struct { /// command reads it while still holding the write lock. dup_index: ?[]const u8 = null, + /// The registry an embedded caller gets without configuring anything; the + /// CLI replaces it through `reconfigure_cursors`. + fn default_cursor_store(gpa: std.mem.Allocator, io: std.Io) !cursor.Store { + return cursor.Store.init(gpa, io, cursor.default_capacity, cursor.default_idle_timeout_ms); + } + pub fn open(gpa: std.mem.Allocator, io: std.Io, path: []const u8) !Engine { var log = try storage.Log.open(gpa, io, path); errdefer log.close(); @@ -346,8 +376,10 @@ pub const Engine = struct { .dbs = .empty, .seq = 0, .compact_threshold = 16 * 1024 * 1024, + .cursors = try default_cursor_store(gpa, io), }; errdefer { + engine.cursors.deinit(); engine.pager.deinit(); engine.dbs.deinit(gpa); } @@ -393,6 +425,17 @@ pub const Engine = struct { return engine; } + /// Replace the cursor registry with one of a different shape. Only legal + /// before the server starts accepting connections, because it drops every + /// cursor -- asserted rather than left to the comment, since the method is + /// public and a later caller would otherwise get silent data loss. + pub fn reconfigure_cursors(self: *Engine, capacity: u32, idle_timeout_ms: i64) !void { + assert_msg(self.cursors.live == 0, "reconfigured the cursor registry with cursors open"); + const fresh = try cursor.Store.init(self.gpa, self.io, capacity, idle_timeout_ms); + self.cursors.deinit(); + self.cursors = fresh; + } + pub fn deinit(self: *Engine) void { var db_it = self.dbs.iterator(); while (db_it.next()) |db_entry| { @@ -400,6 +443,11 @@ pub const Engine = struct { self.gpa.free(db_entry.key_ptr.*); } self.dbs.deinit(self.gpa); + // Before the pager: a cursor's arena is its own, but freeing cursors + // first keeps the teardown order the same as the construction order + // reversed, which is the only order that stays obviously correct as + // cursors grow to hold more. + self.cursors.deinit(); self.pager.deinit(); self.gpa.destroy(self.pager); self.log.close(); @@ -936,6 +984,11 @@ pub const Engine = struct { const removed = db.collections.fetchRemove(coll_name) orelse return false; self.free_collection(removed.value); self.gpa.free(removed.key); + // A cursor on this namespace is already safe -- it holds names, so its + // next getMore finds nothing to lock -- but reaping here frees the slots + // now instead of at the idle timeout, and keeps the open-cursor metric + // describing cursors that can still return something. + _ = self.cursors.kill_namespace(self.io, db_name, coll_name); return true; } @@ -943,6 +996,7 @@ pub const Engine = struct { var removed = self.dbs.fetchRemove(db_name) orelse return false; self.free_db(&removed.value); self.gpa.free(removed.key); + _ = self.cursors.kill_namespace(self.io, db_name, null); return true; } @@ -1156,7 +1210,8 @@ pub const Engine = struct { errdefer self.gpa.free(coll_key); const new_coll = try self.gpa.create(Collection); errdefer self.gpa.destroy(new_coll); - new_coll.* = try Collection.init(self.gpa, self.pager); + self.layout_epoch_seq += 1; + new_coll.* = try Collection.init(self.gpa, self.pager, self.layout_epoch_seq); errdefer new_coll.id_index.deinit(self.gpa); try db.collections.put(self.gpa, coll_key, new_coll); return new_coll; @@ -1389,6 +1444,13 @@ pub const Engine = struct { for (coll.indexes.items) |ix| try self.repack_index(coll, ix, moved.items); for (old_extents) |e| try self.pager.free_pages(e.first, e.pages); + + // Every document has moved, so every offset an open cursor is holding + // now names different bytes. Bumped last, after the rebuild can no + // longer fail: a cursor invalidated by a rebuild that then errored out + // would have been invalidated for nothing. + self.layout_epoch_seq += 1; + coll.layout_epoch = self.layout_epoch_seq; } fn repack_index( @@ -3886,3 +3948,64 @@ const Reader = struct { return self.take(n); } }; + +test "the epochs that invalidate a cursor move exactly when they must" { + // Three separate promises, each one load-bearing for an open cursor: + // + // - a rebuild moves every document, so a saved slab offset is stale; + // - a drop-and-recreate under the same name is a different collection, + // which a cursor holding only namespace strings cannot otherwise see; + // - `Index.reset_tree` re-creates node ids 0 and 1 as different nodes, so a + // saved (leaf, slot) position becomes valid-and-wrong rather than absent. + // + // A cursor's whole safety story is these three bumps, so assert them here + // rather than inferring them from cursor behaviour later. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + + try engine.lock(); + var i: i32 = 0; + while (i < 40) : (i += 1) { + var d = try make_doc(gpa, i, "payload"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + const before = engine.get_collection("app", "c").?.layout_epoch; + engine.unlock(); + try testing.expect(before != 0); + + // A rebuild moves documents, so the epoch must move with them. + try engine.compact(); + try engine.lock(); + const after_rebuild = engine.get_collection("app", "c").?.layout_epoch; + engine.unlock(); + try testing.expect(after_rebuild != before); + + // A recreated collection must not be mistaken for the one that was + // dropped. Starting each collection's epoch at zero would fail here. + try engine.lock(); + try testing.expect(try engine.drop_collection("app", "c")); + var fresh_doc = try make_doc(gpa, 1, "fresh"); + defer fresh_doc.deinit(); + try engine.insert("app", "c", &fresh_doc, &env.gen); + const after_recreate = engine.get_collection("app", "c").?.layout_epoch; + engine.unlock(); + try testing.expect(after_recreate != after_rebuild); + try testing.expect(after_recreate != before); + + // And the index-level token, which guards the position hint. + try engine.lock(); + const coll = engine.get_collection("app", "c").?; + const index_before = coll.id_index.epoch; + try coll.id_index.reset_tree(gpa); + try testing.expect(coll.id_index.epoch != index_before); + engine.unlock(); +} diff --git a/src/index.zig b/src/index.zig index afda333..a06897e 100644 --- a/src/index.zig +++ b/src/index.zig @@ -106,7 +106,9 @@ const page_size = 4096; /// Bytes of node payload: a 32-byte header plus the slotted region. const page_data = page_size - 32; /// Records longer than a quarter of a node spill to the overflow slab. -const inline_limit = page_size / 4; +/// Public because it is also the bound a resumable cursor anchors within: a key +/// past it has already spilled, so the tree itself treats it as exceptional. +pub const inline_limit = page_size / 4; /// Upper bound on the slots one node can hold, since every slot costs at /// least its own size. Bounds the split scratch. const max_slots = page_data / slot_size; @@ -234,6 +236,19 @@ pub const Index = struct { depth: u32, /// Total entries, maintained incrementally. entry_count: usize, + /// Bumped whenever a node id stops meaning what it meant, which is the one + /// thing that makes a saved `(leaf, slot)` position dangerous rather than + /// merely stale. Node ids are otherwise append-only (`alloc_node`, and + /// `drop_child` abandons a page without recycling its id), and `page()` + /// resolves ids through `node_pages`, so copy-on-write and checkpoints move + /// pages without disturbing ids. Only `reset_tree` and + /// `replace_root_with_leaf` reuse an id for different contents. + /// + /// A resumable cursor keeps a position hint to avoid walking an equal-key + /// band on every `getMore`; it must compare this first. Without it the hint + /// would address a live but unrelated leaf after a compaction and the cursor + /// would iterate a tree that no longer exists. + epoch: u64, /// Repack scratch: any single node's record bytes fit here. scratch: [page_data]u8, /// Promoted-key scratch: inline keys being propagated up a split are @@ -268,6 +283,7 @@ pub const Index = struct { .leaf_count = 0, .depth = 0, .entry_count = 0, + .epoch = 0, .scratch = undefined, .promo = undefined, }; @@ -614,6 +630,10 @@ pub const Index = struct { self.depth = 0; self.entry_count = 0; self.multikey = false; + // Node ids 0 and 1 were just re-created as different nodes, so every + // position anyone saved into the old tree now points somewhere valid + // and wrong. This is the bump that tells them apart. + self.epoch += 1; } /// Remove every entry for `id`, in one pass over the leaves. Infallible. @@ -777,6 +797,14 @@ pub const Index = struct { leaf: u32, slot: u32, + /// `next`, plus the position of the entry it yielded. Exact because + /// `next` leaves `leaf` alone on the call that yields and has already + /// incremented `slot` past the entry. + pub fn positioned(self: *Iter) ?Positioned { + const e = self.next() orelse return null; + return .{ .key = e.key, .off = e.off, .leaf = self.leaf, .slot = self.slot - 1 }; + } + pub fn next(self: *Iter) ?EntryRef { const ix = self.ix; while (self.leaf != 0) { @@ -872,6 +900,13 @@ pub const Index = struct { /// One past the slot to yield next, so 0 means this leaf is done. slot: u32, + /// As `Iter.positioned`, but `RevIter.next` decrements *onto* the entry + /// it yields, so the slot needs no adjustment. + pub fn positioned(self: *RevIter) ?Positioned { + const e = self.next() orelse return null; + return .{ .key = e.key, .off = e.off, .leaf = self.leaf, .slot = self.slot }; + } + pub fn next(self: *RevIter) ?EntryRef { const ix = self.ix; while (self.leaf != 0) { @@ -919,6 +954,174 @@ pub const Index = struct { return .{ .ix = self, .leaf = b.leaf, .slot = b.slot }; } + // -- resuming an interrupted scan --------------------------------------- + + /// Entries a resume will walk past before giving up and reporting `capped`. + /// A bound rather than a hope: `seek` lands at the *start* of an equal-key + /// band, so without one a key with millions of duplicates would make every + /// batch cost O(band) and a full drain quadratic. + pub const resume_walk_max: u32 = 1 << 16; + + /// One entry, with enough of its position to resume after it next time. + pub const Positioned = struct { + key: []const u8, + off: u64, + leaf: u32, + slot: u32, + }; + + /// A resumed forward walk. `capped` means the anchor could not be located + /// within `resume_walk_max` steps, so the position is not trustworthy and + /// the caller must fail rather than return documents from the wrong place. + pub const Resumed = struct { it: Iter, capped: bool = false }; + pub const ResumedRev = struct { it: RevIter, capped: bool = false }; + + /// Does `(leaf, slot)` still hold exactly `(key, off)`? + /// + /// A hint is never believed, only checked, and the checks are ordered so the + /// cheap structural ones run first: `off_of` asserts `is_leaf` with + /// `std.debug.assert`, which in ReleaseFast is a promise to the optimizer + /// rather than a check, so `is_leaf` must be tested for real beforehand. + /// + /// Node ids are append-only, so a stale id is always in bounds; what makes a + /// hint dangerous rather than merely wrong is `reset_tree` re-creating ids 0 + /// and 1 as different nodes, and `Index.epoch` is what the caller compares + /// for that. + fn hint_holds(self: *const Index, leaf: u32, slot: u32, key: []const u8, off: u64) bool { + if (leaf == 0 or leaf >= self.node_pages.items.len) return false; + const node = self.page(leaf); + if (node.is_leaf != 1) return false; + if (slot >= node.count) return false; + if (self.off_of(leaf, slot) != off) return false; + return std.mem.eql(u8, self.key_of(leaf, slot), key); + } + + /// Locate the anchor `(key, off)` by walking its equal-key band. + /// + /// Comparison is `std.mem.order`, not `cmp_prefix`, because the band is + /// defined as the entries whose key is byte-equal to the anchor's and prefix + /// semantics would call `"ab"` and `"abc"` equal. In fairness the two happen + /// to agree on where this function resumes -- the fallback is positional, and + /// the first out-of-band entry is the same entry either way -- so this is a + /// clarity choice, not a bug fix; an attempted mutation to `cmp_prefix` does + /// not change any observable result. What does matter is that `lower_bound` + /// uses prefix semantics and therefore errs *before* the band, never past it, + /// so the walk cannot start beyond the anchor and skip it. + /// + /// When the anchor is gone, "gone" turns out to mean two different things and + /// they want opposite answers: + /// + /// - **Deleted.** A sibling has moved up into the anchor's band position, and + /// that sibling has not been returned yet. Resume *at* band position + /// `band_index`. Resuming after the whole band instead would silently drop + /// every remaining member, which on a three-value index is most of the + /// collection. + /// - **Updated.** The document was rewritten, so its key is unchanged but its + /// offset moved. The entry at the anchor's band position *is* the anchor, + /// already returned. Resume *after* it. + /// + /// The index cannot tell these apart in general -- both look like "same key, + /// different offset". On a **unique** index it can: two entries cannot share a + /// key, so a same-key entry is necessarily the same document, hence the update + /// case, hence resume past the band. That covers `_id_` and so every unsorted + /// scan and every `_id` sort, which is where an update-during-drain otherwise + /// returns a document twice -- observed as duplicate `_id`s draining a + /// collection that was being updated underneath. + /// + /// On a non-unique index the positional fallback stands, so an updated document + /// may come back a second time. That is legal: MongoDB documents that a + /// non-snapshot cursor may return a document more than once if an intervening + /// write moves it. + fn band_resume(self: *const Index, key: []const u8, off: u64, band_index: u64) Resumed { + var it = self.seek(key); + var fallback: ?Iter = null; + var pos: u64 = 0; + var steps: u32 = 0; + while (steps < resume_walk_max) : (steps += 1) { + // The iterator state that would yield the entry we are about to + // look at, i.e. "resume *at* this entry". + const before = it; + const e = it.next() orelse break; + if (std.mem.order(u8, e.key, key) != .eq) { + // Past the band. Prefer the fallback if the band held one. + return .{ .it = fallback orelse before }; + } + if (e.off == off) return .{ .it = it }; // resume just after the anchor + // On a unique index a same-key entry can only be the anchor itself, + // rewritten, so there is no sibling to fall back to. + if (!self.unique and pos == band_index and fallback == null) fallback = before; + pos += 1; + } + if (steps == resume_walk_max) return .{ .it = it, .capped = true }; + return .{ .it = fallback orelse it }; + } + + /// A forward walk positioned just after `(key, off)`. + /// + /// O(1) whenever the hint still holds, which is the case unless something + /// wrote to that exact leaf between batches. The band walk is the fallback, + /// and it is what the walk bound exists to contain. + pub fn resume_forward( + self: *const Index, + key: []const u8, + off: u64, + band_index: u64, + hint_leaf: u32, + hint_slot: u32, + hint_trusted: bool, + ) Resumed { + if (hint_trusted and self.hint_holds(hint_leaf, hint_slot, key, off)) { + return .{ .it = .{ .ix = self, .leaf = hint_leaf, .slot = hint_slot + 1 } }; + } + return self.band_resume(key, off, band_index); + } + + /// A reverse walk positioned just before `(key, off)` in key order, i.e. at + /// the next entry a descending scan owes. + /// + /// `RevIter` decrements before yielding, so slot `s` yields `s - 1` -- the + /// entry immediately below the anchor -- and crosses into `prev` when the + /// anchor sat at slot 0. + /// + /// Known limitation, and it is a deliberate trade. When the anchor is gone + /// *and* it had duplicates, this resumes below the whole band rather than at + /// the anchor's position within it, so the band's remaining members are not + /// returned. Placing a reverse fallback exactly would need the band's length, + /// which is only known after walking it, hence a second walk on a path that + /// requires a descending scan over a duplicate-heavy index whose anchor was + /// deleted mid-cursor. Forward resumes -- every unsorted scan and every + /// ascending sort -- use `band_index` and have no such gap. + pub fn resume_reverse( + self: *const Index, + key: []const u8, + off: u64, + hint_leaf: u32, + hint_slot: u32, + hint_trusted: bool, + ) ResumedRev { + if (hint_trusted and self.hint_holds(hint_leaf, hint_slot, key, off)) { + return .{ .it = .{ .ix = self, .leaf = hint_leaf, .slot = hint_slot } }; + } + // Find the anchor by walking forward, then turn around on it. + var it = self.seek(key); + var steps: u32 = 0; + while (steps < resume_walk_max) : (steps += 1) { + const e = it.next() orelse break; + if (std.mem.order(u8, e.key, key) != .eq) break; // past the band + if (e.off == off) { + // `it` has already stepped past the anchor, so the anchor sat at + // `it.slot - 1` and a RevIter there yields the entry below it. + return .{ .it = .{ .ix = self, .leaf = it.leaf, .slot = it.slot - 1 } }; + } + } + if (steps == resume_walk_max) { + return .{ .it = .{ .ix = self, .leaf = 0, .slot = 0 }, .capped = true }; + } + // Anchor gone: resume below the band. + const b = self.lower_bound(key); + return .{ .it = .{ .ix = self, .leaf = b.leaf, .slot = b.slot } }; + } + // -- serialization ------------------------------------------------------ /// The canonical spec document bytes @@ -1710,6 +1913,9 @@ pub const Index = struct { self.first_leaf = self.root; self.leaf_count = 1; self.depth = 0; + // The root's id is unchanged but it is a leaf now, so a saved position + // that named it as an internal node describes a different tree shape. + self.epoch += 1; } /// Pack the sorted staging array into a fresh tree: leaves filled in @@ -3672,3 +3878,258 @@ test "planner picks eq run, ranges, and bails on sparse null" { try testing.expect((try plan(gpa, null, &.{&ix}, &f, &.{})) == null); } } + +/// Drain `ix` by resuming every `stride` entries, the way a cursor with that +/// batch size would, and return the offsets in the order they came out. +fn drain_resuming( + gpa: std.mem.Allocator, + ix: *const Index, + stride: u32, + backward: bool, + out: *std.ArrayListUnmanaged(u64), +) !void { + var started = false; + var anchor: std.ArrayListUnmanaged(u8) = .empty; + defer anchor.deinit(gpa); + var anchor_off: u64 = 0; + var band_index: u64 = 0; + var hint_leaf: u32 = 0; + var hint_slot: u32 = 0; + + while (true) { + // One "batch": open a walk where the last one stopped. + var fwd: Index.Iter = undefined; + var rev: Index.RevIter = undefined; + if (!started) { + if (backward) rev = ix.iter_reverse() else fwd = ix.iter(); + } else if (backward) { + const r = ix.resume_reverse(anchor.items, anchor_off, hint_leaf, hint_slot, true); + try testing.expect(!r.capped); + rev = r.it; + } else { + const r = ix.resume_forward( + anchor.items, + anchor_off, + band_index, + hint_leaf, + hint_slot, + true, + ); + try testing.expect(!r.capped); + fwd = r.it; + } + + var n: u32 = 0; + while (n < stride) : (n += 1) { + const e = if (backward) + rev.positioned() + else + fwd.positioned(); + const got = e orelse return; + try out.append(gpa, got.off); + if (started and std.mem.eql(u8, anchor.items, got.key)) { + band_index += 1; + } else { + band_index = 0; + } + anchor.clearRetainingCapacity(); + try anchor.appendSlice(gpa, got.key); + anchor_off = got.off; + hint_leaf = got.leaf; + hint_slot = got.slot; + started = true; + } + } +} + +test "a resumed walk yields exactly what an uninterrupted one does" { + // The property the whole streaming cursor rests on: stopping and restarting + // a scan changes nothing about what it returns, in either direction, at any + // batch size, including one entry at a time. + // + // Mutation-checked: changing `resume_forward`'s `hint_slot + 1` to + // `hint_slot` makes every batch boundary repeat an entry, and this test goes + // red. (A third mutation was tried and rejected as meaningless: swapping + // `std.mem.order` for `cmp_prefix` inside `band_resume` changes nothing + // observable, so no test can catch it -- see the note there.) + // + // Note this test always resumes from a *valid* hint, since nothing mutates + // the tree between its batches. The band walk is covered by the two tests + // below, which invalidate the hint on purpose. + const gpa = testing.allocator; + + // Three corpora, each hard for a different reason: distinct keys spanning + // several leaves and an interior level; a low-cardinality index whose bands + // span leaves; and *variable-length string keys in prefix relationships* + // ("a" < "ab" < "abc"), which is the only shape that can tell `std.mem.order` + // apart from `cmp_prefix` -- fixed-width integer keys never differ, so an + // integer-only corpus cannot catch that mistake at all. + const Shape = enum { distinct, duplicates, prefixes }; + for ([_]Shape{ .distinct, .duplicates, .prefixes }) |shape| { + var ix = try simple_index(gpa, test_pager(), &.{"a"}, false, false); + defer ix.deinit(gpa); + + const n = 400; + var key_buf: [40]u8 = undefined; + for (0..n) |i| { + const value: bson.Value = switch (shape) { + .distinct => .{ .int32 = @intCast(i + 1) }, + .duplicates => .{ .int32 = @intCast(i % 3) }, + // Every key is a prefix of the next in its group of eight, so + // each band start is also a proper prefix of later keys. + .prefixes => blk: { + const written = try std.fmt.bufPrint(&key_buf, "k{d}", .{i / 8}); + const depth = (i % 8) + 1; + @memset(key_buf[written.len .. written.len + depth], 'x'); + break :blk .{ .string = key_buf[0 .. written.len + depth] }; + }, + }; + const d = try bytes_of(gpa, &.{ + .{ .key = "_id", .value = .{ .int32 = @intCast(i + 1) } }, + .{ .key = "a", .value = value }, + }); + defer gpa.free(d); + _ = try ix.add_doc(gpa, d, @intCast(i + 1), false); + } + try testing.expect(ix.depth >= 1); + + for ([_]bool{ false, true }) |backward| { + var whole: std.ArrayListUnmanaged(u64) = .empty; + defer whole.deinit(gpa); + if (backward) { + var it = ix.iter_reverse(); + while (it.next()) |e| try whole.append(gpa, e.off); + } else { + var it = ix.iter(); + while (it.next()) |e| try whole.append(gpa, e.off); + } + try testing.expectEqual(@as(usize, n), whole.items.len); + + for ([_]u32{ 1, 2, 7, 101, 399, 400, 1000 }) |stride| { + var resumed: std.ArrayListUnmanaged(u64) = .empty; + defer resumed.deinit(gpa); + try drain_resuming(gpa, &ix, stride, backward, &resumed); + try testing.expectEqualSlices(u64, whole.items, resumed.items); + } + } + } +} + +test "a resume survives a split between batches" { + // A cursor holds no lock, so the tree it comes back to is not the tree it + // left. Inserting mid-drain moves entries between leaves and invalidates the + // position hint, which is exactly what the anchor is for. + const gpa = testing.allocator; + var ix = try simple_index(gpa, test_pager(), &.{"a"}, false, false); + defer ix.deinit(gpa); + + const n = 200; + for (0..n) |i| { + // Even keys only, so the inserts below land between existing entries. + const d = try bytes_of(gpa, &.{ + .{ .key = "_id", .value = .{ .int32 = @intCast(i + 1) } }, + .{ .key = "a", .value = .{ .int32 = @intCast((i + 1) * 2) } }, + }); + defer gpa.free(d); + _ = try ix.add_doc(gpa, d, @intCast(i + 1), false); + } + + var seen: std.ArrayListUnmanaged(u64) = .empty; + defer seen.deinit(gpa); + var anchor: std.ArrayListUnmanaged(u8) = .empty; + defer anchor.deinit(gpa); + var anchor_off: u64 = 0; + var hint_leaf: u32 = 0; + var hint_slot: u32 = 0; + var started = false; + var next_id: i32 = 10_000; + + while (true) { + var it = if (!started) ix.iter() else blk: { + const r = ix.resume_forward(anchor.items, anchor_off, 0, hint_leaf, hint_slot, true); + try testing.expect(!r.capped); + break :blk r.it; + }; + var n_in_batch: u32 = 0; + while (n_in_batch < 5) : (n_in_batch += 1) { + const got = it.positioned() orelse break; + try seen.append(gpa, got.off); + anchor.clearRetainingCapacity(); + try anchor.appendSlice(gpa, got.key); + anchor_off = got.off; + hint_leaf = got.leaf; + hint_slot = got.slot; + started = true; + } + if (n_in_batch < 5) break; + + // Between batches, insert odd keys across the whole range: guaranteed to + // split leaves and to appear both before and after the anchor. + for (0..20) |k| { + next_id += 1; + const d = try bytes_of(gpa, &.{ + .{ .key = "_id", .value = .{ .int32 = next_id } }, + .{ .key = "a", .value = .{ .int32 = @intCast(k * 19 + 1) } }, + }); + defer gpa.free(d); + _ = try ix.add_doc(gpa, d, @intCast(next_id), false); + } + } + + // The 200 originals must each appear exactly once. Entries inserted behind + // the cursor may or may not show up -- that is ordinary non-snapshot cursor + // behaviour -- but nothing may be duplicated or lost. + var originals: u32 = 0; + var counts = std.AutoHashMap(u64, u32).init(gpa); + defer counts.deinit(); + for (seen.items) |off| { + const e = try counts.getOrPutValue(off, 0); + e.value_ptr.* += 1; + try testing.expectEqual(@as(u32, 1), e.value_ptr.*); // no duplicates + if (off <= n) originals += 1; + } + try testing.expectEqual(@as(u32, n), originals); +} + +test "a resume whose anchor was deleted keeps the rest of its band" { + // The failure this guards against is silent and large: with the anchor gone, + // resuming after the whole equal-key band drops every remaining member, and + // on a low-cardinality index that is most of the collection. + // + // Mutation-checked: `pos == band_index + 1` in `band_resume` shifts the + // resume by one entry and this test goes red. + const gpa = testing.allocator; + var ix = try simple_index(gpa, test_pager(), &.{"a"}, false, false); + defer ix.deinit(gpa); + + // One key, 50 documents: a single band. + for (0..50) |i| { + const d = try bytes_of(gpa, &.{ + .{ .key = "_id", .value = .{ .int32 = @intCast(i + 1) } }, + .{ .key = "a", .value = .{ .int32 = 7 } }, + }); + defer gpa.free(d); + _ = try ix.add_doc(gpa, d, @intCast(i + 1), false); + } + + // Yield three, then delete the third -- the anchor itself. + var it = ix.iter(); + var third: Index.Positioned = undefined; + var band_index: u64 = 0; + for (0..3) |i| { + third = it.positioned().?; + if (i > 0) band_index += 1; + } + const anchor_key = try gpa.dupe(u8, third.key); + defer gpa.free(anchor_key); + ix.remove_off(third.off); + + const r = ix.resume_forward(anchor_key, third.off, band_index, third.leaf, third.slot, true); + try testing.expect(!r.capped); + var rest: u32 = 0; + var walk = r.it; + while (walk.next()) |_| rest += 1; + + // 50 inserted, 1 deleted, 2 already returned before the anchor: 47 left. + try testing.expectEqual(@as(u32, 47), rest); +} diff --git a/src/lib.zig b/src/lib.zig index 406c10e..ff69beb 100644 --- a/src/lib.zig +++ b/src/lib.zig @@ -10,6 +10,7 @@ pub const db = @import("db.zig"); pub const query = @import("query.zig"); pub const update = @import("update.zig"); pub const index = @import("index.zig"); +pub const cursor = @import("cursor.zig"); pub const pager = @import("pager.zig"); test { @@ -23,5 +24,6 @@ test { _ = @import("query.zig"); _ = @import("update.zig"); _ = @import("index.zig"); + _ = @import("cursor.zig"); _ = @import("pager.zig"); } diff --git a/src/main.zig b/src/main.zig index fe57d94..e47c049 100644 --- a/src/main.zig +++ b/src/main.zig @@ -10,6 +10,18 @@ const usage = \\ --db database file (default multiforadb.log) \\ --ttl-sweep-secs \\ seconds between TTL index sweeps (default 60, 0 disables) + \\ --cursor-timeout-ms + \\ idle milliseconds before an open cursor is reaped + \\ (default 600000, matching MongoDB's cursorTimeoutMillis; + \\ 0 disables expiry) + \\ --cursor-sweep-secs + \\ seconds between idle-cursor sweeps (default 4, matching + \\ MongoDB's clientCursorMonitorFrequencySecs; 0 disables) + \\ --max-open-cursors + \\ cursor registry capacity (default 4096). At capacity the + \\ least recently used cursor is evicted, and its client + \\ sees CursorNotFound -- the same answer an idle timeout + \\ gives, which every driver already handles. \\ --compact-threshold \\ minimum log bytes between compactions; suffixes k/m/g \\ (default 16m). The actual trigger also scales with the @@ -44,34 +56,77 @@ fn parse_size_suffix(v: []const u8) ?u64 { return n * mult; } -pub fn main(init: std.process.Init) !void { - var port: u16 = 27017; - var bind_ip: []const u8 = "127.0.0.1"; - var db_path: []const u8 = "multiforadb.log"; - var ttl_sweep_secs: i64 = 60; - var compact_threshold: u64 = 16 * 1024 * 1024; +/// Everything the CLI can set. Parsed apart from `main` so the option table has +/// room to grow without main outgrowing the 70-line limit. +const Options = struct { + port: u16 = 27017, + bind_ip: []const u8 = "127.0.0.1", + db_path: []const u8 = "multiforadb.log", + ttl_sweep_secs: i64 = 60, + compact_threshold: u64 = 16 * 1024 * 1024, + cursor_timeout_ms: i64 = mongo.cursor.default_idle_timeout_ms, + cursor_sweep_secs: i64 = 4, + max_open_cursors: u32 = mongo.cursor.default_capacity, + /// Set when --help was given: print usage and exit without opening anything. + help: bool = false, +}; +pub fn main(init: std.process.Init) !void { + const opts = try parse_args(init) orelse { + try std.Io.File.writeStreamingAll(.stdout(), init.io, usage); + return; + }; + + const oid_gen = mongo.bson.ObjectIdGen.init(init.io); + var engine = try mongo.db.Engine.open(init.gpa, init.io, opts.db_path); + defer engine.deinit(); + engine.compact_threshold = opts.compact_threshold; + // The engine builds its registry with the defaults so an embedded caller + // needs no configuration; the CLI replaces it when asked for something else. + try engine.reconfigure_cursors(opts.max_open_cursors, opts.cursor_timeout_ms); + std.debug.print("multiforadb: opened database '{s}' (compact threshold {d})\n", .{ + opts.db_path, + opts.compact_threshold, + }); + + var server = mongo.server.Server{ + .gpa = init.gpa, + .port = opts.port, + .bind_ip = opts.bind_ip, + .oid_gen = oid_gen, + .connection_counter = .init(1), + .engine = &engine, + .start_time = std.Io.Timestamp.now(init.io, .real), + .ttl_sweep_secs = opts.ttl_sweep_secs, + .cursor_sweep_secs = opts.cursor_sweep_secs, + }; + try server.run(); +} + +/// Null means --help: the caller prints usage and exits. +fn parse_args(init: std.process.Init) !?Options { + var o = Options{}; var it = std.process.Args.Iterator.init(init.minimal.args); defer it.deinit(); _ = it.next(); // program name while (it.next()) |arg| { if (std.mem.eql(u8, arg, "--port")) { const v = it.next() orelse return error.MissingValue; - port = std.fmt.parseInt(u16, v, 10) catch { + o.port = std.fmt.parseInt(u16, v, 10) catch { std.debug.print("multiforadb: invalid port '{s}'\n", .{v}); return error.InvalidPort; }; } else if (std.mem.eql(u8, arg, "--bind")) { - bind_ip = it.next() orelse return error.MissingValue; + o.bind_ip = it.next() orelse return error.MissingValue; } else if (std.mem.eql(u8, arg, "--db")) { - db_path = it.next() orelse return error.MissingValue; + o.db_path = it.next() orelse return error.MissingValue; } else if (std.mem.eql(u8, arg, "--ttl-sweep-secs")) { const v = it.next() orelse return error.MissingValue; - // i64 is the width std.Io.Duration.fromSeconds takes, so the - // value reaches the sweeper without a cast; negatives are the - // only thing parseInt would otherwise let through. - ttl_sweep_secs = std.fmt.parseInt(i64, v, 10) catch -1; - if (ttl_sweep_secs < 0) { + // i64 is the width std.Io.Duration.fromSeconds takes, so the value + // reaches the sweeper without a cast; negatives are the only thing + // parseInt would otherwise let through. + o.ttl_sweep_secs = std.fmt.parseInt(i64, v, 10) catch -1; + if (o.ttl_sweep_secs < 0) { std.debug.print("multiforadb: invalid ttl sweep interval '{s}'\n", .{v}); return error.InvalidTtlSweepSecs; } @@ -85,34 +140,45 @@ pub fn main(init: std.process.Init) !void { std.debug.print("multiforadb: compact threshold must be at least 1m\n", .{}); return error.InvalidCompactThreshold; } - compact_threshold = parsed; + o.compact_threshold = parsed; } else if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) { - try std.Io.File.writeStreamingAll(.stdout(), init.io, usage); - return; + return null; + } else if (try parse_cursor_flag(arg, &it, &o)) { + // Handled: one of the cursor-registry flags. } else { std.debug.print("multiforadb: unknown option '{s}'\n{s}", .{ arg, usage }); return error.UnknownOption; } } - - const oid_gen = mongo.bson.ObjectIdGen.init(init.io); - var engine = try mongo.db.Engine.open(init.gpa, init.io, db_path); - defer engine.deinit(); - engine.compact_threshold = compact_threshold; - std.debug.print("multiforadb: opened database '{s}' (compact threshold {d})\n", .{ - db_path, - compact_threshold, - }); - - var server = mongo.server.Server{ - .gpa = init.gpa, - .port = port, - .bind_ip = bind_ip, - .oid_gen = oid_gen, - .connection_counter = .init(1), - .engine = &engine, - .start_time = std.Io.Timestamp.now(init.io, .real), - .ttl_sweep_secs = ttl_sweep_secs, - }; - try server.run(); + return o; +} + +/// The cursor-registry flags, grouped so `parse_args` stays one flat table of +/// options. Returns whether `arg` was one of them; consumes its value if so. +fn parse_cursor_flag(arg: []const u8, it: *std.process.Args.Iterator, o: *Options) !bool { + if (std.mem.eql(u8, arg, "--cursor-timeout-ms")) { + const v = it.next() orelse return error.MissingValue; + o.cursor_timeout_ms = std.fmt.parseInt(i64, v, 10) catch -1; + if (o.cursor_timeout_ms < 0) { + std.debug.print("multiforadb: invalid cursor timeout '{s}'\n", .{v}); + return error.InvalidCursorTimeout; + } + } else if (std.mem.eql(u8, arg, "--cursor-sweep-secs")) { + const v = it.next() orelse return error.MissingValue; + o.cursor_sweep_secs = std.fmt.parseInt(i64, v, 10) catch -1; + if (o.cursor_sweep_secs < 0) { + std.debug.print("multiforadb: invalid cursor sweep interval '{s}'\n", .{v}); + return error.InvalidCursorSweepSecs; + } + } else if (std.mem.eql(u8, arg, "--max-open-cursors")) { + const v = it.next() orelse return error.MissingValue; + o.max_open_cursors = std.fmt.parseInt(u32, v, 10) catch 0; + if (o.max_open_cursors == 0 or o.max_open_cursors > mongo.cursor.max_capacity) { + std.debug.print("multiforadb: invalid max open cursors '{s}'\n", .{v}); + return error.InvalidMaxOpenCursors; + } + } else { + return false; + } + return true; } diff --git a/src/server.zig b/src/server.zig index c5b8065..cf21d9e 100644 --- a/src/server.zig +++ b/src/server.zig @@ -20,6 +20,9 @@ pub const Server = struct { /// because that is what std.Io.Duration.fromSeconds takes — the CLI /// rejects negatives. ttl_sweep_secs: i64, + /// Seconds between idle-cursor sweeps; 0 leaves that monitor unspawned. + /// mongod's own `clientCursorMonitorFrequencySecs` default is 4. + cursor_sweep_secs: i64, pub fn run(self: *Server) !void { // Unbounded async limit: connection handlers otherwise fall back to @@ -43,6 +46,12 @@ pub const Server = struct { // The TTL monitor is just another member of the connection group, so // the `group.cancel` above stops it with everything else. if (self.ttl_sweep_secs > 0) group.async(io, ttl_monitor, .{ io, self }); + // A separate fiber rather than a branch inside ttl_monitor, for two + // reasons: the cadences differ by more than an order of magnitude (4 s + // against 60 s), and a TTL sweep that fails must not stop cursors being + // reclaimed. It is also spawned when TTL sweeping is disabled entirely, + // which is the configuration the spec runner uses. + if (self.cursor_sweep_secs > 0) group.async(io, cursor_monitor, .{ io, self }); while (true) { const stream = listener.accept(io) catch |err| switch (err) { @@ -87,6 +96,20 @@ fn ttl_monitor(io: std.Io, server: *Server) error{Canceled}!void { } } +/// Reap cursors nobody has touched for `cursor_timeout_ms`, until the group is +/// canceled. Takes no engine lock: a cursor owns its own arena, and the store's +/// mutex is a leaf. +fn cursor_monitor(io: std.Io, server: *Server) error{Canceled}!void { + const interval: std.Io.Duration = .fromSeconds(server.cursor_sweep_secs); + while (true) { + // Sleep first, for the same reason the TTL monitor does: at startup + // there is nothing to reap and the listener wants the CPU. + try std.Io.sleep(io, interval, .awake); + const now_ms = std.Io.Timestamp.now(io, .real).toMilliseconds(); + _ = server.engine.cursors.sweep(io, now_ms); + } +} + /// Entry point required by `Group.async`: must return only `error.Canceled`. fn handle_connection(io: std.Io, stream: std.Io.net.Stream, server: *Server) error{Canceled}!void { handle_connection_inner(io, stream, server) catch {}; diff --git a/src/wire.zig b/src/wire.zig index 0173340..df6cc9d 100644 --- a/src/wire.zig +++ b/src/wire.zig @@ -299,9 +299,20 @@ fn begin_message( } /// Patch in the total length of the message started at `len_pos`. +/// +/// The bound is `max_message_size`, the same 48 MiB we advertise to drivers as +/// `maxMessageSizeBytes`, not `maxInt(u32)`. A reply past what we told the +/// client to expect is not a large reply, it is a desynchronized connection: +/// the driver reads the length, refuses or mis-frames it, and every later +/// command on that socket reads the wrong bytes. Failing here turns that into +/// one honest error on the request that caused it. +/// +/// Reachable today: nothing caps how many documents a `find` puts in its single +/// batch, so ~3000 documents of 16 KiB clears 48 MB. The cursor batch budget +/// makes it unreachable, which is the point of keeping this as the backstop. fn end_message(out: *std.ArrayListUnmanaged(u8), len_pos: usize) !void { const total = out.items.len - len_pos; - if (total > std.math.maxInt(u32)) return error.MessageTooLarge; + if (total > max_message_size) return error.MessageTooLarge; std.mem.writeInt(u32, out.items[len_pos..][0..4], @intCast(total), .little); } @@ -411,3 +422,31 @@ test "parse OP_QUERY handshake" { try testing.expectEqual(@as(i32, op_code_query), msg.op_code); try testing.expectEqualStrings("isMaster", msg.command_name()); } + +test "a reply past the advertised message size fails to build" { + // The guard exists because exceeding it desynchronizes the connection + // rather than merely making one reply large, so it must be an error return + // and not a truncation. One oversized string is the cheapest way past it + // without allocating 48 MB of documents. + const gpa = testing.allocator; + var reply = Reply.init(gpa); + defer reply.deinit(); + + const big = try reply.arena_alloc().alloc(u8, max_message_size + 1); + @memset(big, 'x'); + try reply.put_ok(); + try reply.put("payload", .{ .string = big }); + + var out: std.ArrayListUnmanaged(u8) = .empty; + defer out.deinit(gpa); + try testing.expectError(error.MessageTooLarge, reply.build(gpa, 1, 1, &out)); + + // And a reply comfortably inside the bound still builds, so the guard is + // not simply rejecting everything. + var small = Reply.init(gpa); + defer small.deinit(); + try small.put_ok(); + out.clearRetainingCapacity(); + try small.build(gpa, 1, 1, &out); + try testing.expect(out.items.len < max_message_size); +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 4a1db62..1e04d19 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -41,6 +41,19 @@ node tests/e2e/e2e6.js # 73 checks, ~15 s, needs no running server E2E6_PORT=27300 node tests/e2e/e2e6.js # different port if 27220 is taken ``` +`e2e7.js` is the cursor suite, self-contained for a different reason: cursor +behaviour is only observable with non-default flags. It spawns three servers in +turn -- default flags for batching/streaming/aggregate, then +`--cursor-timeout-ms 800 --cursor-sweep-secs 1 --max-open-cursors 4` for idle +expiry and registry capacity, then a restart on the same database to confirm a +cursor does not survive one. Most of it uses raw `runCommand`, because the +driver hides `cursor.id` and that is the thing under test: + +```sh +node tests/e2e/e2e7.js # 86 checks, needs no running server +E2E7_PORT=27310 node tests/e2e/e2e7.js # different port if 27230 is taken +``` + Rebuild with `zig build` after any change under `src/` before restarting the server: `zig build test` compiles the test binary only and leaves `zig-out/bin/multiforadb` stale, so the suites keep running against the old diff --git a/tests/e2e/e2e7.js b/tests/e2e/e2e7.js new file mode 100644 index 0000000..69b4293 --- /dev/null +++ b/tests/e2e/e2e7.js @@ -0,0 +1,474 @@ +// E2E part 7: server-side cursors, self-contained. +// +// Spawns its own multiforadb servers, because cursor behaviour is only +// observable with non-default flags (a short idle timeout, a tiny registry) and +// with raw `runCommand` — the driver hides `cursor.id`, which is the thing under +// test. +// +// node tests/e2e/e2e7.js +// +// Env: E2E7_PORT listen port (default 27230) +// MFDB_BIN server binary (default ../../zig-out/bin/multiforadb) +// E2E7_KEEP keep the log files after the run +// +// The one rule most of this file is about: **never look ahead.** A batch ends +// either because it reached its target — cursor stays open — or because the +// source reported EOF, and only then does the cursor close with `id: 0`. So four +// documents at `batchSize: 2` require a third command answering an empty +// `nextBatch` with `id: 0`. That empty terminal batch is correct, and it is what +// real mongod does (measured, not assumed — see checks 5 and 6). +const { MongoClient, Long } = require('mongodb'); +const { spawn } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +const PORT = Number(process.env.E2E7_PORT || 27230); +const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb'); +const DBFILE = path.resolve(__dirname, '../../.zig-cache/e2e7-cursors.log'); +const URL = `mongodb://127.0.0.1:${PORT}`; + +const results = []; +function check(name, cond, detail = '') { + results.push({ name, ok: !!cond, detail: String(detail) }); + if (!cond) console.error(` x ${name} ${detail}`); +} +function eq(name, got, want) { + const ok = JSON.stringify(got) === JSON.stringify(want); + check(name, ok, ok ? '' : `got ${JSON.stringify(got)} want ${JSON.stringify(want)}`); +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/// The error code a command produces, or 0 when it succeeds. +async function codeOf(fn) { + try { + await fn(); + return 0; + } catch (e) { + return e.code === undefined ? -1 : e.code; + } +} + +let server = null; +let serverLog = ''; +let serverDead = false; + +function cleanup() { + if (server && !serverDead) { + try { server.kill('SIGKILL'); } catch {} + } +} +process.on('exit', cleanup); +process.on('SIGINT', () => { cleanup(); process.exit(130); }); +process.on('SIGTERM', () => { cleanup(); process.exit(143); }); + +function startServer(args, fresh = true) { + return new Promise((resolve, reject) => { + if (fresh) { + fs.rmSync(DBFILE, { force: true }); + fs.rmSync(DBFILE + '.data', { force: true }); + } + serverDead = false; + serverLog = ''; + server = spawn(BIN, ['--port', String(PORT), '--db', DBFILE, ...args], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + server.stdout.on('data', (d) => (serverLog += d)); + server.stderr.on('data', (d) => (serverLog += d)); + server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`))); + server.on('exit', (code, sig) => { + serverDead = true; + if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`; + }); + const deadline = Date.now() + 15000; + (async () => { + while (Date.now() < deadline) { + if (serverDead) { + reject(new Error(`server exited during start (port ${PORT} busy?)\n${serverLog}`)); + return; + } + const c = new MongoClient(URL, { serverSelectionTimeoutMS: 1000 }); + try { + await c.connect(); + await c.db('admin').command({ ping: 1 }); + await c.close(); + return resolve(); + } catch { + try { await c.close(); } catch {} + await sleep(100); + } + } + reject(new Error(`server did not come up on :${PORT}\n${serverLog}`)); + })(); + }); +} + +async function stopServer(sig = 'SIGTERM') { + if (!server) return; + const exited = new Promise((r) => server.once('exit', r)); + server.kill(sig); + await Promise.race([exited, sleep(5000)]); + serverDead = true; + server = null; +} + +// --------------------------------------------------------------------------- +// Phase A — batching, lifecycle and errors, on default flags +// --------------------------------------------------------------------------- + +async function phaseA(db) { + const col = db.collection('c'); + await col.deleteMany({}); + await col.insertMany([...Array(250)].map((_, i) => ({ _id: i + 1, x: i }))); + + // 1. A cursor is a real cursor: nonzero id, a namespace with both parts. + let r = await db.command({ find: 'c', filter: {}, batchSize: 2 }); + eq('1 batchSize 2 returns 2', r.cursor.firstBatch.length, 2); + check('1 cursor id is nonzero', r.cursor.id > 0, r.cursor.id); + eq('1 ns is db.coll', r.cursor.ns, 'e2e7.c'); + + // 2. The default first batch is 101, MongoDB's own + // internalQueryFindCommandBatchSize. + eq('2 default first batch is 101', (await db.command({ find: 'c', filter: {} })).cursor.firstBatch.length, 101); + + // 3. A getMore naming no batchSize is bounded by bytes, not by the batchSize + // the cursor was created with. Measured against mongod 8.3.7: find with + // batchSize 2 then a bare getMore returns 4998 of 5000 documents. + const id3 = (await db.command({ find: 'c', filter: {}, batchSize: 2 })).cursor.id; + let g = await db.command({ getMore: id3, collection: 'c' }); + eq('3 a bare getMore drains the rest', g.cursor.nextBatch.length, 248); + eq('3 and closes at EOF', String(g.cursor.id), '0'); + + // 4. A getMore's batchSize applies to that batch only. + const id4 = (await db.command({ find: 'c', filter: {}, batchSize: 2 })).cursor.id; + g = await db.command({ getMore: id4, collection: 'c', batchSize: 3 }); + eq('4 getMore batchSize 3', g.cursor.nextBatch.map((d) => d._id), [3, 4, 5]); + check('4 still open', g.cursor.id > 0); + + // 5. No look-ahead: 4 documents at batchSize 2 needs a third command whose + // nextBatch is empty. Closing on "batch full and source dry" would break + // the command counts the pinned spec suites assert. + const four = db.collection('four'); + await four.deleteMany({}); + await four.insertMany([1, 2, 3, 4].map((i) => ({ _id: i }))); + r = await db.command({ find: 'four', filter: {}, batchSize: 2 }); + g = await db.command({ getMore: r.cursor.id, collection: 'four', batchSize: 2 }); + eq('5 second full batch is 2 documents', g.cursor.nextBatch.length, 2); + check('5 and leaves the cursor open', g.cursor.id > 0); + g = await db.command({ getMore: g.cursor.id, collection: 'four', batchSize: 2 }); + eq('5 terminal batch is empty and closed', [g.cursor.nextBatch.length, String(g.cursor.id)], [0, '0']); + + // 6. limit is an EOF source, so the batch that exhausts it also closes the + // cursor. This is why the driver sends batchSize = limit + 1. + r = await db.command({ find: 'c', filter: {}, sort: { _id: 1 }, limit: 4, batchSize: 5 }); + eq('6 limit 4 batchSize 5 closes in one reply', String(r.cursor.id), '0'); + eq('6 and returns exactly the limit', r.cursor.firstBatch.map((d) => d._id), [1, 2, 3, 4]); + r = await db.command({ find: 'c', filter: {}, sort: { _id: 1 }, limit: 4, batchSize: 2 }); + check('6 limit 4 batchSize 2 stays open', r.cursor.id > 0); + g = await db.command({ getMore: r.cursor.id, collection: 'c', batchSize: 2 }); + eq('6 the batch reaching the limit closes', String(g.cursor.id), '0'); + eq('6 across batches, limit still honoured', g.cursor.nextBatch.map((d) => d._id), [3, 4]); + + // 7. skip is consumed once, at creation, and never re-applied. + r = await db.command({ find: 'c', filter: {}, skip: 20, batchSize: 3 }); + eq('7 skip 20 starts at 21', r.cursor.firstBatch.map((d) => d._id), [21, 22, 23]); + g = await db.command({ getMore: r.cursor.id, collection: 'c', batchSize: 3 }); + eq('7 skip not re-applied on getMore', g.cursor.nextBatch.map((d) => d._id), [24, 25, 26]); + + // 8. batchSize 0 is a real request for an empty batch with a live cursor, not + // "unbounded". Drivers use it to obtain a cursor cheaply. Nothing may be + // consumed. + r = await db.command({ find: 'c', filter: {}, batchSize: 0 }); + eq('8 batchSize 0 returns nothing', r.cursor.firstBatch.length, 0); + check('8 but a live cursor', r.cursor.id > 0); + g = await db.command({ getMore: r.cursor.id, collection: 'c', batchSize: 1 }); + eq('8 nothing was consumed', g.cursor.nextBatch.map((d) => d._id), [1]); + + // 9. singleBatch, and its wire-legacy form, a negative limit. + eq('9 singleBatch closes', String((await db.command({ find: 'c', filter: {}, batchSize: 2, singleBatch: true })).cursor.id), '0'); + r = await db.command({ find: 'c', filter: {}, limit: -3 }); + eq('9 negative limit is one batch', [r.cursor.firstBatch.length, String(r.cursor.id)], [3, '0']); + + // 10. A cursor is not pinned to the connection that created it: the driver + // spec allows a getMore from any connection to the same server. + const other = new MongoClient(URL); + await other.connect(); + const shared = (await db.command({ find: 'c', filter: {}, batchSize: 2 })).cursor.id; + eq('10 getMore from another connection', await codeOf(() => other.db('e2e7').command({ getMore: shared, collection: 'c', batchSize: 2 })), 0); + await other.close(); + + // 11. A getMore naming the wrong collection is Unauthorized (13), not 43, and + // leaves the cursor alive — the request is wrong, not the cursor. + // Measured against mongod, which answers exactly this code. + const live = (await db.command({ find: 'c', filter: {}, batchSize: 2 })).cursor.id; + eq('11 wrong collection is Unauthorized 13', await codeOf(() => db.command({ getMore: live, collection: 'four' })), 13); + eq('11 the cursor survived it', (await db.command({ getMore: live, collection: 'c', batchSize: 1 })).cursor.nextBatch.length, 1); + + // 12. killCursors: all four arrays, and the right partitioning. + let k = await db.command({ killCursors: 'c', cursors: [live] }); + eq('12 a live cursor is killed', k.cursorsKilled.map(String), [String(live)]); + check('12 all four arrays present', ['cursorsKilled', 'cursorsNotFound', 'cursorsAlive', 'cursorsUnknown'].every((f) => Array.isArray(k[f])), Object.keys(k).join(',')); + k = await db.command({ killCursors: 'c', cursors: [live] }); + eq('12 killing it twice reports notFound', k.cursorsNotFound.map(String), [String(live)]); + const other_ns = (await db.command({ find: 'four', filter: {}, batchSize: 1 })).cursor.id; + k = await db.command({ killCursors: 'c', cursors: [other_ns] }); + eq('12 a wrong-namespace id reports notFound', k.cursorsNotFound.map(String), [String(other_ns)]); + eq('12 and that cursor still lives', await codeOf(() => db.command({ getMore: other_ns, collection: 'four', batchSize: 1 })), 0); + + // 13. Malformed and unknown ids. + eq('13 getMore after kill is 43', await codeOf(() => db.command({ getMore: live, collection: 'c' })), 43); + eq('13 id 0 is 43', await codeOf(() => db.command({ getMore: Long.fromNumber(0), collection: 'c' })), 43); + eq('13 an unknown id is 43', await codeOf(() => db.command({ getMore: Long.fromString('987654321'), collection: 'c' })), 43); + eq('13 a non-numeric id is TypeMismatch 14', await codeOf(() => db.command({ getMore: 'nope', collection: 'c' })), 14); + eq('13 a missing collection is BadValue 2', await codeOf(() => db.command({ getMore: Long.fromNumber(1) })), 2); + + // 14. tailable is refused, which is parity: mongod rejects it on a non-capped + // collection and this engine has none. Ignoring it would make a driver's + // tail loop exit, which the application reads as data loss. + eq('14 tailable is BadValue 2', await codeOf(() => db.command({ find: 'c', filter: {}, tailable: true })), 2); + eq('14 awaitData alone is BadValue 2', await codeOf(() => db.command({ find: 'c', filter: {}, awaitData: true })), 2); + + // 15. The driver's own iteration, which is the point of all of the above. + const ids = (await col.find({}).batchSize(7).toArray()).map((d) => d._id); + eq('15 driver drains 250 at batchSize 7', ids.length, 250); + eq('15 no duplicates and no gaps', [new Set(ids).size, Math.min(...ids), Math.max(...ids)], [250, 1, 250]); + eq('15 sort+skip+limit unchanged', (await col.find({ _id: { $gt: 2 } }, { sort: { _id: 1 }, skip: 2, limit: 2 }).toArray()).map((d) => d._id), [5, 6]); + // A sort no index provides must materialize; it still has to drain correctly. + eq('15 an unindexed sort drains in order', (await col.find({}, { sort: { x: -1 } }).batchSize(10).toArray()).map((d) => d.x)[0], 249); +} + +// --------------------------------------------------------------------------- +// Phase B — streaming cursors: resume across writes, and what survives a rebuild +// --------------------------------------------------------------------------- + +async function phaseB(db) { + const col = db.collection('s'); + await col.deleteMany({}); + await col.insertMany([...Array(300)].map((_, i) => ({ _id: i + 1, a: i % 5, pad: 'q'.repeat(200) }))); + + // 16. A whole-index walk holds a key, not a list, so it resumes across writes + // that move documents. The bug this caught: an update rewrites a document + // to a new offset, and resuming by band position returned it twice. + let r = await db.command({ find: 's', filter: {}, batchSize: 10 }); + const seen = new Set(r.cursor.firstBatch.map((d) => d._id)); + let dupes = 0; + let id = r.cursor.id; + let rounds = 0; + let errored = 0; + while (String(id) !== '0' && rounds++ < 200) { + // Churn between every batch: updates rewrite documents, which both moves + // them in the slab and can split leaves. + await col.updateMany({ _id: { $lt: 60 } }, { $inc: { n: 1 } }); + let g; + try { + g = await db.command({ getMore: id, collection: 's', batchSize: 10 }); + } catch (e) { + errored = e.code; + break; + } + for (const d of g.cursor.nextBatch) { + if (seen.has(d._id)) dupes++; + seen.add(d._id); + } + id = g.cursor.id; + } + eq('16 draining across churn did not error', errored, 0); + eq('16 no document came back twice', dupes, 0); + eq('16 every document was returned', seen.size, 300); + check('16 and nothing outside the collection', [...seen].every((v) => v >= 1 && v <= 300)); + + // 17. Both directions stream, over the _id_ index and a secondary one. + eq('17 ascending _id sort drains', (await col.find({}, { sort: { _id: 1 } }).batchSize(9).toArray()).length, 300); + const desc = (await col.find({}, { sort: { _id: -1 } }).batchSize(9).toArray()).map((d) => d._id); + eq('17 descending drains in order', [desc.length, desc[0], desc[299]], [300, 300, 1]); + await col.createIndex({ a: 1 }); + const bya = await col.find({}, { sort: { a: 1 } }).batchSize(11).toArray(); + eq('17 a secondary-index sort drains', bya.length, 300); + check('17 and in the index order', bya.every((d, i) => i === 0 || bya[i - 1].a <= d.a)); + + // 18. Dropping the index a stream is following cannot be resumed — the walk + // has nothing left to walk. That must be a clean error, not garbage. + await col.createIndex({ b: 1 }); + const onb = (await db.command({ find: 's', filter: {}, sort: { b: 1 }, batchSize: 3 })).cursor.id; + await col.dropIndex('b_1'); + eq('18 dropping the streamed index is 175', await codeOf(() => db.command({ getMore: onb, collection: 's', batchSize: 3 })), 175); + + // 19. Dropping the collection kills every kind of cursor. + const doomed = (await db.command({ find: 's', filter: {}, batchSize: 3 })).cursor.id; + await col.drop(); + const dc = await codeOf(() => db.command({ getMore: doomed, collection: 's', batchSize: 3 })); + check('19 dropping the collection kills the cursor', dc === 175 || dc === 43, dc); +} + +// --------------------------------------------------------------------------- +// Phase C — aggregate, the listing commands, and count +// --------------------------------------------------------------------------- + +async function phaseC(db) { + const col = db.collection('g'); + await col.deleteMany({}); + await col.insertMany([...Array(250)].map((_, i) => ({ _id: i + 1, g: i % 40, v: i }))); + + // 20. aggregate batches through cursor.batchSize; a bare cursor is the default. + let r = await db.command({ aggregate: 'g', pipeline: [], cursor: { batchSize: 3 } }); + eq('20 aggregate batchSize 3', r.cursor.firstBatch.length, 3); + check('20 aggregate cursor is real', r.cursor.id > 0); + eq('20 aggregate ns', r.cursor.ns, 'e2e7.g'); + const g20 = await db.command({ getMore: r.cursor.id, collection: 'g', batchSize: 5 }); + eq('20 aggregate getMore continues', g20.cursor.nextBatch.map((d) => d._id), [4, 5, 6, 7, 8]); + eq('20 bare cursor defaults to 101', (await db.command({ aggregate: 'g', pipeline: [], cursor: {} })).cursor.firstBatch.length, 101); + eq('20 driver aggregate drains', (await col.aggregate([], { batchSize: 7 }).toArray()).length, 250); + const groups = await col.aggregate([{ $group: { _id: '$g', n: { $sum: 1 } } }], { batchSize: 6 }).toArray(); + eq('20 $group drains across batches', [groups.length, groups.reduce((a, x) => a + x.n, 0)], [40, 250]); + eq('20 $count stage', await col.aggregate([{ $count: 'total' }]).toArray(), [{ total: 250 }]); + + // 21. listCollections' namespace. It used to be "." with an empty + // collection part, and the driver throws client-side on a namespace like + // that — so the moment the cursor stopped being id 0 it would have broken. + for (let i = 0; i < 12; i++) await db.createCollection('k' + i); + r = await db.command({ listCollections: 1, cursor: { batchSize: 4 } }); + eq('21 listCollections ns has a collection part', r.cursor.ns, 'e2e7.$cmd.listCollections'); + eq('21 listCollections honours batchSize', r.cursor.firstBatch.length, 4); + check('21 listCollections cursor is real', r.cursor.id > 0); + const g21 = await db.command({ getMore: r.cursor.id, collection: '$cmd.listCollections', batchSize: 100 }); + check('21 its getMore works', g21.cursor.nextBatch.length >= 8, g21.cursor.nextBatch.length); + const listed = await db.listCollections({}, { batchSize: 3 }).toArray(); + check('21 driver listCollections drains', listed.length >= 13, listed.length); + + // 22. listIndexes. + await col.createIndex({ v: 1 }); + await col.createIndex({ g: 1 }); + await col.createIndex({ v: -1, g: 1 }); + r = await db.command({ listIndexes: 'g', cursor: { batchSize: 2 } }); + eq('22 listIndexes honours batchSize', r.cursor.firstBatch.length, 2); + eq('22 listIndexes ns', r.cursor.ns, 'e2e7.g'); + eq('22 driver listIndexes drains', (await col.listIndexes({ batchSize: 1 }).toArray()).length, 4); + + // 23. count honoured neither skip nor limit before, which made + // countDocuments(f, {limit}) a silent wrong answer. + eq('23 count plain', (await db.command({ count: 'g' })).n, 250); + eq('23 count limit', (await db.command({ count: 'g', limit: 10 })).n, 10); + eq('23 count skip', (await db.command({ count: 'g', skip: 240 })).n, 10); + eq('23 count skip and limit', (await db.command({ count: 'g', skip: 245, limit: 10 })).n, 5); + eq('23 count skip past the end', (await db.command({ count: 'g', skip: 1000 })).n, 0); + eq('23 count with a query and limit', (await db.command({ count: 'g', query: { g: 0 }, limit: 3 })).n, 3); + eq('23 driver countDocuments limit', await col.countDocuments({}, { limit: 7 }), 7); + + // 24. A batch is capped by bytes as well as by documents, so a large-document + // result splits instead of building a reply past the advertised message + // size. 40 documents of ~1 MiB cannot all fit one 16 MiB batch. + const big = db.collection('big'); + await big.deleteMany({}); + const pad = 'p'.repeat(1024 * 1024 - 64); + for (let i = 0; i < 40; i++) await big.insertOne({ _id: i + 1, pad }); + r = await db.command({ find: 'big', filter: {}, batchSize: 40 }); + check('24 the byte cap split the batch', r.cursor.firstBatch.length >= 1 && r.cursor.firstBatch.length <= 16, r.cursor.firstBatch.length); + check('24 and left the cursor open', r.cursor.id > 0); + eq('24 the whole result still drains', (await big.find({}).batchSize(40).toArray()).length, 40); +} + +// --------------------------------------------------------------------------- +// Phase D — expiry, capacity, and what a restart does +// --------------------------------------------------------------------------- + +async function phaseD(db) { + const col = db.collection('e'); + await col.deleteMany({}); + await col.insertMany([...Array(50)].map((_, i) => ({ _id: i + 1 }))); + + // 25. An idle cursor is reaped; noCursorTimeout exempts one from that. + const perishable = (await db.command({ find: 'e', filter: {}, batchSize: 2 })).cursor.id; + const immortal = (await db.command({ find: 'e', filter: {}, batchSize: 2, noCursorTimeout: true })).cursor.id; + await sleep(300); + eq('25 before the timeout it is alive', await codeOf(() => db.command({ getMore: perishable, collection: 'e', batchSize: 1 })), 0); + await sleep(2500); + eq('25 an idle cursor is reaped', await codeOf(() => db.command({ getMore: perishable, collection: 'e', batchSize: 1 })), 43); + eq('25 noCursorTimeout survives', await codeOf(() => db.command({ getMore: immortal, collection: 'e', batchSize: 1 })), 0); + const k = await db.command({ killCursors: 'e', cursors: [immortal] }); + eq('25 but is still killable', k.cursorsKilled.map(String), [String(immortal)]); + + // 26. A full registry evicts the least recently used cursor rather than + // refusing the new one. The victim sees the same 43 an idle timeout gives, + // which every driver already handles. + const ids = []; + for (let i = 0; i < 5; i++) ids.push((await db.command({ find: 'e', filter: {}, batchSize: 1 })).cursor.id); + eq('26 the oldest was evicted', await codeOf(() => db.command({ getMore: ids[0], collection: 'e', batchSize: 1 })), 43); + const alive = []; + for (const id of ids.slice(1)) alive.push(await codeOf(() => db.command({ getMore: id, collection: 'e', batchSize: 1 }))); + eq('26 the newest four are alive', alive, [0, 0, 0, 0]); +} + +async function phaseE(db, staleId) { + // 27. Cursors do not survive a restart, and a stale id must be a clean 43 — + // not a hang, and not an empty batch claiming the result ended. + eq('27 a cursor from before the restart is 43', await codeOf(() => db.command({ getMore: staleId, collection: 'e', batchSize: 1 })), 43); + const fresh = await db.command({ find: 'e', filter: {}, batchSize: 2 }); + check('27 and new cursors work after a restart', fresh.cursor.id > 0); +} + +async function main() { + // The same guard e2e6.js and big.js carry: without it a missing binary + // surfaces as a generic spawn error instead of saying what to do about it. + if (!fs.existsSync(BIN)) { + console.error(`E2E7_FAIL server binary not found: ${BIN}\n run: zig build`); + process.exit(1); + } + + // Phase A-C on default cursor flags. + await startServer(['--ttl-sweep-secs', '0', '--compact-threshold', '1m'], true); + let client = new MongoClient(URL); + await client.connect(); + let db = client.db('e2e7'); + console.log('phase A: batching, lifecycle, errors'); + await phaseA(db); + console.log('phase B: streaming cursors across writes'); + await phaseB(db); + console.log('phase C: aggregate, listings, count'); + await phaseC(db); + await client.close(); + await stopServer('SIGTERM'); + + // Phase D needs a short timeout and a tiny registry. + console.log('phase D: idle expiry and registry capacity'); + await startServer( + ['--ttl-sweep-secs', '0', '--cursor-timeout-ms', '800', '--cursor-sweep-secs', '1', '--max-open-cursors', '4'], + true, + ); + client = new MongoClient(URL); + await client.connect(); + db = client.db('e2e7'); + await phaseD(db); + const staleId = (await db.command({ find: 'e', filter: {}, batchSize: 1 })).cursor.id; + await client.close(); + await stopServer('SIGTERM'); + + // Phase E: the same database, a new process. + console.log('phase E: a cursor does not survive a restart'); + await startServer(['--ttl-sweep-secs', '0'], false); + client = new MongoClient(URL); + await client.connect(); + await phaseE(client.db('e2e7'), staleId); + await client.close(); + + if (process.env.E2E7_KEEP !== '1') { + fs.rmSync(DBFILE, { force: true }); + fs.rmSync(DBFILE + '.data', { force: true }); + } + await stopServer('SIGTERM'); + + const failed = results.filter((r) => !r.ok); + console.log(`\n${results.length - failed.length}/${results.length} checks passed`); + if (failed.length) { + console.log('FAILED:', failed.map((f) => f.name).join(', ')); + console.log('--- server log tail ---'); + console.log(serverLog.split('\n').slice(-30).join('\n')); + process.exit(1); + } + console.log('E2E7_OK'); +} + +main().catch((e) => { + console.error('E2E7_FAIL', e); + console.log('--- server log tail ---'); + console.log(serverLog.split('\n').slice(-40).join('\n')); + process.exit(1); +}); diff --git a/tests/spec/run.js b/tests/spec/run.js index 831dbe3..fda126b 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -314,11 +314,38 @@ class Unsupported extends Error {} // Argument keys the spec passes positionally rather than as driver options. const POSITIONAL = new Set(['filter', 'document', 'documents', 'update', 'replacement', 'pipeline', 'fieldName', 'models', 'requests', 'keys', 'name', 'indexes', 'command', 'session', 'entity', 'to']); +// Driver options the driver only honours as a JavaScript number. The suites are +// parsed with `EJSON.parse(text, {relaxed: false})` so that `$numberLong` and +// friends keep their exact BSON type in *data* -- but that also turns a plain +// JSON `2` in an *option* into a BSON Int32 object, and the driver gates every +// one of these on `typeof options.skip === 'number'` +// (node_modules/mongodb/lib/operations/find.js:68-95). A BSON wrapper therefore +// failed the check and the option was dropped on the floor: `skip`, `limit` and +// `batchSize` never reached the wire at all, and three find.json cases failed +// with the *unclipped* match count while the engine was applying both correctly. +// Read as an engine bug for a whole milestone. Coerce by name, not by shape: +// unwrapping every numeric-looking value would rewrite the wire type of the +// `comment` and `hint` values that other suites assert on. +const NUMERIC_OPTIONS = new Set([ + 'skip', + 'limit', + 'batchSize', + 'maxTimeMS', + 'maxAwaitTimeMS', + 'expireAfterSeconds', +]); + +function numeric_option(v) { + if (v === null || typeof v !== 'object' || typeof v.valueOf !== 'function') return v; + const n = v.valueOf(); + return typeof n === 'number' ? n : v; +} + function options(args, drop = []) { const o = {}; for (const [k, v] of Object.entries(args || {})) { if (POSITIONAL.has(k) || drop.includes(k)) continue; - o[k] = v; + o[k] = NUMERIC_OPTIONS.has(k) ? numeric_option(v) : v; } return Object.keys(o).length ? o : undefined; } diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index a182ab2..a622491 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -13,7 +13,7 @@ # semantics; ignoring them makes some cases pass that a full runner would # fail, so treat `pass` as an upper bound until M1 wires events up. -total 163 pass 129 fail 195 skip 175 files 0 errored +total 168 pass 124 fail 195 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -78,7 +78,7 @@ client-bulkWrite-updateOne-sort.json 0 0 1 count-collation.json 1 0 1 count-empty.json 2 0 1 count-rawdata.json 0 0 2 -count.json 3 1 3 +count.json 4 0 3 countDocuments-comment.json 2 0 1 countDocuments-rawdata.json 1 0 1 create-null-ids.json 0 6 1 @@ -116,8 +116,8 @@ find-collation.json 0 1 0 find-comment.json 1 2 2 find-let.json 0 1 1 find-rawdata.json 1 0 1 -find.json 2 3 0 -findOne.json 1 1 0 +find.json 5 0 0 +findOne.json 2 0 0 findOneAndDelete-collation.json 0 1 0 findOneAndDelete-comment.json 2 0 1 findOneAndDelete-hint-serverError.json 0 0 2 @@ -292,7 +292,6 @@ count-collation.json SKIP Deprecated count with collation runner: operation coun count-empty.json SKIP Deprecated count with empty collection runner: operation count count-rawdata.json SKIP Deprecated count with rawData option needs server >= 8.2.0 count-rawdata.json SKIP Deprecated count with rawData option on less than 8.2.0 - ignore argument runner: operation count -count.json FAIL Count documents with skip and limit countDocuments: expected 2, got 3 count.json SKIP Deprecated count without a filter runner: operation count count.json SKIP Deprecated count with a filter runner: operation count count.json SKIP Deprecated count with skip and limit runner: operation count @@ -354,10 +353,6 @@ find-comment.json SKIP find with comment does not set comment on getMore - pre 4 find-let.json SKIP Find with let option needs server >= 5.0 find-let.json FAIL Find with let option unsupported (server-side error) find: expected an error, the operation succeeded find-rawdata.json SKIP Find with rawData option needs server >= 8.2.0 -find.json FAIL Find with filter, sort, skip, and limit find: expected 2 elements, got 4 -find.json FAIL Find with limit, sort, and batchsize find: expected 4 elements, got 6 -find.json FAIL Find with batchSize equal to limit find: expected 4 elements, got 5 -findOne.json FAIL FindOne with filter, sort, and skip findOne._id: expected 5, got 3 findOneAndDelete-collation.json FAIL FindOneAndDelete when one document matches with collation findOneAndDelete: expected a document, got null findOneAndDelete-comment.json SKIP findOneAndDelete with comment - pre 4.4 needs server <= 4.2.99 findOneAndDelete-hint-serverError.json SKIP * needs server <= 4.3.3 -- 2.39.5 From 5c3a759429502a8c797aa09f1bdb5b34968a18b7 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 10:34:34 +0300 Subject: [PATCH 03/37] pager: the persisted free list survives allocating its own pages `write_freelist` captured the entry count, sized the buffer from it, and only then called `alloc_pages` for the pages it was about to write into. That allocation goes through `take_free` like any other, and on an exact fit `take_free` removes the entry it took. The header then claimed one entry more than the loop wrote, the hash landed eight bytes short of where `read_freelist` looks for it, and the next open printed "data file free list is corrupt" and dropped the whole list -- every page on it staying in use forever. This is the ordinary case, not a corner. The stream is one page whenever the list is smaller than 511 entries, and a one-page run is the commonest thing on the list because copy-on-write returns thousands of them per generation. So the free list was being discarded at essentially every reopen that had anything to discard, which is the same symptom class as the reclamation bugs the M0 churn gate found: the mechanism works once and never twice. The existing two-generation test misses it because its free run is two pages and the stream asks for one -- shrinking an entry leaves the count right, only removing one does not. Fixed by sizing from an upper bound and counting the entries actually written. `take_free` never adds an entry, so one allocation is enough and the bound holds; an assertion pins that the list shrank by at most the one entry the allocation could have taken. Found while designing the M1 document free list, which multiplies the traffic through this path. Verified: `zig build test` 159/159 in ReleaseFast and ReleaseSafe, `zig build fuzz`, e2e 49, e2e2 concurrent 2 + crash pair 1/3, e2e3 16, e2e4 17, e2e6 72, e2e7 86, and `crash-fuzz.js` 60 cycles with the prefix invariant holding. Mutation-checked per the repo's second ground rule: restoring the count-before-allocate ordering turns the new test red with the corruption warning. --- src/pager.zig | 73 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/src/pager.zig b/src/pager.zig index 0dc1036..6104b56 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -998,13 +998,23 @@ pub const Pager = struct { } fn write_freelist(self: *Pager) !struct { first: u32, len: u64 } { - const count = self.free_ready.items.len + self.free_hold.items.len + self.free_pending.items.len; - const len: u64 = 8 + @as(u64, count) * 8 + 8; - const pages: u32 = @intCast((len + page_size - 1) / page_size); + // Size from an upper bound, then count the entries actually written. + // + // The allocation below takes its pages off this very list, and an exact + // fit removes the entry it took (`take_free`). A count captured + // beforehand therefore claims one entry more than the loop writes: the + // hash lands eight bytes short of where `read_freelist` looks for it and + // the whole list is dropped as corrupt on the next open. The stream is + // one page and a one-page run is the commonest thing on the list, so + // that is the ordinary case rather than a corner. + // + // `take_free` never *adds* an entry, so the bound holds and one + // allocation is enough. + const bound = self.free_ready.items.len + self.free_hold.items.len + self.free_pending.items.len; + const pages: u32 = @intCast((8 + bound * 8 + 8 + page_size - 1) / page_size); const first = try self.alloc_pages(pages); - const buf = self.bytes_mut(@as(u64, first) << page_shift, @intCast(len)); + const buf = self.bytes_mut(@as(u64, first) << page_shift, @as(usize, pages) << page_shift); @memset(buf, 0); - std.mem.writeInt(u64, buf[0..8], count, .little); var at: usize = 8; for ([_][]const Extent{ self.free_ready.items, @@ -1017,8 +1027,14 @@ pub const Pager = struct { at += 8; } } + const count = (at - 8) / 8; + assert_msg( + count == bound or count + 1 == bound, + "the free list changed size while it was being written", + ); + std.mem.writeInt(u64, buf[0..8], count, .little); std.mem.writeInt(u64, buf[at..][0..8], header_hash(buf[0..at]), .little); - return .{ .first = first, .len = len }; + return .{ .first = first, .len = @as(u64, at) + 8 }; } /// Load the free list a watermark points at. A damaged one is dropped with a @@ -1536,6 +1552,51 @@ test "freed pages are withheld for two generations and survive a reopen" { try testing.expectEqual(@as(u32, 2), again.free_ready_pages()); } +test "the persisted free list survives allocating its own pages" { + // `write_freelist` allocates the pages it is about to write into, and that + // allocation goes through `take_free` like any other. On an exact fit the + // entry is removed, so a count captured beforehand describes one entry more + // than the loop writes, the hash lands short of where the reader looks, and + // the whole list is dropped as corrupt on the next open. + // + // The stream is one page and a one-page run is the commonest thing on the + // list, so this is the normal case, not a corner. The two-generation test + // above misses it because its free run is two pages and the stream asks for + // one: shrinking an entry keeps the count right, only removing it does not. + // + // Mutation check: compute `count` before `alloc_pages` again and the reopen + // assertion goes red with "data file free list is corrupt". + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + + // Five pages, of which three go back one at a time with a gap between each + // -- adjacent runs would be coalesced back into one and the list would be + // too short for a lost entry to show. + const base = try tp.pg().alloc_pages(5); + try tp.pg().publish(.{ .seq = 1 }); + try tp.pg().free_pages(base, 1); + try tp.pg().free_pages(base + 2, 1); + try tp.pg().free_pages(base + 4, 1); + + try tp.pg().publish(.{ .seq = 2 }); // pending -> hold + try tp.pg().publish(.{ .seq = 3 }); // hold -> ready + try testing.expectEqual(@as(u32, 3), tp.pg().free_ready_pages()); + + // This is the publish that trips it: the free list is now non-empty and + // holds a run of exactly the one page the stream needs. + try tp.pg().publish(.{ .seq = 4 }); + const ready_before = tp.pg().free_ready_pages(); + try testing.expectEqual(@as(u32, 2), ready_before); + tp.close(); + + var again = try reopen(io, tp.path); + defer again.deinit(); + try testing.expectEqual(ready_before, again.free_ready_pages()); +} + test "the watermark is never published before the pages it describes" { // The load-bearing ordering of the whole design: every page a watermark // describes is durable before the watermark that describes it. Reverse them -- 2.39.5 From f5471f73fc353f4fe2440f1faed738f4327de267 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 10:44:39 +0300 Subject: [PATCH 04/37] pager: the free lists are read and written under the allocation lock `free_pages` appended to `free_pending` with no lock at all, and `write_freelist` walked all three lists the same way -- while `publish` rotated them under `alloc_lock`. Both are reachable concurrently in production: the hot caller of `free_pages` is `page_mut_cow`, which runs under a *collection* lock, and a checkpoint holds only the shared catalog lock, so copy-on-write in one collection races a checkpoint and a second collection's copy-on-write freely. The append race loses or duplicates entries. The read race is worse: an append that reallocates leaves `write_freelist`'s loop walking freed memory, and it is walking it to decide which pages are safe to hand out again. Both now take `alloc_lock`. `write_freelist` holds it across reading the lists *and* allocating the pages it writes them into, which the mutex being non-reentrant makes awkward, so `reserve_pages` and `alloc_pages_assume_reserved` grow `_locked` bodies and thin locking wrappers. A free that lands while the stream is being written simply waits for the next generation's list -- the page stays allocated one generation longer, which is the safe direction. The read race is what the new test actually caught: written to assert only the append side, it tripped the size assertion added in the previous commit on its first run, because a concurrent free had grown the list between the bound and the loop. That is a bug no reading of `free_pages` alone would have found. The test asserts page *identity* rather than a total, because a publish allocates its stream off this very list and a plain count is short by however many publishes found a fit. Every page left on the lists must be one a freer put there, exactly once. Probabilistic, as any test of a data race is -- it is evidence only when red. Mutation-checked per the repo's second ground rule: dropping the lock from `free_pages` crashes it in roughly two runs out of three; five consecutive runs with the lock in place are green. Verified: `zig build test` 160/160 in ReleaseFast and ReleaseSafe, e2e 49, e2e2 concurrent 2 + crash pair 1/3, e2e3 16, e2e4 17, e2e6 72, e2e7 86, `crash-fuzz.js` 60 cycles. --- src/pager.zig | 112 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/src/pager.zig b/src/pager.zig index 6104b56..8521555 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -557,6 +557,13 @@ pub const Pager = struct { pub fn reserve_pages(self: *Pager, hold: *Reservation, n: u32) !void { self.alloc_lock.lockUncancelable(self.io); defer self.alloc_lock.unlock(self.io); + return self.reserve_pages_locked(hold, n); + } + + /// For callers already holding `alloc_lock`. The lock is not reentrant, so + /// the split is what lets `write_freelist` hold it across reading the lists + /// *and* allocating the pages it writes them into. + fn reserve_pages_locked(self: *Pager, hold: *Reservation, n: u32) !void { // Additive: room for every promise outstanding anywhere *plus* this one. // Two consumers reserving before the same log append must both be able to // rely on their promise. @@ -646,9 +653,14 @@ pub const Pager = struct { } pub fn alloc_pages_assume_reserved(self: *Pager, hold: *Reservation, n: u32) u32 { - assert(n > 0); self.alloc_lock.lockUncancelable(self.io); defer self.alloc_lock.unlock(self.io); + return self.alloc_assume_reserved_locked(hold, n); + } + + /// For callers already holding `alloc_lock`; see `reserve_pages_locked`. + fn alloc_assume_reserved_locked(self: *Pager, hold: *Reservation, n: u32) u32 { + assert(n > 0); assert_msg( n <= hold.pages, "page allocation overran reserve_pages' promise", @@ -985,8 +997,18 @@ pub const Pager = struct { /// Give back a run of pages. They become reusable two generations later -- /// see the field comment on `free_ready`. + /// + /// Under the allocation lock, like every other mutation of the free lists. + /// It was not, and the hot caller is `page_mut_cow`, which runs under a + /// *collection* lock: two collections doing copy-on-write concurrently + /// appended to the same list, and `publish` rotated all three lists + /// underneath them. No caller holds the lock already -- `page_mut_cow` takes + /// it inside `alloc_pages` and has released it by here -- so this cannot + /// recurse. pub fn free_pages(self: *Pager, first: u32, pages: u32) !void { if (pages == 0) return; + self.alloc_lock.lockUncancelable(self.io); + defer self.alloc_lock.unlock(self.io); try self.free_pending.append(self.gpa, .{ .first = first, .pages = pages }); } @@ -1010,9 +1032,22 @@ pub const Pager = struct { // // `take_free` never *adds* an entry, so the bound holds and one // allocation is enough. + // + // Under `alloc_lock` for the whole of it, allocation included. Reading + // the three lists is as much a use of them as appending is: a concurrent + // `free_pages` -- copy-on-write in some collection, which a checkpoint + // does not exclude -- grows `free_pending` while the loop below walks it, + // and a growth that reallocates leaves the loop on freed memory. A free + // that lands after this point simply waits for the next generation's + // list; the page stays allocated one generation longer, which is the + // safe direction. + self.alloc_lock.lockUncancelable(self.io); + defer self.alloc_lock.unlock(self.io); const bound = self.free_ready.items.len + self.free_hold.items.len + self.free_pending.items.len; const pages: u32 = @intCast((8 + bound * 8 + 8 + page_size - 1) / page_size); - const first = try self.alloc_pages(pages); + var hold: Reservation = .{}; + try self.reserve_pages_locked(&hold, pages); + const first = self.alloc_assume_reserved_locked(&hold, pages); const buf = self.bytes_mut(@as(u64, first) << page_shift, @as(usize, pages) << page_shift); @memset(buf, 0); var at: usize = 8; @@ -1809,6 +1844,79 @@ test "one-page requests do not carve up the runs the extents need" { try testing.expectEqual(tail_before, pg.alloc_tail); } +test "concurrent frees lose no pages while a publish rotates the lists" { + // `free_pages` mutates the same three lists `publish` rotates, and its hot + // caller is `page_mut_cow` under a *collection* lock -- so two collections + // copying nodes concurrently were appending to one `ArrayList` unserialized + // while a checkpoint moved it out from under them. + // + // Pages are conserved across the rotation and across coalescing, so the sum + // over all three lists is the invariant to assert. Probabilistic by nature, + // as any test of a data race is: it says nothing when green and is only + // evidence when red. Mutation check: drop the lock from `free_pages` and + // this fails or crashes within a few runs. + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + + const freers = 4; + const per_freer = 200; + // One page each, allocated up front so no fiber is also growing the file. + var pages: [freers * per_freer]u32 = undefined; + for (&pages) |*p| p.* = try tp.pg().alloc_pages(1); + try tp.pg().publish(.{ .seq = 1 }); + + const Worker = struct { + fn freer(p: *Pager, run: []const u32) error{Canceled}!void { + for (run) |page| p.free_pages(page, 1) catch return error.Canceled; + } + fn publisher(p: *Pager, seq: *std.atomic.Value(u64)) error{Canceled}!void { + for (0..8) |_| { + p.publish(.{ .seq = seq.fetchAdd(1, .monotonic) }) catch return error.Canceled; + } + } + }; + + var seq = std.atomic.Value(u64).init(2); + var group: std.Io.Group = .init; + defer group.cancel(io); + for (0..freers) |i| { + group.async(io, Worker.freer, .{ tp.pg(), pages[i * per_freer ..][0..per_freer] }); + } + group.async(io, Worker.publisher, .{ tp.pg(), &seq }); + try group.await(io); + + // Page identity, not a total: the publisher's own free-list streams are + // allocated *off this list*, so a plain count would be short by however many + // publishes found a fit. Every page still on the list must therefore be one + // the freers put there, exactly once -- a lost or half-written append shows + // up as a duplicate or as a page nobody freed, neither of which recycling + // can produce. + const lo = pages[0]; + var seen = try std.DynamicBitSetUnmanaged.initEmpty(gpa, pages.len); + defer seen.deinit(gpa); + var on_list: usize = 0; + for ([_][]const Extent{ + tp.pg().free_ready.items, + tp.pg().free_hold.items, + tp.pg().free_pending.items, + }) |list| for (list) |e| { + for (0..e.pages) |i| { + const p = e.first + @as(u32, @intCast(i)); + try testing.expect(p >= lo and p - lo < pages.len); + try testing.expect(!seen.isSet(p - lo)); + seen.set(p - lo); + on_list += 1; + } + }; + // The only pages missing are the ones a publish recycled into its stream, + // and there were nine publishes at one page each. + try testing.expect(pages.len - on_list <= 9); +} + test "freed pages that touch merge back into a usable run" { // Mutation check: drop the `coalesce_free_ready()` call from `publish`. Red // -- the four one-page frees below stay four separate holes and the run of -- 2.39.5 From 51eed826fb96f4d36988ac80c0bf3fd686ba8992 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 10:52:15 +0300 Subject: [PATCH 05/37] pager: a checkpoint gives back the generation it replaced Both streams a publish writes -- the catalog and the free list -- are allocated into fresh pages every time, so that a crash leaves the previous copy readable. Nothing ever gave those pages back. A server checkpoints on log volume rather than on having anything new to say, so an idle database grew its data file forever, two runs per checkpoint. The magnitude is not the two pages it looks like: the catalog carries a `u32` per index node page, so at the tens-of-GB target that is hundreds of KB abandoned at every checkpoint. It is the same shape as the reclamation bugs the M0 churn gate found -- a mechanism that works once and never twice -- and it was invisible for the same reason, that no test ran enough checkpoints to see a trend. A publish overwrites the watermark slot of the generation *two* back, since the two slots hold the new generation and its predecessor. That is the generation whose streams nothing can reach again, so `Pager` now remembers where the last two generations put theirs and frees the older pair. Process-local rather than recorded in the watermark: only a running pager needs to know, because an open reads the slot it loads and the other slot is its fallback. The pages go through `free_pages` like anything else, so they are still withheld for two more generations. Steady state is therefore a handful of pages in flight, not zero growth, and the test asserts the number does not track the publish count: forty publishes over an otherwise idle pager move `alloc_tail` by at most eight pages. An open derives the loaded generation's page counts from the lengths in its watermark, which can be one page short for a free-list stream whose final length fell inside the page its bound reserved. One page, once per open, against an unbounded leak. Two existing tests had pinned the leak's arithmetic and now assert the invariant instead of the number. Verified: `zig build test` 161/161 in ReleaseFast and ReleaseSafe, `zig build fuzz`, e2e 49, e2e2 concurrent 2 + crash pair 1/3, e2e3 16, e2e4 17, e2e6 72, e2e7 86, `crash-fuzz.js` 60 cycles. Mutation-checked: dropping the two frees takes the growth from a handful of pages to one per publish. --- src/pager.zig | 109 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/src/pager.zig b/src/pager.zig index 8521555..485131c 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -134,6 +134,21 @@ pub const Extent = struct { /// What a checkpoint publishes. Everything here is authoritative except the /// two cached counters, which are hints the engine recomputes if they look /// wrong. +/// Pages a stream of `len` bytes occupies. +fn pages_for(len: u64) u32 { + return @intCast((len + page_size - 1) / page_size); +} + +/// One generation's two streams, as page runs. Page counts rather than byte +/// lengths because the free list speaks pages and because the free-list stream +/// is allocated from an upper bound that its final length can fall short of. +pub const Streams = struct { + catalog_page: u32 = 0, + catalog_pages: u32 = 0, + freelist_page: u32 = 0, + freelist_pages: u32 = 0, +}; + pub const Watermark = struct { generation: u64 = 0, /// The log sequence this image covers. The crash-recovery invariant is that @@ -235,6 +250,22 @@ pub const Pager = struct { /// The generation the next checkpoint will publish. generation: u64, + /// Where the catalog and free-list streams of the two most recent + /// generations live, so the one that falls out of reference can be freed. + /// + /// Both streams are written into *fresh* pages at every publish, so that a + /// crash leaves the previous copy intact. Nothing ever gave those pages + /// back: a database checkpointing every 32 MiB of log leaked two runs per + /// checkpoint forever, and the catalog carries a `u32` per index node page, + /// so at the tens-of-GB target that is hundreds of KB each time. + /// + /// Process-local rather than in the watermark: only a running pager needs to + /// know, because an open reads the slot it is loading and the other slot is + /// the fallback. Two generations because a publish overwrites the slot of + /// the generation *two* back -- that is the one nothing can reach again. + streams_cur: Streams, + streams_prev: Streams, + /// Pages below this belong to the last published image and must never be /// stored into (PLAN amendment A1). Zero until a checkpoint publishes one, /// which is why copy-on-write is inert before then. @@ -330,6 +361,8 @@ pub const Pager = struct { .fresh = created, .loaded = .{}, .generation = 0, + .streams_cur = .{}, + .streams_prev = .{}, .stable_pages = 0, .unpublished = .{}, .free_ready = .empty, @@ -859,6 +892,18 @@ pub const Pager = struct { // Everything the published image references is off limits to // writes from here on. self.stable_pages = wm.alloc_tail; + // So the next publish but one gives this generation's streams back. + // Page counts are derived from the lengths here rather than + // remembered, which can be one page short for a free-list stream + // whose final length fell inside the page its bound reserved. That + // loses at most one page, once per open, against the unbounded leak + // this replaces. + self.streams_cur = .{ + .catalog_page = wm.catalog_page, + .catalog_pages = pages_for(wm.catalog_len), + .freelist_page = wm.freelist_page, + .freelist_pages = pages_for(wm.freelist_len), + }; try self.read_freelist(wm); } else if (self.watermark_attempted()) { // Only worth saying when a watermark was *written* and cannot be @@ -973,7 +1018,24 @@ pub const Pager = struct { // made durable -- so a test can discard everything else and assert the // image still loads. Clearing it here would make that check vacuous. - // 5. only now is the new image current: advance the stable mark and + // 5. the streams of the generation two back are now unreachable: the + // slot that named them is the one this publish just overwrote, and + // the two slots hold the new generation and its predecessor. Give + // their pages back -- via the free list, so they are still withheld + // for two more generations like anything else. + // + // Before the lock below, which `free_pages` takes for itself. + try self.free_pages(self.streams_prev.catalog_page, self.streams_prev.catalog_pages); + try self.free_pages(self.streams_prev.freelist_page, self.streams_prev.freelist_pages); + self.streams_prev = self.streams_cur; + self.streams_cur = .{ + .catalog_page = wm.catalog_page, + .catalog_pages = pages_for(wm.catalog_len), + .freelist_page = fl.first, + .freelist_pages = fl.pages, + }; + + // 6. only now is the new image current: advance the stable mark and // rotate the free lists by one generation. self.generation = wm.generation; self.loaded = wm; @@ -1019,7 +1081,7 @@ pub const Pager = struct { return n; } - fn write_freelist(self: *Pager) !struct { first: u32, len: u64 } { + fn write_freelist(self: *Pager) !struct { first: u32, len: u64, pages: u32 } { // Size from an upper bound, then count the entries actually written. // // The allocation below takes its pages off this very list, and an exact @@ -1069,7 +1131,10 @@ pub const Pager = struct { ); std.mem.writeInt(u64, buf[0..8], count, .little); std.mem.writeInt(u64, buf[at..][0..8], header_hash(buf[0..at]), .little); - return .{ .first = first, .len = @as(u64, at) + 8 }; + // `pages` and not `pages_for(len)`: the allocation was sized from the + // bound, and the whole run has to go back when this generation falls + // out of reference. + return .{ .first = first, .len = @as(u64, at) + 8, .pages = pages }; } /// Load the free list a watermark points at. A damaged one is dropped with a @@ -1623,8 +1688,12 @@ test "the persisted free list survives allocating its own pages" { // This is the publish that trips it: the free list is now non-empty and // holds a run of exactly the one page the stream needs. try tp.pg().publish(.{ .seq = 4 }); + // Not a fixed number: a publish also recycles the streams of the generation + // two back, so what is on the list is the three frees above minus whatever + // the stream allocations took plus whatever they gave back. The invariant + // under test is that a reopen agrees with it, whatever it is. const ready_before = tp.pg().free_ready_pages(); - try testing.expectEqual(@as(u32, 2), ready_before); + try testing.expect(ready_before > 0); tp.close(); var again = try reopen(io, tp.path); @@ -1844,6 +1913,33 @@ test "one-page requests do not carve up the runs the extents need" { try testing.expectEqual(tail_before, pg.alloc_tail); } +test "a quiet checkpoint stops growing the file" { + // Both streams a publish writes are allocated fresh every time, so that a + // crash leaves the previous copy readable. Nothing gave them back, and a + // database checkpoints on log volume rather than on having anything to say + // -- so an idle server grew the data file forever. + // + // Steady state is not zero growth: a publish allocates this generation's + // streams and frees the ones from two generations back, and those take two + // more publishes to become reusable. So a few pages are always in flight, + // and the assertion is that the number does not track the publish count. + // + // Mutation check: drop the two `free_pages` calls from `publish` and this + // goes red at 40-something pages instead of a handful. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + + // Let the rotation reach its steady state before measuring. + for (1..5) |i| try tp.pg().publish(.{ .seq = i }); + const settled = tp.pg().alloc_tail; + + for (5..45) |i| try tp.pg().publish(.{ .seq = i }); + try testing.expect(tp.pg().alloc_tail - settled <= 8); +} + test "concurrent frees lose no pages while a publish rotates the lists" { // `free_pages` mutates the same three lists `publish` rotates, and its hot // caller is `page_mut_cow` under a *collection* lock -- so two collections @@ -1906,7 +2002,10 @@ test "concurrent frees lose no pages while a publish rotates the lists" { }) |list| for (list) |e| { for (0..e.pages) |i| { const p = e.first + @as(u32, @intCast(i)); - try testing.expect(p >= lo and p - lo < pages.len); + // Pages outside the set the freers own are the publisher's own + // stream runs coming back two generations later; they are not what + // this test is about. + if (p < lo or p - lo >= pages.len) continue; try testing.expect(!seen.isSet(p - lo)); seen.set(p - lo); on_list += 1; -- 2.39.5 From bafbc95898fe272b1672f595f9484737746587a2 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 11:01:54 +0300 Subject: [PATCH 06/37] db: a checkpoint publishes only what the log has made durable `checkpoint` snapshotted `self.seq`, walked the catalog, and then asserted that the snapshot was at or below `committed_seq`. It is not, whenever a writer appended before the snapshot and has not finished committing -- between `insert` and `commit`, or inside `commit` waiting on the leader's fsync. The existing `self.seq != snapshot_seq` retry does not catch it: nothing appended *during* the walk, the append was already there when it started. The window is as wide as an fsync, and it reproduces in seconds: four writers following the dispatch epilogue's insert-then-commit against a checkpoint loop trip it on every run. It has stayed hidden because a checkpoint fires once per 32 MiB of log, so the two rarely meet -- which stops being true for exactly the churn workloads M1 is about to measure. Publishing there would claim durability for a record still in the log's buffer, and `truncate_to_header` immediately afterwards would throw it away: the client gets its acknowledgement, the record is gone. That is the failure the whole watermark ordering exists to prevent (PLAN D6), and expressing it as an assertion turned it into a server abort rather than a wrong answer -- which is the better of the two, but it is not a fix. Now a retry. `commit` seals every append made so far, so sealing and re-snapshotting converges in one more round rather than spinning against sustained writes. The assertion moves to the line above `publish`, where `log_lock` has been held since the check and `committed_seq` only grows, so it is a tripwire for future edits rather than a live hazard. Also here, because the same test found it: the catalog's live-byte sum was asserted against the engine total *inside* `write_catalog`, where the sum is accumulated across collections over time while the total moves under it. A writer landing mid-walk tripped it on a database that was perfectly consistent. The check moves to the caller and runs only when the engine total did not move across the walk. What it guards against -- a path that updates one level and not the other -- is deterministic wherever it exists, so a check that skips under sustained writes still catches it. Verified: `zig build test` 162/162 in ReleaseFast and ReleaseSafe, three consecutive runs of the new concurrency test. Reverting either half reproduces its own panic within one run. --- src/db.zig | 152 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 137 insertions(+), 15 deletions(-) diff --git a/src/db.zig b/src/db.zig index 57ba93e..1ca867d 100644 --- a/src/db.zig +++ b/src/db.zig @@ -1612,14 +1612,20 @@ pub const Engine = struct { const catalog_magic: u32 = 0x4D464354; // "MFCT" const catalog_version: u32 = 1; - fn write_catalog(self: *Engine, out: *std.ArrayListUnmanaged(u8)) !void { + /// Serialize the catalog and return the live-byte total it observed. + /// + /// The engine's own total is by definition the sum over collections, and + /// `read_catalog` rebuilds it that way, so a divergence means some path + /// published or evicted bytes at one level and not the other -- with a + /// compaction trigger that fires never or always as the visible symptom. + /// The check is worth making and this is where every collection is walked + /// anyway, but it cannot be made *here*: the sum is accumulated across + /// collections over time while the engine's total moves under it, so a + /// writer landing mid-walk would trip it on a database that is perfectly + /// consistent. The caller asserts it after the `seq` check has established + /// that no writer landed at all. + fn write_catalog(self: *Engine, out: *std.ArrayListUnmanaged(u8)) !u64 { const gpa = self.gpa; - // The engine's live-byte total is by definition the sum over - // collections, and `read_catalog` rebuilds it that way. Check it here, - // where every collection is being walked regardless: a divergence means - // some path published or evicted bytes at one level and not the other, - // and the visible symptom would be a compaction trigger that fires - // never or always. var live_sum: u64 = 0; try put_u32(gpa, out, catalog_magic); try put_u32(gpa, out, catalog_version); @@ -1653,11 +1659,8 @@ pub const Engine = struct { for (coll.indexes.items) |ix| try write_index_catalog(gpa, out, ix); } } - assert_msg( - live_sum == self.live_bytes, - "the engine's live-byte total must equal the sum over collections", - ); try put_u64(gpa, out, std.hash.XxHash3.hash(0, out.items)); + return live_sum; } fn write_index_catalog( @@ -1883,7 +1886,8 @@ pub const Engine = struct { buf.clearRetainingCapacity(); try self.catalog_lock.lockShared(self.io); const snapshot_seq = self.seq; - self.write_catalog(&buf) catch |err| { + const live_before = self.live_bytes; + const live_sum = self.write_catalog(&buf) catch |err| { self.catalog_lock.unlockShared(self.io); return err; }; @@ -1896,9 +1900,42 @@ pub const Engine = struct { self.log_lock.unlock(self.io); continue; } - assert_msg( - snapshot_seq <= self.committed_seq, - "checkpoint watermark past the durable log tail", + if (snapshot_seq > self.committed_seq) { + // A writer appended before the snapshot and its commit has not + // landed yet -- it is between `insert` and `commit`, or inside + // one, waiting on the leader's fsync. The seq check above does + // not catch this: nothing appended *during* the walk, the + // append was already there when it started. + // + // Publishing here would claim durability for a record that is + // still in the log's buffer, and the truncation that follows a + // checkpoint would then throw it away. That is the one thing + // the whole watermark ordering exists to prevent (PLAN D6), and + // it used to be an assertion -- so the failure mode was a + // server abort under exactly the load that makes checkpoints + // frequent. Reproduced in seconds by four writers against a + // checkpoint loop, and the window is as wide as an fsync. + // + // Seal it and take the snapshot again rather than spinning: + // `commit` covers every append made so far, so one more round + // is enough. Outside `log_lock`, which `commit` takes itself. + self.log_lock.unlock(self.io); + try self.commit(); + continue; + } + // Only when the walk was quiet. An unchanged `seq` is not enough on + // its own: a writer bumps it when it appends the log record and + // updates the byte counters afterwards, so it can be past the seq + // the snapshot captured and still be about to move `live_bytes` + // under a collection the walk has already been through. Requiring + // the engine total to be unmoved across the whole walk closes that, + // at the cost of skipping the check under sustained writes -- which + // is the right trade, because what it guards against is a code path + // that updates one level and not the other, and that is + // deterministic wherever it exists. + if (self.live_bytes == live_before) assert_msg( + live_sum == live_before, + "the engine's live-byte total must equal the sum over collections", ); const pages: u32 = @intCast((buf.items.len + pgr.page_size - 1) / pgr.page_size); const first = self.pager.alloc_pages(pages) catch |err| { @@ -1906,6 +1943,12 @@ pub const Engine = struct { return err; }; @memcpy(self.pager.bytes_mut(@as(u64, first) << pgr.page_shift, buf.items.len), buf.items); + // The invariant, on the line that would break it: `log_lock` has + // been held since the check above and `committed_seq` only grows. + assert_msg( + snapshot_seq <= self.committed_seq, + "checkpoint watermark past the durable log tail", + ); self.pager.publish(.{ .seq = snapshot_seq, .catalog_page = first, @@ -2889,6 +2932,85 @@ test "compaction rewrites log and keeps data" { engine3.unlock(); } +test "a checkpoint runs alongside writers on several collections" { + // `write_catalog` reads each collection's slab extents, indexes and byte + // counters while holding only the *shared catalog* lock -- and a writer + // holds that same lock shared, taking the collection's lock exclusively. + // So the snapshot walked structures its owner was free to mutate, and + // `slab_extents` is an ArrayList a new extent appends to: a reallocation + // mid-walk leaves the serializer reading freed memory. + // + // Several collections rather than one, because the interesting overlap is a + // writer on collection B while the catalog is serializing collection A. + // + // Mutation check: drop the `lockShared` from `write_catalog`'s collection + // loop. Not reliably red -- a data race never is -- but it runs under + // ReleaseSafe, where the reads it makes are bounds-checked. + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + + const colls = [_][]const u8{ "a", "b", "c", "d" }; + const per_coll: i32 = 150; + var done = std.atomic.Value(usize).init(colls.len); + + const Worker = struct { + fn writer( + e: *Engine, + name: []const u8, + left: *std.atomic.Value(usize), + alloc: std.mem.Allocator, + ) error{Canceled}!void { + defer _ = left.fetchSub(1, .release); + for (1..per_coll + 1) |i| { + var doc = make_doc(alloc, @intCast(i), "user") catch return error.Canceled; + defer doc.deinit(); + { + e.lock() catch return error.Canceled; + defer e.unlock(); + e.insert("app", name, &doc, undefined) catch return error.Canceled; + } + // As the dispatch epilogue does (commands.zig): the append bumps + // `seq`, the commit is what makes it durable, and a checkpoint + // may only describe what is durable. + e.commit() catch return error.Canceled; + } + } + + fn checkpointer(e: *Engine, left: *std.atomic.Value(usize)) error{Canceled}!void { + while (left.load(.acquire) > 0) { + // Errors are the point of the retry loop inside `checkpoint`, + // not a failure of this test; a checkpoint that gives up under + // sustained writes has still not corrupted anything. + e.checkpoint() catch {}; + } + } + }; + + var group: std.Io.Group = .init; + defer group.cancel(io); + for (colls) |name| group.async(io, Worker.writer, .{ &engine, name, &done, gpa }); + group.async(io, Worker.checkpointer, .{ &engine, &done }); + try group.await(io); + + // Every write is still there, and the catalog the checkpoints wrote agrees + // with the engine -- the second half is what `write_catalog`'s own assertion + // checks on the way through. + try engine.checkpoint(); + try engine.lock_read(); + defer engine.unlock_read(); + for (colls) |name| { + const coll = engine.get_collection("app", name) orelse return error.TestUnexpectedResult; + try testing.expectEqual(@as(usize, @intCast(per_coll)), coll.id_index.count()); + } +} + test "concurrent readers and writers on a threaded Io" { // Real worker threads: writers hold the exclusive lock, readers the // shared lock. Proves the RwLock split keeps committed writes visible -- 2.39.5 From 44be4274901c845d39860ee9943727f71fb7efc2 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 11:13:22 +0300 Subject: [PATCH 07/37] db/pager: a document append cannot land in the published image `slab_reserve` asks `is_unpublished_at` whether the append cursor is still writable; `slab_append` copies the bytes there. Between them sits the log append and its fsync, and `publish` clears the entire unpublished set and mprotects the image. So the answer was routinely stale by the time it was used, and the copy stored into the durable image. In ReleaseSafe that is a bus error. In ReleaseFast, where `protect_stable` is compiled out, there is no fault at all: the store simply overwrites bytes the last checkpoint published, and the damage surfaces later as a document that reads back as something else. ReleaseFast is the mode the server ships in. Present since M0 -- reproduced on f2844e7 with the same test -- and invisible because nothing paired concurrent writers with a checkpoint. The existing concurrent suites run against ReleaseFast, where the corruption is silent, and the unit tests that do run under the protection had no checkpoint racing them. It has stayed harmless in practice only because a checkpoint fires once per 32 MiB of log; the churn workloads M1 is about to measure change that. The pager gains an `append_lock`. Appenders hold it shared, so writers on different collections still proceed concurrently and the lock decomposition ROADMAP item 5 measured is not given back; `publish` holds it exclusively for the step that freezes the image, which happens once per checkpoint. Order is append lock then allocation lock, which is what `mark_appendable` already used. `slab_append` re-asks the question under that lock and re-arms the cursor if a checkpoint has published since the reservation. Re-arming has to be infallible, because this runs after the log record is durable, so `slab_reserve` now measures its room from the rounded-up cursor rather than the raw one -- under a system page per extent, and the two share one `appendable_end` so they cannot disagree about what "there is room" means. Measured, since this is on the write path: concurrent durable writes, 8 clients x 1500 inserts at `{w:1, j:true}`, two runs each -- 23779 and 24879 docs/s before, 23823 and 24452 after. No regression outside run-to-run spread. Verified: `zig build test` in ReleaseFast and ReleaseSafe, three consecutive ReleaseSafe runs of the checkpoint concurrency test, `zig build fuzz`, e2e 49, e2e2 concurrent 2 + crash pair 1/3, e2e3 16, e2e4 17, e2e6 72, e2e7 86, `crash-fuzz.js` 60 cycles. --- src/db.zig | 37 ++++++++++++++++++++++++++++++++++--- src/pager.zig | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/src/db.zig b/src/db.zig index 1ca867d..d197603 100644 --- a/src/db.zig +++ b/src/db.zig @@ -165,7 +165,13 @@ pub const Collection = struct { // an extent recycled off the free list starts *below* the mark and is // still writable. Asking the mark meant every recycled extent was thrown // away after one document, so churn never reused anything. - if (self.pager.is_unpublished_at(self.slab_tail) and self.slab_tail + len <= self.slab_end) return; + // + // Room is checked from the *rounded-up* cursor rather than the cursor + // itself, so the round-up `slab_append` may have to do is guaranteed to + // fit. Without that the append's own re-check could discover it needs a + // fresh extent, which is fallible, after the log record is already + // durable. Costs under one system page per extent. + if (self.pager.is_unpublished_at(self.slab_tail) and self.appendable_end(len)) return; // The page holding the tail is frozen, but the *rest* of the extent is // not: nothing above the live cursor is referenced by the image or by an // index. So skip to the next system page and keep the extent, instead of @@ -179,8 +185,8 @@ pub const Collection = struct { // collections: the data file reached 11.8x the live data and grew by // ~335 MB per checkpoint, heading for DatabaseTooLarge at around 6 GB of // real data. - const resumed = std.mem.alignForward(u64, self.slab_tail, pgr.map_align); - if (resumed + len <= self.slab_end) { + if (self.appendable_end(len)) { + const resumed = std.mem.alignForward(u64, self.slab_tail, pgr.map_align); self.pager.mark_appendable(resumed, self.slab_end); self.slab_tail = resumed; return; @@ -198,9 +204,34 @@ pub const Collection = struct { self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift); } + /// Whether a document of `len` bytes fits in this extent even if the cursor + /// first has to be rounded up to a system page. The reservation and the + /// append both ask this, so they agree on what "there is room" means. + fn appendable_end(self: *const Collection, len: usize) bool { + return std.mem.alignForward(u64, self.slab_tail, pgr.map_align) + len <= self.slab_end; + } + /// Copy `bytes` into the slab and return its absolute file offset. /// Infallible: slab_reserve must have run for at least this many bytes. fn slab_append(self: *Collection, bytes: []const u8) u64 { + // The cursor was checked in `slab_reserve`, but a checkpoint can have + // published since -- the reservation runs before the log append and this + // runs after it, with an fsync in between. `publish` clears the whole + // unpublished set, so a cursor that was writable then can be inside the + // frozen image now, and the copy below would store into it: a bus error + // where the protection is compiled in, and a silent overwrite of durable + // data in ReleaseFast, where it is not. + // + // Re-arming is infallible because `slab_reserve` measured its room from + // the rounded-up cursor. The pager's append lock holds off the next + // publish for the rest of this function, so the answer stays true. + self.pager.lock_append(); + defer self.pager.unlock_append(); + if (!self.pager.is_unpublished_at(self.slab_tail)) { + const resumed = std.mem.alignForward(u64, self.slab_tail, pgr.map_align); + self.pager.mark_appendable(resumed, self.slab_end); + self.slab_tail = resumed; + } assert_msg( self.slab_tail + bytes.len <= self.slab_end, "document append overran the slab reservation", diff --git a/src/pager.zig b/src/pager.zig index 485131c..5d51f8d 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -241,6 +241,21 @@ pub const Pager = struct { /// the log append -- that is what per-consumer reservations buy. alloc_lock: std.Io.Mutex, + /// Separates a publish from the appends that are mid-flight. + /// + /// An appender asks `is_unpublished_at` whether its cursor is still + /// writable, and copies bytes there afterwards. `publish` clears the whole + /// unpublished set and mprotects the image between those two steps, so the + /// answer was stale by the time it was used and the copy landed in the + /// published image: SIGBUS where the protection is compiled in, a silent + /// store into the durable image in ReleaseFast, where it is not. + /// + /// Shared by appenders so writers on different collections still run + /// concurrently -- the decomposition ROADMAP item 5 measured is not given + /// back. Exclusive only for the tail of a publish, which happens once per + /// checkpoint. + append_lock: std.Io.RwLock, + /// True when this file was created by this open (no checkpoint to load). fresh: bool, @@ -358,6 +373,7 @@ pub const Pager = struct { .alloc_tail = page_first_data, .reserved_pages = 0, .alloc_lock = .init, + .append_lock = .init, .fresh = created, .loaded = .{}, .generation = 0, @@ -529,6 +545,18 @@ pub const Pager = struct { return self.is_unpublished(@intCast(off >> page_shift)); } + /// Hold off the next publish while an append decides where to put its bytes + /// and puts them there. Uncancelable and infallible: the append runs after + /// the log record is durable, where there is nowhere to report a failure. + /// See `append_lock`. + pub fn lock_append(self: *Pager) void { + self.append_lock.lockSharedUncancelable(self.io); + } + + pub fn unlock_append(self: *Pager) void { + self.append_lock.unlockShared(self.io); + } + /// The one page a write may legitimately land on below the stable mark: a /// watermark slot. Overwriting the *inactive* slot is the whole mechanism -- /// alternating by generation parity is what makes it safe, where every other @@ -1037,6 +1065,15 @@ pub const Pager = struct { // 6. only now is the new image current: advance the stable mark and // rotate the free lists by one generation. + // + // Exclusive against the appenders for this step alone. Freezing the + // image while one of them is between "is my cursor still writable" + // and the copy that relies on the answer is what put documents into + // the durable image; holding them off here is what makes the answer + // still true when it is used. Taken before `alloc_lock`, the order + // `mark_appendable` uses on the appender's side. + self.append_lock.lockUncancelable(self.io); + defer self.append_lock.unlock(self.io); self.generation = wm.generation; self.loaded = wm; // The free lists and the unpublished set are allocator state, so the -- 2.39.5 From f8a39a096568f8db788639f657f3c68a0c6853ea Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 11:14:17 +0300 Subject: [PATCH 08/37] db: the catalog snapshot is read under each collection's lock `write_catalog` walks every collection's slab extents, byte counters and index metadata while holding only the *shared catalog* lock -- which is the same lock a writer holds, taking the collection's lock exclusively. So the snapshot read structures their owners were free to mutate underneath it. `slab_extents` makes it more than a torn read: it is an ArrayList that `slab_reserve` appends to, and an append that reallocates leaves the serializer walking freed memory. What it writes from that walk is the catalog the next open trusts to find every extent the collection owns. Now under each collection's lock, shared, taken inside the catalog lock -- the same order `compact` uses, so no new ordering to reason about. Not the commit that found the concurrency bugs above; those needed a checkpoint racing writers, which this lock is orthogonal to. It is the one that makes the snapshot legal rather than merely lucky. Verified: `zig build test` in ReleaseFast and ReleaseSafe. --- src/db.zig | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/db.zig b/src/db.zig index d197603..0f80a43 100644 --- a/src/db.zig +++ b/src/db.zig @@ -1670,6 +1670,14 @@ pub const Engine = struct { var coll_it = colls.iterator(); while (coll_it.next()) |ce| { const coll = ce.value_ptr.*; + // Everything below this line is written by a collection's own + // writer under its own lock, and `slab_extents` is an ArrayList + // that `slab_reserve` appends to -- so reading it under only the + // shared catalog lock could walk a slice a concurrent append had + // already reallocated. Lock order is catalog then collection, + // the same order `compact` uses. + try coll.lock.lockShared(self.io); + defer coll.lock.unlockShared(self.io); try put_bytes(gpa, out, ce.key_ptr.*); try put_u64(gpa, out, coll.slab_tail); try put_u64(gpa, out, coll.slab_end); -- 2.39.5 From 992cc2a5ab100eba33a55aec810b606706b6803d Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 11:31:37 +0300 Subject: [PATCH 09/37] db: a dropped collection is reclaimed, not deadened `free_collection` charged the engine's `dead_bytes` with the dropped collection's live bytes, and then, three lines down, handed every page that collection owned back to the pager. A drop therefore asked for a rebuild -- a full copy of every collection that was left -- to reclaim space that had already been reclaimed. Its own garbage was wrong the other way: bytes that died before the drop stayed on the engine's books after the pages holding them were freed. Both halves of that are the same statement: `dead_bytes` is the sum of `slab_used - live_bytes` over the collections that still exist. Make it so on the drop path, then stop storing it separately at all -- `read_catalog` recomputes it from the collections the catalog lists, so the watermark's copy is now a hint for anything inspecting the header rather than a second source of truth. It would be wrong in one specific way if it stayed one: a collection dropped after the last checkpoint is gone from the catalog but still charged for in the hint. `write_catalog` now returns the dead sum beside the live one and the checkpoint asserts it the same way, under the same quiescence condition. That is what makes the accounting checkable rather than merely intended. Mutation-checked three ways, each red on its own: charge the drop again; delete the subtraction of the collection's own garbage; delete the accumulation in `read_catalog`. 163/163 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz, e2e 49, e2e2 concurrent 2 + crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/db.zig | 171 ++++++++++++++++++++++++++++++++++++++++++++------ src/pager.zig | 3 +- 2 files changed, 155 insertions(+), 19 deletions(-) diff --git a/src/db.zig b/src/db.zig index 0f80a43..1a4015f 100644 --- a/src/db.zig +++ b/src/db.zig @@ -438,9 +438,14 @@ pub const Engine = struct { engine.seq = replay_from; engine.committed_seq = replay_from; engine.live_docs = engine.pager.loaded.live_docs; - // Without this a restart forgets its garbage, and a churned - // database would never compact again. - engine.dead_bytes = engine.pager.loaded.dead_bytes; + // `dead_bytes` is *not* restored from the watermark. It is + // derived, not stored: `read_catalog` has already summed + // `slab_used - live_bytes` over the collections the catalog + // still lists. The watermark's copy is a hint for anything + // inspecting the header without parsing the catalog, and it + // would be wrong here in one specific way -- a collection + // dropped after the last checkpoint takes its garbage with it, + // and the hint would keep charging the engine for it. } } @@ -498,7 +503,28 @@ pub const Engine = struct { self.dead_docs += coll.doc_count; assert_msg(self.live_bytes >= coll.live_bytes, "dropping a collection would underflow the engine's live bytes"); self.live_bytes -= coll.live_bytes; - self.dead_bytes += coll.live_bytes; + // A drop *reclaims*, it does not deaden. The loop below hands every page + // this collection owned back to the pager, so its live bytes are not + // garbage waiting for a rebuild -- they are already gone. Adding them to + // `dead_bytes` armed a compaction for space that had just been returned, + // and a rebuild costs a full copy of every *other* collection. + // + // Its garbage goes the other way, for the same reason: the bytes this + // collection had already lost to eviction were counted in `dead_bytes` + // when they died, and those pages are being freed too. That keeps + // `dead_bytes` exactly the sum of `slab_used - live_bytes` over the + // collections that still exist, which is what `read_catalog` recomputes + // on open and what `write_catalog` asserts. + assert_msg( + coll.slab_used >= coll.live_bytes, + "a collection cannot hold more live bytes than it ever appended", + ); + const coll_dead = coll.slab_used - coll.live_bytes; + assert_msg( + self.dead_bytes >= coll_dead, + "dropping a collection would underflow the engine's dead bytes", + ); + self.dead_bytes -= coll_dead; coll.id_index.deinit(self.gpa); for (coll.indexes.items) |ix| { ix.deinit(self.gpa); @@ -1643,21 +1669,26 @@ pub const Engine = struct { const catalog_magic: u32 = 0x4D464354; // "MFCT" const catalog_version: u32 = 1; - /// Serialize the catalog and return the live-byte total it observed. + /// What a catalog walk observed, for the caller to check the engine's own + /// running totals against. + const CatalogSums = struct { live: u64, dead: u64 }; + + /// Serialize the catalog and return the byte totals it observed. /// - /// The engine's own total is by definition the sum over collections, and - /// `read_catalog` rebuilds it that way, so a divergence means some path + /// The engine's own totals are by definition the sums over collections, and + /// `read_catalog` rebuilds them that way, so a divergence means some path /// published or evicted bytes at one level and not the other -- with a /// compaction trigger that fires never or always as the visible symptom. /// The check is worth making and this is where every collection is walked - /// anyway, but it cannot be made *here*: the sum is accumulated across - /// collections over time while the engine's total moves under it, so a + /// anyway, but it cannot be made *here*: the sums are accumulated across + /// collections over time while the engine's totals move under them, so a /// writer landing mid-walk would trip it on a database that is perfectly - /// consistent. The caller asserts it after the `seq` check has established + /// consistent. The caller asserts them after the `seq` check has established /// that no writer landed at all. - fn write_catalog(self: *Engine, out: *std.ArrayListUnmanaged(u8)) !u64 { + fn write_catalog(self: *Engine, out: *std.ArrayListUnmanaged(u8)) !CatalogSums { const gpa = self.gpa; var live_sum: u64 = 0; + var dead_sum: u64 = 0; try put_u32(gpa, out, catalog_magic); try put_u32(gpa, out, catalog_version); try put_u64(gpa, out, self.live_docs); @@ -1688,6 +1719,7 @@ pub const Engine = struct { coll.live_bytes <= coll.slab_used, "a collection cannot hold more live bytes than it ever appended", ); + dead_sum += coll.slab_used - coll.live_bytes; try put_u32(gpa, out, @intCast(coll.slab_extents.items.len)); for (coll.slab_extents.items) |e| { try put_u32(gpa, out, e.first); @@ -1699,7 +1731,7 @@ pub const Engine = struct { } } try put_u64(gpa, out, std.hash.XxHash3.hash(0, out.items)); - return live_sum; + return .{ .live = live_sum, .dead = dead_sum }; } fn write_index_catalog( @@ -1764,9 +1796,14 @@ pub const Engine = struct { coll.slab_end = try r.read_u64(); coll.slab_used = try r.read_u64(); coll.live_bytes = try r.read_u64(); - // The engine's total is the sum over collections rather than a - // separately stored field, so the two cannot disagree. + // Both engine totals are sums over collections rather than + // separately stored fields, so neither can disagree with the + // catalog. `slab_used - live_bytes` is this collection's slab + // garbage by definition -- bytes it appended and no longer + // reaches -- which is exactly what the rebuild trigger counts. + if (coll.slab_used < coll.live_bytes) return error.CorruptCatalog; self.live_bytes += coll.live_bytes; + self.dead_bytes += coll.slab_used - coll.live_bytes; const nex = try r.read_u32(); var e: u32 = 0; while (e < nex) : (e += 1) { @@ -1926,7 +1963,8 @@ pub const Engine = struct { try self.catalog_lock.lockShared(self.io); const snapshot_seq = self.seq; const live_before = self.live_bytes; - const live_sum = self.write_catalog(&buf) catch |err| { + const dead_before = self.dead_bytes; + const sums = self.write_catalog(&buf) catch |err| { self.catalog_lock.unlockShared(self.io); return err; }; @@ -1973,9 +2011,16 @@ pub const Engine = struct { // that updates one level and not the other, and that is // deterministic wherever it exists. if (self.live_bytes == live_before) assert_msg( - live_sum == live_before, + sums.live == live_before, "the engine's live-byte total must equal the sum over collections", ); + // The same argument, for the total the rebuild trigger reads. This + // is what makes a drop's accounting checkable: charge the engine for + // a dropped collection's bytes and the two sides part company here. + if (self.dead_bytes == dead_before) assert_msg( + sums.dead == dead_before, + "the engine's dead-byte total must equal the sum over collections", + ); const pages: u32 = @intCast((buf.items.len + pgr.page_size - 1) / pgr.page_size); const first = self.pager.alloc_pages(pages) catch |err| { self.log_lock.unlock(self.io); @@ -2697,8 +2742,9 @@ test "compaction still triggers after a checkpoint has truncated the log" { // then never fires and the doc slab grows without bound. This test exists // because the churn gate measured exactly that: 4.1x live data. // - // Second mutation: drop `dead_bytes` from the watermark, or from the restore - // beside `live_docs` in `open`. Red on the reopened engine below. + // Second mutation: drop the `dead_bytes` accumulation from `read_catalog`. + // Red in "reopen carries the garbage counter across a restart", which is + // where the counter has to survive a restart. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); var env = test_env(&threaded); @@ -2807,6 +2853,14 @@ test "a rebuild leaves the space it reclaimed ready to reuse" { } test "reopen carries the garbage counter across a restart" { + // It carries it by *recomputing* it: `read_catalog` sums + // `slab_used - live_bytes` over the collections the catalog lists, rather + // than trusting the watermark's cached copy. That is a stronger claim than + // the hint was -- a collection dropped since the last checkpoint is simply + // not in the sum, where the hint kept charging the engine for it. + // + // Mutation: drop the accumulation in `read_catalog`; `engine2.dead_bytes` + // reads zero below. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); var env = test_env(&threaded); @@ -2844,6 +2898,87 @@ test "reopen carries the garbage counter across a restart" { defer engine2.deinit(); try testing.expectEqual(dead_before, engine2.dead_bytes); try testing.expectEqual(live_before, engine2.live_bytes); + // And it is the sum over collections on both sides of the restart, not a + // number kept beside them. + const reopened = engine2.get_collection("app", "c").?; + try testing.expectEqual(reopened.slab_used - reopened.live_bytes, engine2.dead_bytes); +} + +test "dropping a collection does not arm compaction" { + // `free_collection` charged the engine's `dead_bytes` with the dropped + // collection's *live* bytes, having just handed every page it owned back to + // the pager on the following line. A drop of a large collection therefore + // asked for a rebuild -- a full copy of every collection that was left -- + // to reclaim space that had already been reclaimed. Its own garbage was + // wrong the other way: it stayed on the engine's books after the pages + // holding it were gone. + // + // Mutation: restore `self.dead_bytes += coll.live_bytes;`, or delete the + // subtraction of `coll.slab_used - coll.live_bytes`. Either one is red on + // the equality below, and the first also re-arms the trigger. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // no rebuild while setting up + try engine.lock(); + defer engine.unlock(); + + // A small collection that survives, and a large one that does not. Both + // hold garbage, so the drop has to keep one collection's and discard the + // other's. + for (0..40) |i| { + var d = try make_doc(gpa, @intCast(i), "x"); + defer d.deinit(); + try engine.insert("app", "keep", &d, &env.gen); + } + for (0..200) |i| { + var d = try make_doc(gpa, @intCast(i), "x"); + defer d.deinit(); + try engine.insert("app", "gone", &d, &env.gen); + } + for (0..10) |i| { + var d = try make_doc(gpa, @intCast(i), "yy"); + defer d.deinit(); + _ = try engine.replace("app", "keep", &d, &env.gen); + } + for (0..200) |i| { + var d = try make_doc(gpa, @intCast(i), "yy"); + defer d.deinit(); + _ = try engine.replace("app", "gone", &d, &env.gen); + } + try engine.commit(); + + const keep = engine.get_collection("app", "keep").?; + const keep_dead = keep.slab_used - keep.live_bytes; + const keep_live = keep.live_bytes; + try testing.expect(keep_dead > 0); + try testing.expect(engine.dead_bytes > keep_dead); + + // Arm the trigger for what the two of them hold between them. + engine.compact_threshold = keep_dead + 1; + engine.note_compact(); + try testing.expect(engine.take_compact()); + + try testing.expect(try engine.drop_collection("app", "gone")); + try testing.expectEqual(keep_dead, engine.dead_bytes); + try testing.expectEqual(keep_live, engine.live_bytes); + + // The next write reconsiders the trigger and finds nothing worth a rebuild: + // what the drop reclaimed is not garbage, it is free. + engine.note_compact(); + try testing.expect(!engine.take_compact()); + + // The catalog agrees, which is what the reopened engine will read. + try engine.checkpoint(); + try testing.expectEqual(keep_dead, engine.dead_bytes); } test "reopen replays log" { diff --git a/src/pager.zig b/src/pager.zig index 5d51f8d..7435389 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -27,7 +27,8 @@ //! [56..64) u64 freelist_len //! [64..72) u64 prev_generation -- kept intact for fallback //! [72..80) u64 live_docs -- cached hint -//! [80..88) u64 dead_bytes -- cached, drives the rebuild trigger +//! [80..88) u64 dead_bytes -- cached hint; the engine recomputes it +//! from the catalog on open //! [88..4088) reserved (zero) //! [4088..4096) u64 xxhash3 over [0..4088) //! pages 3.. data, handed out by a tail-bump extent allocator -- 2.39.5 From e15d7f2ed00dbb664661fdab5f25484c3891bd10 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Sun, 9 Aug 2026 12:09:47 +0300 Subject: [PATCH 10/37] db/pager: the concurrency test writes the way the server does "a checkpoint runs alongside writers on several collections" drove its writers through `Engine.lock()` -- the legacy whole-engine lock, which the server has not used since the locks were decomposed. That serialized the writers against each other, so the overlap the test is named for never happened: `write_catalog` takes each collection's lock shared, and nothing it was racing against took that lock at all. Drive them the way `commands.zig` dispatch does instead: catalog shared, then the target collection exclusive. The test then does what it says, and immediately found something -- two appenders on different collections calling `bytes_mut` at the same time corrupt the pager's `dirty` set, which is an unsynchronized hash map. ReleaseSafe aborts in `getOrPutContextAdapted`; three runs in five. `dirty` is test-only instrumentation (`track_dirty = builtin.is_test`), so this is a harness bug rather than a server one -- but it is the one shared structure on a write path whose writers are otherwise kept apart by owning different pages, and it needs a lock of its own. Six ReleaseSafe runs clean afterwards. 163/163 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz. --- src/db.zig | 15 +++++++++++++-- src/pager.zig | 32 ++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/db.zig b/src/db.zig index 1a4015f..461d6f6 100644 --- a/src/db.zig +++ b/src/db.zig @@ -3146,8 +3146,19 @@ test "a checkpoint runs alongside writers on several collections" { var doc = make_doc(alloc, @intCast(i), "user") catch return error.Canceled; defer doc.deinit(); { - e.lock() catch return error.Canceled; - defer e.unlock(); + // The server's discipline, not the legacy whole-engine + // lock: catalog shared, then the target collection + // exclusive (commands.zig dispatch). `write_catalog` takes + // the same two in the same order, and that is the whole + // reason its walk of a collection's counters and extents is + // safe -- a writer that skipped the collection lock would + // not be excluded by it, and the test would be checking + // nothing. + e.lock_catalog(false) catch return error.Canceled; + defer e.unlock_catalog(false); + const coll = (e.lock_collection("app", name, true, true) catch + return error.Canceled) orelse return error.Canceled; + defer e.unlock_collection(coll, true); e.insert("app", name, &doc, undefined) catch return error.Canceled; } // As the dispatch epilogue does (commands.zig): the append bumps diff --git a/src/pager.zig b/src/pager.zig index 7435389..3642a35 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -321,6 +321,13 @@ pub const Pager = struct { /// See `track_dirty`. Pages written since the last sync. dirty: if (track_dirty) std.AutoHashMapUnmanaged(u32, void) else void, + /// Guards `dirty`, and only `dirty`. Writers to the file itself are kept + /// apart by the locks their *owners* hold -- a collection's, an index's -- + /// and the pages two of them touch never overlap. This set is the one thing + /// they share: two appenders on different collections record into the same + /// hash map at the same time, which is a torn map rather than a torn page. + /// Test-only instrumentation, so it costs the server nothing. + dirty_lock: if (track_dirty) std.Io.Mutex else void, /// Whether the last `protect_image` actually took effect. Checked by a test: /// an mprotect that silently fails would leave the belt looking present and /// doing nothing, which is worse than not having it. @@ -386,6 +393,7 @@ pub const Pager = struct { .free_hold = .empty, .free_pending = .empty, .dirty = if (track_dirty) .empty else {}, + .dirty_lock = if (track_dirty) .init else {}, .protect_ok = false, }; // The bit set is the one heap allocation `self` owns before `deinit` can @@ -467,7 +475,7 @@ pub const Pager = struct { /// recycling hands back. pub inline fn page_mut(self: *Pager, p: u32) *align(page_size) [page_size]u8 { assert_msg(p < self.mapped_pages, "write to a page past the mapped end of the data file"); - if (track_dirty) self.dirty.put(self.gpa, p, {}) catch {}; + self.note_dirty(p, p); return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } @@ -569,7 +577,7 @@ pub const Pager = struct { inline fn page_mut_slot(self: *Pager, p: u32) *align(page_size) [page_size]u8 { assert(p == page_watermark_a or p == page_watermark_b); assert_msg(p < self.mapped_pages, "write to a watermark slot past the mapped end"); - if (track_dirty) self.dirty.put(self.gpa, p, {}) catch {}; + self.note_dirty(p, p); return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } @@ -589,13 +597,21 @@ pub const Pager = struct { pub inline fn bytes_mut(self: *Pager, off: u64, len: usize) []u8 { assert_msg(off + len <= @as(u64, self.mapped_pages) << page_shift, "write past the mapped end of the data file"); if (track_dirty) { - var pg_i: u32 = @intCast(off >> page_shift); - const last: u32 = @intCast((off + len - 1) >> page_shift); - while (pg_i <= last) : (pg_i += 1) self.dirty.put(self.gpa, pg_i, {}) catch {}; + self.note_dirty(@intCast(off >> page_shift), @intCast((off + len - 1) >> page_shift)); } return self.reserve[@intCast(off)..][0..len]; } + /// Record pages `first..=last` as written since the last sync. See + /// `dirty_lock` for why this is the one shared structure on the write path. + inline fn note_dirty(self: *Pager, first: u32, last: u32) void { + if (!track_dirty) return; + self.dirty_lock.lockUncancelable(self.io); + defer self.dirty_lock.unlock(self.io); + var p = first; + while (p <= last) : (p += 1) self.dirty.put(self.gpa, p, {}) catch {}; + } + // -- allocation --------------------------------------------------------- /// Hand out `n` contiguous pages, growing the file if needed. For callers @@ -778,7 +794,11 @@ pub const Pager = struct { try std.posix.msync(self.reserve[0..len], std.posix.MSF.SYNC); } try self.file.sync(self.io); - if (track_dirty) self.dirty.clearRetainingCapacity(); + if (track_dirty) { + self.dirty_lock.lockUncancelable(self.io); + defer self.dirty_lock.unlock(self.io); + self.dirty.clearRetainingCapacity(); + } } /// Make the published image read-only at the hardware level. See -- 2.39.5 From 332206e5dd65696e5640f87f853349b57688fb97 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 12:30:07 +0300 Subject: [PATCH 11/37] db: the slab counts what the appender skips `slab_used` only ever grew by a document's length, so the two places the appender writes slab off went uncounted: the gap left when a checkpoint freezes the page the cursor points into and the cursor resumes at the next system page, and the tail of an extent abandoned for a document that no longer fits. Both are real garbage -- only a rebuild gets them back -- and both were invisible to the trigger that decides whether a rebuild is worth doing. An abandoned tail can be most of 8 MiB. `note_skip` counts them into `slab_used` where they happen and hands the number back for the caller to charge to the engine, which keeps the identity the last commit established: `dead_bytes` is the sum of `slab_used - live_bytes` over the collections that exist. That identity is also why `compact` no longer zeroes `dead_bytes`. A repack appends through the same slab, so it abandons a tail of its own whenever the next document does not fit; zeroing was true only if a rebuild leaves nothing behind, and it does not. `sum_dead_bytes` recomputes it from the collections, each under its own lock. Carried with it, because this commit is what exposed it: the four engine counters get a lock of their own. They are the only engine-wide mutable state a writer touches while holding nothing but its own collection's lock, so two writers on different collections reach them with no lock in common -- and the checkpoint's consistency check read them ordered against nothing, while the per-collection figures it compares them to were read under each collection's lock. Before this commit `dead_bytes` moved on nearly every write, so that check was skipped almost every time; with skips counted it stands still between checkpoints, the check runs, and it aborted three of eight ReleaseSafe runs of "a checkpoint runs alongside writers on several collections". Not mutation-checked, and worth saying so: reverting the lock did not re-trigger the abort in 24 further runs, and neither did the exact pre-fix revision in 10. The rate depends on machine load, and a mutation check that cannot be relied on to go red is not a check. The lock stands on inspection instead -- an unsynchronized read-modify-write on a counter shared by threads holding no common lock is a defect whatever its rate -- and the concurrency test now asserts the identity once everything is quiet, which is the half of it that does not depend on a race being caught in the act. Mutation-checked, each red on its own: drop either `note_skip` call in `slab_reserve`; put `self.dead_bytes = 0;` back in `compact`. The `note_skip` in `slab_append` is covered by the concurrency test, the only place a publish lands between a reservation and its append. 165/165 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz, e2e 49, e2e2 concurrent 2 + crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/db.zig | 383 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 340 insertions(+), 43 deletions(-) diff --git a/src/db.zig b/src/db.zig index 461d6f6..e4808c9 100644 --- a/src/db.zig +++ b/src/db.zig @@ -155,7 +155,10 @@ pub const Collection = struct { /// record is durable, where failure has nowhere to go: the write is already /// committed and reporting an error for it would be a lie the next open /// contradicts. Reserving first keeps the fallible half before the log. - fn slab_reserve(self: *Collection, gpa: std.mem.Allocator, len: usize) !void { + /// + /// Returns the slab bytes it wrote off along the way, for the caller to + /// charge to the engine's dead total. See `note_skip`. + fn slab_reserve(self: *Collection, gpa: std.mem.Allocator, len: usize) !u64 { // A checkpoint can land in the middle of an extent, which freezes the // page the tail points into. Appending there would store inside the // durable image, so abandon the rest of the extent and start a fresh @@ -171,7 +174,7 @@ pub const Collection = struct { // fit. Without that the append's own re-check could discover it needs a // fresh extent, which is fallible, after the log record is already // durable. Costs under one system page per extent. - if (self.pager.is_unpublished_at(self.slab_tail) and self.appendable_end(len)) return; + if (self.pager.is_unpublished_at(self.slab_tail) and self.appendable_end(len)) return 0; // The page holding the tail is frozen, but the *rest* of the extent is // not: nothing above the live cursor is referenced by the image or by an // index. So skip to the next system page and keep the extent, instead of @@ -187,10 +190,18 @@ pub const Collection = struct { // real data. if (self.appendable_end(len)) { const resumed = std.mem.alignForward(u64, self.slab_tail, pgr.map_align); + const skipped = self.note_skip(resumed - self.slab_tail); self.pager.mark_appendable(resumed, self.slab_end); self.slab_tail = resumed; - return; + return skipped; } + // Nothing will ever be written between the cursor and the end of the + // extent this collection is walking away from -- the extent stays + // allocated to it and every byte above the cursor is unreachable. That + // is garbage, in the whole 8 MiB, and the reservation is the only place + // that knows about it. + assert_msg(self.slab_tail <= self.slab_end, "the slab cursor is past the end of its extent"); + const skipped = self.note_skip(self.slab_end - self.slab_tail); // A document larger than the standard extent gets one of its own; BSON // reaches 16 MB and the extent is 8 MiB. const want_pages: u32 = @intCast(@max( @@ -202,6 +213,26 @@ pub const Collection = struct { try self.slab_extents.append(gpa, .{ .first = first, .pages = want_pages }); self.slab_tail = @as(u64, first) << pgr.page_shift; self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift); + return skipped; + } + + /// Count `bytes` of slab that no document will ever occupy, and hand the + /// same number back so the caller can charge the engine's dead total. + /// + /// The appender skips slab in two places -- rounding the cursor up to a + /// system page after a checkpoint froze the page it pointed into, and + /// abandoning the tail of an extent that no longer has room. Neither used to + /// be counted anywhere: not in `slab_used`, which only ever grew by a + /// document's length, and so not in the engine's `dead_bytes` either. It is + /// garbage all the same -- only a rebuild gets it back -- and it was + /// invisible to the trigger that decides whether a rebuild is worth doing. + /// + /// Two collections churning against a checkpoint every 32 MiB skip up to a + /// system page each per checkpoint, and an abandoned extent tail can be + /// most of 8 MiB. Counted here, that garbage arms compaction like any other. + fn note_skip(self: *Collection, bytes: u64) u64 { + self.slab_used += bytes; + return bytes; } /// Whether a document of `len` bytes fits in this extent even if the cursor @@ -211,9 +242,13 @@ pub const Collection = struct { return std.mem.alignForward(u64, self.slab_tail, pgr.map_align) + len <= self.slab_end; } + /// Where a document landed, and what the landing cost besides its own + /// length. `skipped` is `note_skip`'s tally for this append. + const Appended = struct { off: u64, skipped: u64 }; + /// Copy `bytes` into the slab and return its absolute file offset. /// Infallible: slab_reserve must have run for at least this many bytes. - fn slab_append(self: *Collection, bytes: []const u8) u64 { + fn slab_append(self: *Collection, bytes: []const u8) Appended { // The cursor was checked in `slab_reserve`, but a checkpoint can have // published since -- the reservation runs before the log append and this // runs after it, with an fsync in between. `publish` clears the whole @@ -227,8 +262,10 @@ pub const Collection = struct { // publish for the rest of this function, so the answer stays true. self.pager.lock_append(); defer self.pager.unlock_append(); + var skipped: u64 = 0; if (!self.pager.is_unpublished_at(self.slab_tail)) { const resumed = std.mem.alignForward(u64, self.slab_tail, pgr.map_align); + skipped = self.note_skip(resumed - self.slab_tail); self.pager.mark_appendable(resumed, self.slab_end); self.slab_tail = resumed; } @@ -243,7 +280,7 @@ pub const Collection = struct { // Here rather than at the call site: a rebuild appends through this same // path, and its copies are live by definition. self.live_bytes += bytes.len; - return off; + return .{ .off = off, .skipped = skipped }; } /// The canonical bytes of the document stored at `off` — a slice into a @@ -352,6 +389,24 @@ pub const Engine = struct { /// 16 KiB documents and one of 40 B documents look identical. live_bytes: u64 = 0, dead_bytes: u64 = 0, + /// Guards the four counters above -- `live_docs`, `dead_docs`, + /// `live_bytes`, `dead_bytes` -- and nothing else. + /// + /// They are the only engine-wide mutable state a writer touches while + /// holding nothing but its own collection's lock, so two writers on + /// different collections reach them with no lock in common. The lost update + /// that allows is the smaller half of the problem. The larger half is that a + /// reader had no way to see them consistently with the per-collection totals + /// they are supposed to equal: `checkpoint` reads each collection's counters + /// under that collection's lock -- which orders it against that + /// collection's writer -- and then read these with no lock at all, so it + /// could see a total that predated a write it had just serialized. Its own + /// assertion then aborted the server, correctly, about a database that was + /// consistent. + /// + /// A leaf: nothing else is taken while it is held, and it is never held + /// across an append, an fsync, or an allocation. + counter_lock: std.Io.Mutex = .init, /// The checkpoint's own page promise, for the catalog and free-list pages it /// writes. Separate from any collection's for the same reason those are /// separate from each other. @@ -489,10 +544,40 @@ pub const Engine = struct { self.log.close(); } + /// Slab a reservation wrote off, on the engine's books. Its own acquisition + /// rather than the reservation's caller adding to the field: skips cluster + /// at a checkpoint -- `publish` freezes every collection's append cursor at + /// once, so the next write to each of them skips -- which is precisely when + /// several writers reach this counter at the same moment. + fn count_slab_skip(self: *Engine, bytes: u64) void { + if (bytes == 0) return; + self.counter_lock.lockUncancelable(self.io); + defer self.counter_lock.unlock(self.io); + self.dead_bytes += bytes; + } + + /// A snapshot of the four counters, taken together. Both readers reason + /// about a *relation* -- the compaction trigger about the ratio of two of + /// them, the checkpoint about how they compare to the sum over collections + /// -- so reading them one at a time would be comparing two moments. + const Counters = struct { live_docs: u64, dead_docs: u64, live_bytes: u64, dead_bytes: u64 }; + + fn counters(self: *Engine) Counters { + self.counter_lock.lockUncancelable(self.io); + defer self.counter_lock.unlock(self.io); + return .{ + .live_docs = self.live_docs, + .dead_docs = self.dead_docs, + .live_bytes = self.live_bytes, + .dead_bytes = self.dead_bytes, + }; + } + /// Free every document in a collection along with its owned _id keys /// and secondary indexes (whose entries alias the documents — freed /// first). fn free_collection(self: *Engine, coll: *Collection) void { + self.counter_lock.lockUncancelable(self.io); // Dropping a collection turns all of its records into garbage. The // engine's live count includes every collection's documents, so it can // never be smaller than this one's -- and a u64 underflow here would @@ -525,6 +610,7 @@ pub const Engine = struct { "dropping a collection would underflow the engine's dead bytes", ); self.dead_bytes -= coll_dead; + self.counter_lock.unlock(self.io); coll.id_index.deinit(self.gpa); for (coll.indexes.items) |ix| { ix.deinit(self.gpa); @@ -575,10 +661,21 @@ pub const Engine = struct { for (coll.indexes.items) |ix| self.pager.release_reservation(&ix.hold); } + /// A document becoming live: its bytes into the slab, and every engine + /// counter that describes. The document count moved here from the two call + /// sites so that one document costs one acquisition of `counter_lock` and + /// leaves the totals agreeing at every moment a reader could look. fn publish_doc_bytes(self: *Engine, coll: *Collection, bytes: []const u8) u64 { - const off = coll.slab_append(bytes); + const appended = coll.slab_append(bytes); + self.counter_lock.lockUncancelable(self.io); + defer self.counter_lock.unlock(self.io); self.live_bytes += bytes.len; - return off; + self.live_docs += 1; + // Slab the append had to write off. Charged here, under the same + // collection lock the append ran under, so a checkpoint's catalog walk + // never sees the collection's total moved and the engine's not. + self.dead_bytes += appended.skipped; + return appended.off; } fn evict_doc(self: *Engine, coll: *Collection, id_enc: []const u8) void { @@ -589,12 +686,16 @@ pub const Engine = struct { coll.id_index.remove_doc(self.gpa, old_bytes, off); for (coll.indexes.items) |ix| ix.remove_doc(self.gpa, old_bytes, off); // This document's log record (and its slab bytes) just became garbage. - assert_msg(self.live_docs >= 1, "evicting a document would underflow the engine's live count"); - self.live_docs -= 1; - self.dead_docs += 1; - assert_msg(self.live_bytes >= old_bytes.len, "evicting a document would underflow the engine's live bytes"); - self.live_bytes -= old_bytes.len; - self.dead_bytes += old_bytes.len; + { + self.counter_lock.lockUncancelable(self.io); + defer self.counter_lock.unlock(self.io); + assert_msg(self.live_docs >= 1, "evicting a document would underflow the engine's live count"); + self.live_docs -= 1; + self.dead_docs += 1; + assert_msg(self.live_bytes >= old_bytes.len, "evicting a document would underflow the engine's live bytes"); + self.live_bytes -= old_bytes.len; + self.dead_bytes += old_bytes.len; + } assert_msg(coll.doc_count >= 1, "evicting a document would underflow the collection's count"); coll.doc_count -= 1; assert_msg(coll.live_bytes >= old_bytes.len, "evicting a document would underflow the collection's live bytes"); @@ -951,7 +1052,7 @@ pub const Engine = struct { for (built_list.items) |*b| { try b.ix.reserve_for(self.gpa, b.built.entries.items); } - try coll.slab_reserve(self.gpa, doc_bytes.len); + self.count_slab_skip(try coll.slab_reserve(self.gpa, doc_bytes.len)); // 5. Log (and sync) before anything becomes visible. The append // takes the log lock; durability (fsync) is the command's commit. @@ -963,7 +1064,6 @@ pub const Engine = struct { // 7. Publish the document and its entries: copy the bytes into the // slab and record the offset. Infallible from here. const off = self.publish_doc_bytes(coll, doc_bytes); - self.live_docs += 1; coll.doc_count += 1; for (built_list.items) |*b| { if (b.built.multikey) b.ix.multikey = true; @@ -1359,15 +1459,20 @@ pub const Engine = struct { // the live data, with a trigger that had been dead since the log started // being reclaimed. // + // One snapshot, because the second gate is a ratio: reading the two + // totals separately compares a live figure from one moment against a + // dead figure from another, and under concurrent writers that is how a + // rebuild fires on a database that does not want one. + const c = self.counters(); // Absolute volume first: a rewrite costs a full copy of the live data, // so it is not worth doing for a few kilobytes however bad the ratio. - if (self.dead_bytes < self.compact_threshold) return; + if (c.dead_bytes < self.compact_threshold) return; // Then the share, dead / (live + dead), firing at ~20%: the file stays // near 1.25x the live data and each rebuild is paid for by the space it // reclaims. Bytes rather than document counts, because a rewrite copies // bytes -- 100k evicted 40 B documents are not worth the same rebuild as // 100k evicted 16 KiB ones. - if (self.dead_bytes * 4 < self.live_bytes) return; + if (c.dead_bytes * 4 < c.live_bytes) return; self.compact_pending.store(true, .release); } @@ -1433,12 +1538,21 @@ pub const Engine = struct { }; } } + // Before releasing the catalog, and whether the walk finished or gave up + // part way: a rebuild resets a collection's totals, so an incremental + // `dead_bytes` now describes collections that no longer exist in that + // shape. It used to be zeroed here, which was true only if a repack + // leaves nothing behind -- it does not. A checkpoint landing mid-rebuild + // forces the copy's own append cursor up to a system page, and those + // skipped bytes are as dead as the ones being reclaimed. + const dead_after = self.sum_dead_bytes(); self.catalog_lock.unlockShared(self.io); + self.counter_lock.lockUncancelable(self.io); + self.dead_bytes = dead_after; + self.dead_docs = 0; + self.counter_lock.unlock(self.io); if (rebuild_err) |err| return err; - self.dead_docs = 0; - // Every collection's slab was just repacked to hold only live bytes. - self.dead_bytes = 0; // Publish the rebuilt layout, which is also what reclaims the log. Until // this lands the old pages are still referenced by the previous // watermark, so a crash mid-rebuild simply loses the rebuild. @@ -1463,6 +1577,7 @@ pub const Engine = struct { /// Documents and indexes have to move together: an index leaf holds a /// physical offset, so a document that moves without its indexes being /// rebuilt is a stale entry pointing at whatever now occupies those bytes. + /// fn rebuild_collection(self: *Engine, coll: *Collection) !void { try coll.lock.lock(self.io); defer coll.lock.unlock(self.io); @@ -1489,10 +1604,13 @@ pub const Engine = struct { var it = coll.id_index.iter(); while (it.next()) |entry| { const bytes = doc_bytes_in(self.pager, entry.off); - try coll.slab_reserve(self.gpa, bytes.len); - const new_off = coll.slab_append(bytes); + // The skips are dropped rather than charged: this collection's + // totals were reset above and `compact` recomputes the engine's from + // what the rebuild leaves behind. + _ = try coll.slab_reserve(self.gpa, bytes.len); + const appended = coll.slab_append(bytes); self.pager.release_reservation(&coll.hold); - try moved.append(self.gpa, .{ .off = new_off }); + try moved.append(self.gpa, .{ .off = appended.off }); } // Republish the offsets. @@ -1510,6 +1628,34 @@ pub const Engine = struct { coll.layout_epoch = self.layout_epoch_seq; } + /// The engine's dead-byte total, recomputed from the collections that + /// exist. Everywhere else the counter is incremental; a rebuild is the one + /// place that has to reset it, and the answer after a rebuild is not zero -- + /// the copying skips slab of its own whenever a checkpoint lands mid-walk. + /// + /// Each collection is read under its own lock, catalog then collection: the + /// order `compact` and `write_catalog` both use. Uncancelable, because the + /// caller has already rewritten the collections and the counter describing + /// them cannot be left behind. + fn sum_dead_bytes(self: *Engine) u64 { + var sum: u64 = 0; + var db_it = self.dbs.iterator(); + while (db_it.next()) |db_entry| { + var coll_it = db_entry.value_ptr.collections.iterator(); + while (coll_it.next()) |ce| { + const coll = ce.value_ptr.*; + coll.lock.lockSharedUncancelable(self.io); + defer coll.lock.unlockShared(self.io); + assert_msg( + coll.slab_used >= coll.live_bytes, + "a collection cannot hold more live bytes than it ever appended", + ); + sum += coll.slab_used - coll.live_bytes; + } + } + return sum; + } + fn repack_index( self: *Engine, coll: *Collection, @@ -1691,7 +1837,7 @@ pub const Engine = struct { var dead_sum: u64 = 0; try put_u32(gpa, out, catalog_magic); try put_u32(gpa, out, catalog_version); - try put_u64(gpa, out, self.live_docs); + try put_u64(gpa, out, self.counters().live_docs); try put_u32(gpa, out, @intCast(self.dbs.count())); var db_it = self.dbs.iterator(); while (db_it.next()) |db_entry| { @@ -1962,8 +2108,7 @@ pub const Engine = struct { buf.clearRetainingCapacity(); try self.catalog_lock.lockShared(self.io); const snapshot_seq = self.seq; - const live_before = self.live_bytes; - const dead_before = self.dead_bytes; + const before = self.counters(); const sums = self.write_catalog(&buf) catch |err| { self.catalog_lock.unlockShared(self.io); return err; @@ -2010,15 +2155,24 @@ pub const Engine = struct { // is the right trade, because what it guards against is a code path // that updates one level and not the other, and that is // deterministic wherever it exists. - if (self.live_bytes == live_before) assert_msg( - sums.live == live_before, + // + // Both reads go through `counters`, which is the only way the + // comparison means anything: the sums were gathered under each + // collection's lock, which orders them against that collection's + // writer, and an unsynchronized read of the engine's own totals is + // ordered against nothing at all -- so it could return a figure from + // before a write the walk had already serialized, and the assertion + // would abort a server whose accounting was correct. + const after = self.counters(); + if (after.live_bytes == before.live_bytes) assert_msg( + sums.live == before.live_bytes, "the engine's live-byte total must equal the sum over collections", ); // The same argument, for the total the rebuild trigger reads. This // is what makes a drop's accounting checkable: charge the engine for // a dropped collection's bytes and the two sides part company here. - if (self.dead_bytes == dead_before) assert_msg( - sums.dead == dead_before, + if (after.dead_bytes == before.dead_bytes) assert_msg( + sums.dead == before.dead_bytes, "the engine's dead-byte total must equal the sum over collections", ); const pages: u32 = @intCast((buf.items.len + pgr.page_size - 1) / pgr.page_size); @@ -2037,8 +2191,8 @@ pub const Engine = struct { .seq = snapshot_seq, .catalog_page = first, .catalog_len = buf.items.len, - .live_docs = self.live_docs, - .dead_bytes = self.dead_bytes, + .live_docs = after.live_docs, + .dead_bytes = after.dead_bytes, }) catch |err| { self.log_lock.unlock(self.io); return err; @@ -2195,9 +2349,8 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an self.evict_doc(coll, id_enc); const doc_bytes = try serialize_doc(self.gpa, doc); defer self.gpa.free(doc_bytes); - try coll.slab_reserve(self.gpa, doc_bytes.len); + self.count_slab_skip(try coll.slab_reserve(self.gpa, doc_bytes.len)); const off = self.publish_doc_bytes(coll, doc_bytes); - self.live_docs += 1; coll.doc_count += 1; // The `_id_` entry is added *now*, not after replay: it is the only // way the next record can find this document to supersede it. The @@ -2787,7 +2940,14 @@ test "compaction still triggers after a checkpoint has truncated the log" { try engine.commit(); try testing.expectEqual(live_after_load + 200, engine.live_bytes); - try testing.expectEqual(live_after_load, engine.dead_bytes); + // Every superseded document, plus what the first replace after the + // checkpoint had to skip: that checkpoint froze the page the append cursor + // pointed into, so the cursor moved up to the next system page and the gap + // it stepped over is garbage too. Both are in the same total, which is the + // point -- the trigger reads one number. + const superseded = live_after_load; + try testing.expect(engine.dead_bytes >= superseded); + try testing.expect(engine.dead_bytes - superseded < pgr.map_align); try testing.expect(engine.dead_bytes >= engine.compact_threshold); try testing.expect(engine.take_compact()); @@ -2904,6 +3064,136 @@ test "reopen carries the garbage counter across a restart" { try testing.expectEqual(reopened.slab_used - reopened.live_bytes, engine2.dead_bytes); } +test "the slab counts what the appender skips" { + // `slab_used` only ever grew by a document's length, so the two places the + // appender writes slab off went uncounted: the gap left when a checkpoint + // freezes the page the cursor points into and the cursor moves up to the + // next system page, and the tail of an extent abandoned for a document that + // no longer fits. Real garbage -- only a rebuild gets it back -- and + // invisible to the trigger that decides whether a rebuild is worth doing. + // + // Mutation: drop either `note_skip` call in `slab_reserve`. Red on the + // matching half below. Dropping the one in `slab_append` is not covered + // here: it needs a publish between the reservation and the append, which is + // what "a checkpoint runs alongside writers on several collections" + // arranges, and the accounting assertion in `checkpoint` is what catches it. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // no rebuild mid-test + try engine.lock(); + defer engine.unlock(); + + for (0..200) |i| { + var d = try make_doc(gpa, @intCast(i), "x"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + const coll = engine.get_collection("app", "c").?; + // Nothing is dead yet: every document inserted is still live and the slab + // has been walked straight through. + try testing.expectEqual(@as(u64, 0), engine.dead_bytes); + try testing.expectEqual(coll.slab_used, coll.live_bytes); + + // 1. The round-up. The checkpoint freezes the page the cursor is in, so the + // next append resumes at the next system page and the bytes in between + // are never written. + try engine.checkpoint(); + const tail_before = coll.slab_tail; + const gap = std.mem.alignForward(u64, tail_before, pgr.map_align) - tail_before; + try testing.expect(gap > 0); + { + var d = try make_doc(gpa, 1000, "x"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + try testing.expectEqual(gap, engine.dead_bytes); + try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); + + // 2. The abandoned tail. A document bigger than the standard extent takes + // one of its own, and everything left in the extent being walked away + // from is unreachable -- the extent stays allocated to this collection. + const abandoned = coll.slab_end - coll.slab_tail; + try testing.expect(abandoned > 4 * 1024 * 1024); + { + const big = try gpa.alloc(u8, 9 * 1024 * 1024); + defer gpa.free(big); + @memset(big, 'z'); + var d = try make_doc(gpa, 1001, big); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + try testing.expectEqual(gap + abandoned, engine.dead_bytes); + try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); + + // And it survives the round trip, because it is in `slab_used`. + try engine.checkpoint(); + try testing.expectEqual(gap + abandoned, engine.dead_bytes); +} + +test "a rebuild leaves behind what its own copying skipped" { + // `compact` used to set `dead_bytes = 0`, on the reasoning that a repack + // holds only live bytes. It does not: the repack appends through the same + // slab, so it abandons an extent tail whenever the next document no longer + // fits. Zeroing the counter there broke the identity the checkpoint asserts + // -- and understated the garbage, so a collection that fragments on every + // rebuild would never be rebuilt again. + // + // Mutation: put `self.dead_bytes = 0;` back. Red below, and the checkpoint + // inside `compact` aborts on the accounting assertion before it gets there. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // rebuild only when told to + try engine.lock(); + defer engine.unlock(); + + // Documents big enough that two of them do not fit in one 8 MiB extent, so + // the copying itself has to abandon a tail. + const big = try gpa.alloc(u8, 5 * 1024 * 1024); + defer gpa.free(big); + @memset(big, 'z'); + for (0..3) |i| { + var d = try make_doc(gpa, @intCast(i), big); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + + // One of them becomes garbage, which is what the rebuild is for. + try testing.expect(try engine.remove_by_id("app", "c", .{ .int32 = 1 })); + try engine.commit(); + + try engine.compact(); + + const coll = engine.get_collection("app", "c").?; + try testing.expectEqual(@as(u64, 2), coll.doc_count); + // The deleted document is gone from the slab, but the gap the copy left + // between the two survivors is not -- and the engine says so. + try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); + try testing.expect(engine.dead_bytes > 2 * 1024 * 1024); + try testing.expect(engine.dead_bytes < 4 * 1024 * 1024); +} + test "dropping a collection does not arm compaction" { // `free_collection` charged the engine's `dead_bytes` with the dropped // collection's *live* bytes, having just handed every page it owned back to @@ -3146,14 +3436,6 @@ test "a checkpoint runs alongside writers on several collections" { var doc = make_doc(alloc, @intCast(i), "user") catch return error.Canceled; defer doc.deinit(); { - // The server's discipline, not the legacy whole-engine - // lock: catalog shared, then the target collection - // exclusive (commands.zig dispatch). `write_catalog` takes - // the same two in the same order, and that is the whole - // reason its walk of a collection's counters and extents is - // safe -- a writer that skipped the collection lock would - // not be excluded by it, and the test would be checking - // nothing. e.lock_catalog(false) catch return error.Canceled; defer e.unlock_catalog(false); const coll = (e.lock_collection("app", name, true, true) catch @@ -3190,10 +3472,25 @@ test "a checkpoint runs alongside writers on several collections" { try engine.checkpoint(); try engine.lock_read(); defer engine.unlock_read(); + var live_sum: u64 = 0; + var dead_sum: u64 = 0; + var docs_sum: u64 = 0; for (colls) |name| { const coll = engine.get_collection("app", name) orelse return error.TestUnexpectedResult; try testing.expectEqual(@as(usize, @intCast(per_coll)), coll.id_index.count()); + live_sum += coll.live_bytes; + dead_sum += coll.slab_used - coll.live_bytes; + docs_sum += coll.doc_count; } + // The engine's counters are the sums over collections, checked once + // everything is quiet rather than left to `checkpoint`'s own assertion -- + // which only runs on a checkpoint that happened to fall in a gap between + // writes, so under sustained load it can go a whole run without firing. + // Every writer here held nothing but its own collection's lock while moving + // these, so a lost update lands exactly as a mismatch on one of these three. + try testing.expectEqual(live_sum, engine.live_bytes); + try testing.expectEqual(dead_sum, engine.dead_bytes); + try testing.expectEqual(docs_sum, engine.live_docs); } test "concurrent readers and writers on a threaded Io" { -- 2.39.5 From e66030f43e86bfd18c8c58216c4096aa9b91a483 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 12:31:56 +0300 Subject: [PATCH 12/37] plan: the eight bugs cleared before the free list Four of them were not in the plan that started this work -- they were found by tests written for the three that were, which is M0's gate lesson arriving a milestone early. The record has to match what happened, or the next session reads a milestone that looks like it went as designed. Also records what nobody is fixing yet: `rebuild_collection` frees pages under only the collection's lock while a concurrent checkpoint may have snapshotted a catalog that claims them, and the `seq` retry cannot see it because a rebuild appends no log record. Written down so the free list does not add a second instance of the same shape. --- PLAN.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 1 deletion(-) diff --git a/PLAN.md b/PLAN.md index e0f2270..713fd7c 100644 --- a/PLAN.md +++ b/PLAN.md @@ -631,6 +631,96 @@ The pattern worth noting for M1: both were found by *running* the suites, not by reading them, and the second was only visible because the first stopped masking it. +### Before the free list: eight bugs the M1 design work turned up + +The doc-level free list multiplies traffic through exactly the reclamation +paths, so those paths were read closely before anything was built. Three of the +eight were found that way, by reading. Five were found by the tests written for +the other three — the same lesson M0's gate taught, arriving one milestone +early: a reclamation bug is invisible until something runs long enough, or +concurrently enough, to reach the state that exposes it. + +1. **`write_freelist` counted its entries before allocating its own pages.** + The allocation goes through `take_free`, which `swapRemove`s an exact-fit + entry, so the loop wrote one entry fewer than the count it had already + committed to and the hash landed eight bytes short. `read_freelist` then + declared the list corrupt and dropped all of it. A one-page freelist stream + and a one-page hole are both the common case, so this fired at essentially + every reopen: the free list has been discarded on restart since it existed. + +2. **The free lists were read and written with no lock.** `free_pages` appended + to `free_pending` while `publish` rotated the three lists under `alloc_lock`, + and `write_freelist` walked all three while a concurrent `free_pages` could + reallocate them. Found by the test written for (1). + +3. **A checkpoint never gave back the generation it replaced.** The catalog + stream and the freelist stream are allocated fresh every publish and were + never freed, so an idle server grew forever. + +4. **A checkpoint could publish a watermark above the durable log tail.** The + snapshot's `seq` check does not catch a writer that appended *before* the + walk started and has not committed yet; the window is as wide as an fsync. + This was an assertion, so the failure mode was a server abort under exactly + the load that makes checkpoints frequent — and without the assertion it is + the loss of an acknowledged write, since the truncation that follows a + checkpoint would discard the record. Now a retry: seal and re-snapshot. + +5. **A document append could land in the published image.** `slab_reserve` + checks that the append cursor is writable; `publish` can clear the + unpublished set between that check and `slab_append`'s copy, because the log + append and its fsync sit in between. SIGBUS where `protect_stable` is + compiled in, a silent overwrite of durable data in ReleaseFast, where it is + not. Fixed with a pager-level append lock, held shared by appenders and + exclusively by `publish`; measured at no cost on the write path (8 clients × + 1500 inserts at `{w:1,j:true}`: 23779–24879 docs/s before, 23823–24452 + after). + +6. **`write_catalog` read `slab_extents` under only the shared catalog lock**, + while `slab_reserve` appended to that ArrayList under the collection's. + +7. **The engine's counters were shared by writers holding no lock in common.** + `live_docs`, `dead_docs`, `live_bytes` and `dead_bytes` are updated by a + writer holding its own collection's lock and the catalog's shared — so two + writers on different collections lose each other's updates, and a reader had + no way to see the totals consistently with the per-collection figures they + are supposed to equal. Now `counter_lock`, a leaf, with a `Counters` + snapshot for the two readers that compare them. The only one of the eight + that is **not** mutation-checked: it aborted three of eight ReleaseSafe runs + once (8) made the checkpoint's consistency check reachable, and then would + not re-trigger in 34 further runs, on the reverted fix and on the pre-fix + revision alike. The rate depends on machine load. It stands on inspection, + and the concurrency test now asserts the identity once everything is quiet + rather than relying on catching the race in the act. + +8. **The slab did not count what the appender skips.** `slab_used` only ever + grew by a document's length, so the gap left when a checkpoint pushes the + append cursor up to a system page, and the tail of an extent abandoned for a + document that no longer fits, were counted nowhere — real garbage, + invisible to the trigger that decides whether a rebuild is worth doing, and + part of the 1.65×/2.47× the churn gate measured. + +Accounting is now an identity rather than four independent counters: +`dead_bytes` is the sum of `slab_used - live_bytes` over the collections that +exist. `read_catalog` recomputes it on open instead of trusting the watermark's +hint, `compact` recomputes it from what a rebuild leaves behind instead of +zeroing it, and `write_catalog` returns both sums for the checkpoint to assert +under the quiescence condition it already had. That assertion is what surfaced +(7) and what mutation-checks (5) and (8). + +Two of the eight — the drop that charged a dropped collection's live bytes to +`dead_bytes`, and the counters — also changed what the watermark is for: its +`dead_bytes` field is now a hint for anything inspecting the header, not a +source of truth, because a collection dropped after the last checkpoint is gone +from the catalog and would still be charged for in the hint. + +**Still open, deliberately.** `rebuild_collection` frees the pages it abandoned +while holding only the collection's lock, and a concurrent `checkpoint` may +already have snapshotted a catalog that claims them. The `seq` retry does not +see it, because a rebuild appends no log record. The fix is mutual exclusion +between `compact` and `checkpoint`; it is out of this scope because it wants +its own design pass, and because the free list must not add a second instance +of the same shape. + --- ## 6. Deferred designs (grill each at its milestone) @@ -670,7 +760,9 @@ it. caught by draining a collection being updated underneath. Still open in M1: the doc-level free list, sessions plumbing (`lsid` - accepted), and command-monitoring (`expectEvents`) in the spec runner. + accepted), and command-monitoring (`expectEvents`) in the spec runner. The + eight reclamation bugs above were cleared first, as preconditions for the + free list rather than as work of their own. **A prerequisite the free list must honour**, recorded here while it is still being designed: *an offset that was ever a record start must remain a record start.* `doc_bytes` reads a `u32` length prefix in place, so an -- 2.39.5 From afa5c6ef9db97c9306f08e420d8aa598f00b1e15 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:02:36 +0300 Subject: [PATCH 13/37] tests/spec: $$unsetOrMatches does not change root-ness `special()` passed a hard `false` for `root` into its recursion, so a value standing behind `$$unsetOrMatches` was matched as a nested document even when it sat at the top of an `expectResult`. The spec says the opposite in so many words -- "This operator does not influence whether or not an actual document value is considered a root-level document" (unified-test-format.md:2873, and :2821 for `$$matchesEntity`) -- and that distinction is the whole of the extra-key rule: only a root document may carry keys the expectation does not mention. `root` now threads through `match` -> `special` -> the recursion. From under a key it is always false, which is what it already was; from the top level it is whatever the caller had. 25 cases go from FAIL to pass and none moves the other way. Every one is the same shape -- an `expectResult` of `{$$unsetOrMatches: {acknowledged: false}}` against a driver write result that also carries its counts, or the `insertedId`/`insertedIds` forms of the same thing -- and every one was the runner failing a result the engine had got right. 168/124/195 becomes 193/99/195; the scorecard is rewritten here so the delta belongs to this change alone. Mutation-checked: put the `false` back in the `$$unsetOrMatches` arm and bulkWrite-deleteMany-hint-unacknowledged.json returns to 0 pass, 2 fail. The `$$matchesEntity` arm is the same one-word change on the same sentence of the spec, but the crud corpus does not use that operator once, so it rests on the spec text rather than on a red test. Two consecutive full runs, both 193/99/195, 175/175 files, 0 errored, no lingering timers. --- tests/spec/run.js | 14 +++++++---- tests/spec/scorecard.txt | 53 +++++++++++----------------------------- 2 files changed, 23 insertions(+), 44 deletions(-) diff --git a/tests/spec/run.js b/tests/spec/run.js index fda126b..1d604de 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -233,7 +233,7 @@ function scalarEqual(expected, actual) { // one: only a root document may carry keys the expectation does not mention. function match(expected, actual, entities, pathStr = '', root = true) { const sk = specialKey(expected); - if (sk) return special(sk, expected[sk], actual, entities, pathStr, true); + if (sk) return special(sk, expected[sk], actual, entities, pathStr, true, root); if (isPlainDoc(expected)) { if (!isPlainDoc(actual)) fail(pathStr, `expected a document, got ${describe(actual)}`); @@ -241,7 +241,7 @@ function match(expected, actual, entities, pathStr = '', root = true) { const kp = pathStr ? `${pathStr}.${k}` : k; const vsk = specialKey(v); if (vsk) { - const consumed = special(vsk, v[vsk], actual[k], entities, kp, Object.prototype.hasOwnProperty.call(actual, k)); + const consumed = special(vsk, v[vsk], actual[k], entities, kp, Object.prototype.hasOwnProperty.call(actual, k), false); if (!consumed) continue; continue; } @@ -268,7 +268,11 @@ function match(expected, actual, entities, pathStr = '', root = true) { } // Returns whether the actual value still has to be matched by the caller. -function special(op, arg, actual, entities, pathStr, present) { +// `root` is the root-ness of the value the operator stands in for: these +// operators wrap a value, they do not reposition it. A `$$unsetOrMatches` at +// the top of an `expectResult` still matches a root document, so the actual +// document may carry fields the expectation does not mention. +function special(op, arg, actual, entities, pathStr, present, root) { switch (op) { case '$$exists': if (arg && !present) fail(pathStr, 'expected the key to exist'); @@ -283,11 +287,11 @@ function special(op, arg, actual, entities, pathStr, present) { } case '$$unsetOrMatches': if (!present || actual === undefined) return false; - match(arg, actual, entities, pathStr, false); + match(arg, actual, entities, pathStr, root); return false; case '$$matchesEntity': { if (!(arg in entities.map)) fail(pathStr, `entity ${arg} not found`); - match(entities.map[arg], actual, entities, pathStr, false); + match(entities.map[arg], actual, entities, pathStr, root); return false; } case '$$matchesHexBytes': diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index a622491..1973557 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -13,7 +13,7 @@ # semantics; ignoring them makes some cases pass that a full runner would # fail, so treat `pass` as an upper bound until M1 wires events up. -total 168 pass 124 fail 195 skip 175 files 0 errored +total 193 pass 99 fail 195 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -31,28 +31,28 @@ bulkWrite-collation.json 0 2 0 bulkWrite-comment.json 2 0 1 bulkWrite-delete-hint-serverError.json 0 0 2 bulkWrite-delete-hint.json 2 0 0 -bulkWrite-deleteMany-hint-unacknowledged.json 0 2 2 +bulkWrite-deleteMany-hint-unacknowledged.json 2 0 2 bulkWrite-deleteMany-let.json 0 1 1 bulkWrite-deleteMany-rawdata.json 1 0 1 -bulkWrite-deleteOne-hint-unacknowledged.json 0 2 2 +bulkWrite-deleteOne-hint-unacknowledged.json 2 0 2 bulkWrite-deleteOne-let.json 0 1 1 bulkWrite-deleteOne-rawdata.json 1 0 1 bulkWrite-errorResponse.json 0 0 1 bulkWrite-insertOne-dots_and_dollars.json 3 1 1 bulkWrite-replaceOne-dots_and_dollars.json 2 1 1 -bulkWrite-replaceOne-hint-unacknowledged.json 0 2 0 +bulkWrite-replaceOne-hint-unacknowledged.json 2 0 0 bulkWrite-replaceOne-let.json 0 1 1 bulkWrite-replaceOne-rawdata.json 1 0 1 bulkWrite-replaceOne-sort.json 1 0 1 bulkWrite-update-hint.json 3 0 0 bulkWrite-update-validation.json 3 0 0 bulkWrite-updateMany-dots_and_dollars.json 0 0 4 -bulkWrite-updateMany-hint-unacknowledged.json 0 2 0 +bulkWrite-updateMany-hint-unacknowledged.json 2 0 0 bulkWrite-updateMany-let.json 0 1 1 bulkWrite-updateMany-pipeline.json 0 1 0 bulkWrite-updateMany-rawdata.json 0 1 1 bulkWrite-updateOne-dots_and_dollars.json 0 0 4 -bulkWrite-updateOne-hint-unacknowledged.json 0 2 0 +bulkWrite-updateOne-hint-unacknowledged.json 2 0 0 bulkWrite-updateOne-let.json 0 1 1 bulkWrite-updateOne-pipeline.json 0 1 0 bulkWrite-updateOne-rawdata.json 0 1 1 @@ -88,7 +88,7 @@ db-aggregate.json 0 2 0 deleteMany-collation.json 0 1 0 deleteMany-comment.json 2 0 1 deleteMany-hint-serverError.json 0 0 2 -deleteMany-hint-unacknowledged.json 0 2 2 +deleteMany-hint-unacknowledged.json 2 0 2 deleteMany-hint.json 2 0 0 deleteMany-let.json 0 1 1 deleteMany-rawdata.json 1 0 1 @@ -97,7 +97,7 @@ deleteOne-collation.json 0 1 0 deleteOne-comment.json 2 0 1 deleteOne-errorResponse.json 0 0 1 deleteOne-hint-serverError.json 0 0 2 -deleteOne-hint-unacknowledged.json 0 2 2 +deleteOne-hint-unacknowledged.json 2 0 2 deleteOne-hint.json 2 0 0 deleteOne-let.json 0 1 1 deleteOne-rawdata.json 1 0 1 @@ -149,18 +149,18 @@ findOneAndUpdate-pipeline.json 0 1 0 findOneAndUpdate-rawdata.json 0 1 1 findOneAndUpdate.json 5 3 0 insertMany-comment.json 2 0 1 -insertMany-dots_and_dollars.json 0 4 1 +insertMany-dots_and_dollars.json 3 1 1 insertMany-rawdata.json 1 0 1 -insertMany.json 2 1 0 +insertMany.json 3 0 0 insertOne-comment.json 2 0 1 -insertOne-dots_and_dollars.json 5 3 1 +insertOne-dots_and_dollars.json 6 2 1 insertOne-errorResponse.json 0 0 1 insertOne-rawdata.json 1 0 1 insertOne.json 1 0 0 replaceOne-collation.json 0 1 0 replaceOne-comment.json 2 0 1 replaceOne-dots_and_dollars.json 3 1 1 -replaceOne-hint-unacknowledged.json 0 2 0 +replaceOne-hint-unacknowledged.json 2 0 0 replaceOne-hint.json 2 0 0 replaceOne-let.json 0 1 1 replaceOne-rawdata.json 1 0 1 @@ -171,7 +171,7 @@ updateMany-arrayFilters.json 0 3 0 updateMany-collation.json 0 1 0 updateMany-comment.json 2 0 1 updateMany-dots_and_dollars.json 0 0 4 -updateMany-hint-unacknowledged.json 0 2 0 +updateMany-hint-unacknowledged.json 2 0 0 updateMany-hint.json 2 0 0 updateMany-let.json 0 1 1 updateMany-pipeline.json 0 1 0 @@ -183,7 +183,7 @@ updateOne-collation.json 0 1 0 updateOne-comment.json 2 0 1 updateOne-dots_and_dollars.json 0 0 4 updateOne-errorResponse.json 0 0 1 -updateOne-hint-unacknowledged.json 0 2 0 +updateOne-hint-unacknowledged.json 2 0 0 updateOne-hint.json 2 0 0 updateOne-let.json 0 1 1 updateOne-pipeline.json 0 1 0 @@ -220,15 +220,11 @@ bulkWrite-comment.json SKIP BulkWrite with comment - pre 4.4 needs server <= 4.2 bulkWrite-delete-hint-serverError.json SKIP * needs server <= 4.3.3 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] -bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] bulkWrite-deleteMany-let.json SKIP BulkWrite deleteMany with let option needs server >= 5.0 bulkWrite-deleteMany-let.json FAIL BulkWrite deleteMany with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteMany-rawdata.json SKIP BulkWrite deleteMany with rawData option needs server >= 8.2.0 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] -bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] bulkWrite-deleteOne-let.json SKIP BulkWrite deleteOne with let option needs server >= 5.0 bulkWrite-deleteOne-let.json FAIL BulkWrite deleteOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteOne-rawdata.json SKIP BulkWrite deleteOne with rawData option needs server >= 8.2.0 @@ -237,8 +233,6 @@ bulkWrite-insertOne-dots_and_dollars.json SKIP Inserting document with top-level bulkWrite-insertOne-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-dots_and_dollars.json SKIP Replacing document with dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-replaceOne-dots_and_dollars.json FAIL Replacing document with dollar-prefixed key in embedded doc on pre-5.0 server yields server-side error bulkWrite: expected an error, the operation succeeded -bulkWrite-replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint string on 4.2+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] -bulkWrite-replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint document on 4.2+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] bulkWrite-replaceOne-let.json SKIP BulkWrite replaceOne with let option needs server >= 5.0 bulkWrite-replaceOne-let.json FAIL BulkWrite replaceOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-rawdata.json SKIP BulkWrite replaceOne with rawData option needs server >= 8.2.0 @@ -247,8 +241,6 @@ bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-lev bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 -bulkWrite-updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint string on 4.2+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] -bulkWrite-updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint document on 4.2+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] bulkWrite-updateMany-let.json SKIP BulkWrite updateMany with let option needs server >= 5.0 bulkWrite-updateMany-let.json FAIL BulkWrite updateMany with let option unsupported (server-side error) bulkWrite: error message "update spec requires u" does not contain "'update.let' is an unknown field" bulkWrite-updateMany-pipeline.json FAIL UpdateMany in bulk write using pipelines MongoBulkWriteError: update spec requires u @@ -258,8 +250,6 @@ bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set top-leve bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 -bulkWrite-updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint string on 4.2+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] -bulkWrite-updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint document on 4.2+ server bulkWrite: unexpected extra keys ["insertedCount","matchedCount","modifiedCount","deletedCount","upsertedCount","upsertedIds","insertedIds"] bulkWrite-updateOne-let.json SKIP BulkWrite updateOne with let option needs server >= 5.0 bulkWrite-updateOne-let.json FAIL BulkWrite updateOne with let option unsupported (server-side error) bulkWrite: error message "update spec requires u" does not contain "'update.let' is an unknown field" bulkWrite-updateOne-pipeline.json FAIL UpdateOne in bulk write using pipelines MongoBulkWriteError: update spec requires u @@ -314,8 +304,6 @@ deleteMany-comment.json SKIP deleteMany with comment - pre 4.4 needs server <= 4 deleteMany-hint-serverError.json SKIP * needs server <= 4.3.3 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server deleteMany: unexpected extra keys ["deletedCount"] -deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server deleteMany: unexpected extra keys ["deletedCount"] deleteMany-let.json SKIP deleteMany with let option needs server >= 5.0 deleteMany-let.json FAIL deleteMany with let option unsupported (server-side error) deleteMany: expected an error, the operation succeeded deleteMany-rawdata.json SKIP deleteMany with rawData option needs server >= 8.2.0 @@ -325,8 +313,6 @@ deleteOne-errorResponse.json SKIP delete operations support errorResponse assert deleteOne-hint-serverError.json SKIP * needs server <= 4.3.3 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server deleteOne: unexpected extra keys ["deletedCount"] -deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server deleteOne: unexpected extra keys ["deletedCount"] deleteOne-let.json SKIP deleteOne with let option needs server >= 5.0 deleteOne-let.json FAIL deleteOne with let option unsupported (server-side error) deleteOne: expected an error, the operation succeeded deleteOne-rawdata.json SKIP deleteOne with rawData option needs server >= 8.2.0 @@ -408,24 +394,17 @@ findOneAndUpdate.json FAIL FindOneAndUpdate when no documents match with upsert insertMany-comment.json SKIP insertMany with comment - pre 4.4 needs server <= 4.2.99 insertMany-dots_and_dollars.json SKIP Inserting document with top-level dollar-prefixed key on 5.0+ server needs server >= 5.0 insertMany-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error insertMany: expected an error, the operation succeeded -insertMany-dots_and_dollars.json FAIL Inserting document with top-level dotted key insertMany: unexpected extra keys ["insertedCount"] -insertMany-dots_and_dollars.json FAIL Inserting document with dollar-prefixed key in embedded doc insertMany: unexpected extra keys ["insertedCount"] -insertMany-dots_and_dollars.json FAIL Inserting document with dotted key in embedded doc insertMany: unexpected extra keys ["insertedCount"] insertMany-rawdata.json SKIP insertMany with rawData option needs server >= 8.2.0 -insertMany.json FAIL InsertMany with non-existing documents insertMany: unexpected extra keys ["insertedCount"] insertOne-comment.json SKIP insertOne with comment - pre 4.4 needs server <= 4.2.99 insertOne-dots_and_dollars.json SKIP Inserting document with top-level dollar-prefixed key on 5.0+ server needs server >= 5.0 insertOne-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error insertOne: expected an error, the operation succeeded insertOne-dots_and_dollars.json FAIL Inserting document with dollar-prefixed key in _id yields server-side error insertOne: expected an error, the operation succeeded -insertOne-dots_and_dollars.json FAIL Unacknowledged write using dollar-prefixed or dotted keys may be silently rejected on pre-5.0 server insertOne: unexpected extra keys ["insertedId"] insertOne-errorResponse.json SKIP insert operations support errorResponse assertions runner: failPoint insertOne-rawdata.json SKIP insertOne with rawData option needs server >= 8.2.0 replaceOne-collation.json FAIL ReplaceOne when one document matches with collation replaceOne.matchedCount: expected 1, got 0 replaceOne-comment.json SKIP ReplaceOne with comment - pre 4.4 needs server <= 4.2.99 replaceOne-dots_and_dollars.json SKIP Replacing document with dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 replaceOne-dots_and_dollars.json FAIL Replacing document with dollar-prefixed key in embedded doc on pre-5.0 server yields server-side error replaceOne: expected an error, the operation succeeded -replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint string on 4.2+ server replaceOne: unexpected extra keys ["modifiedCount","upsertedId","upsertedCount","matchedCount"] -replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint document on 4.2+ server replaceOne: unexpected extra keys ["modifiedCount","upsertedId","upsertedCount","matchedCount"] replaceOne-let.json SKIP ReplaceOne with let option needs server >= 5.0 replaceOne-let.json FAIL ReplaceOne with let option unsupported (server-side error) replaceOne: expected an error, the operation succeeded replaceOne-rawdata.json SKIP ReplaceOne with rawData option needs server >= 8.2.0 @@ -439,8 +418,6 @@ updateMany-dots_and_dollars.json SKIP Updating document to set top-level dollar- updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 updateMany-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 -updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint string on 4.2+ server updateMany: unexpected extra keys ["modifiedCount","upsertedId","upsertedCount","matchedCount"] -updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint document on 4.2+ server updateMany: unexpected extra keys ["modifiedCount","upsertedId","upsertedCount","matchedCount"] updateMany-let.json SKIP updateMany with let option needs server >= 5.0 updateMany-let.json FAIL updateMany with let option unsupported (server-side error) updateMany: error message "update spec requires u" does not contain "'update.let' is an unknown field" updateMany-pipeline.json FAIL UpdateMany using pipelines MongoServerError: update spec requires u @@ -457,8 +434,6 @@ updateOne-dots_and_dollars.json SKIP Updating document to set top-level dotted k updateOne-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 updateOne-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 updateOne-errorResponse.json SKIP update operations support errorResponse assertions runner: failPoint -updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint string on 4.2+ server updateOne: unexpected extra keys ["modifiedCount","upsertedId","upsertedCount","matchedCount"] -updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint document on 4.2+ server updateOne: unexpected extra keys ["modifiedCount","upsertedId","upsertedCount","matchedCount"] updateOne-let.json SKIP UpdateOne with let option needs server >= 5.0 updateOne-let.json FAIL UpdateOne with let option unsupported (server-side error) updateOne: error message "update spec requires u" does not contain "'update.let' is an unknown field" updateOne-pipeline.json FAIL UpdateOne using pipelines MongoServerError: update spec requires u -- 2.39.5 From 6560aec9153aab30f4879fe29f1ca4715d8617d5 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:05:52 +0300 Subject: [PATCH 14/37] tests/spec: buffer command-monitoring events per client entity Plumbing only: a client entity that declares `observeEvents` now gets `monitorCommands` and a buffer, and nothing reads the buffer. That is the point of splitting it out -- the totals not moving *is* this commit's test. Command monitoring changes how the driver builds every command it sends, and if that alone shifted a result there would be no way to tell it apart from the assertions landing in the next commit. 193/99/195 before, 193/99/195 after, 175/175 files, 0 errored. The rules the buffer already enforces, so that the next commit is only about comparing: `ignoreCommandMonitoringEvents` by command name; sensitive commands dropped unless `observeSensitiveCommands` says otherwise, with `hello` and legacy hello inferred sensitive from the driver having redacted them to empty documents (unified-test-format.md:3070-3075). Neither fires on this corpus -- 136 client entities observe `commandStartedEvent`, 6 also `commandSucceededEvent`, and not one sets either field -- but a rule that only exists where it is exercised is a rule that will be missing when M7 brings auth. `cmap` and `sdam` observations are collected by nobody; a test that goes on to assert them is reported unsupported where it asserts, not where it declares. Two things about placement, both load-bearing. Listeners are attached after `connect()`, so a client's own handshake is not in its own buffer -- measured rather than assumed: with the buffers dumped, find.json's five cases show exactly `find`, `getMore`, `getMore` and nothing else. And they are disabled after the operations and before the outcome check (unified-test-format.md:3081), plus again unconditionally in the teardown `finally`, because the outcome check and the teardown both issue commands and a buffer still growing through them would make the assertion a function of the harness rather than of the engine. `MFDB_DUMP_EVENTS=1` prints each case's buffer. That is how the handshake question above was settled and how a failing event assertion will be triaged. --- tests/spec/run.js | 81 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/tests/spec/run.js b/tests/spec/run.js index 1d604de..e62988d 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -490,9 +490,62 @@ async function seedInitialData(initialData) { } } +// --------------------------------------------------------------------------- +// Command monitoring +// --------------------------------------------------------------------------- + +// The three event types a `client` entity can ask to observe that this runner +// can produce, mapped to the driver's own event names. `cmap` and `sdam` types +// are simply not collected; a test that goes on to *assert* them is reported +// unsupported at that point rather than here, so declaring an observation this +// runner ignores costs a case nothing. +const COMMAND_EVENTS = { + commandStartedEvent: 'commandStarted', + commandSucceededEvent: 'commandSucceeded', + commandFailedEvent: 'commandFailed', +}; + +// Sensitive commands, per the command-logging-and-monitoring spec's Security +// section. Events for these are dropped unless the entity sets +// `observeSensitiveCommands` (unified-test-format.md:3070-3075). None of them +// can be issued by this engine yet -- there is no auth and no user management +// before M7 -- so this is here to keep the rule where the rule belongs rather +// than to filter anything today. +const SENSITIVE_COMMANDS = new Set([ + 'authenticate', 'saslstart', 'saslcontinue', 'getnonce', 'createuser', + 'updateuser', 'copydbgetnonce', 'copydbsaslstart', 'copydb', +]); + +function isSensitive(ev) { + const name = String(ev.commandName || '').toLowerCase(); + if (SENSITIVE_COMMANDS.has(name)) return true; + // `hello` and legacy hello are sensitive only when they carry + // `speculativeAuthenticate`, which the driver does not report either way -- + // it redacts both the command and the reply to an empty document, and the + // spec says to infer sensitivity from exactly that. + if (name === 'hello' || name === 'ismaster') { + const body = ev.command || ev.reply; + return !!body && Object.keys(body).length === 0; + } + return false; +} + +// Listeners are disabled rather than removed, and disabled before the outcome +// check rather than after it (unified-test-format.md:3081): the teardown that +// follows a case issues commands of its own, and a buffer that kept growing +// through it would make the assertion a function of the harness. +function disableEvents(events) { + if (process.env.MFDB_DUMP_EVENTS) { + for (const [id, buf] of events) console.log(` events ${id}: ${buf.map((e) => `${e.kind}:${e.ev.commandName}`).join(', ')}`); + } + for (const buf of events.values()) buf.enabled = false; +} + // `clients` is supplied by the caller so that entities created before a // failure are still closed: returning them only on success is what leaked. -async function buildEntities(url, createEntities, clients) { +// `events` is supplied for the same reason -- a case that dies partway still +// has to be able to turn its listeners off. +async function buildEntities(url, createEntities, clients, events) { const map = {}; for (const spec of createEntities || []) { const [kind, def] = Object.entries(spec)[0]; @@ -506,10 +559,12 @@ async function buildEntities(url, createEntities, clients) { // buildEntities -- can create further clients *after* cleanup // has already run. That is what leaked, and what turned into // 190 phantom timeout FAILs. + const observed = (def.observeEvents || []).filter((e) => e in COMMAND_EVENTS); const c = new MongoClient(url, Object.assign({ serverSelectionTimeoutMS: 2000, connectTimeoutMS: 2000, timeoutMS: OP_TIMEOUT_MS, + monitorCommands: observed.length > 0, }, def.uriOptions || {})); // Registered before connect, so a client whose connect throws // or is abandoned is still closed by the caller. @@ -520,6 +575,23 @@ async function buildEntities(url, createEntities, clients) { await c.close().catch(() => { }); throw new Error('case abandoned'); } + // Subscribed after connect, so the handshake this client just + // performed is not in its own buffer. + if (observed.length) { + const ignore = new Set((def.ignoreCommandMonitoringEvents || []).map((s) => String(s).toLowerCase())); + const buf = []; + buf.enabled = true; + events.set(def.id, buf); + for (const name of observed) { + const kind = name.replace(/Event$/, ''); + c.on(COMMAND_EVENTS[name], (ev) => { + if (!buf.enabled) return; + if (ignore.has(String(ev.commandName).toLowerCase())) return; + if (!def.observeSensitiveCommands && isSensitive(ev)) return; + buf.push({ kind, ev }); + }); + } + } map[def.id] = c; break; } @@ -531,7 +603,7 @@ async function buildEntities(url, createEntities, clients) { default: throw new Unsupported(`entity type ${kind}`); } } - return { map, clients }; + return { map, clients, events }; } async function verifyOutcome(outcome, entities) { @@ -570,11 +642,13 @@ async function runFile(file, url, server) { // Owned out here, not by buildEntities, so a case that dies partway // still has every client it managed to open closed below. const clients = []; + const events = new Map(); try { await withTimeout(async () => { await seedInitialData(doc.initialData); - const entities = await buildEntities(url, doc.createEntities, clients); + const entities = await buildEntities(url, doc.createEntities, clients, events); for (const op of test.operations) await runOne(op, entities); + disableEvents(events); await verifyOutcome(test.outcome, entities); }, CASE_TIMEOUT_MS, `case exceeded ${CASE_TIMEOUT_MS} ms`); out.pass++; @@ -587,6 +661,7 @@ async function runFile(file, url, server) { // its continuation may still be running and about to open another // client, which buildEntities closes itself on seeing this. clients.abandoned = true; + disableEvents(events); for (const c of clients) await c.close().catch(() => { }); } } -- 2.39.5 From 97e3e3a556b91c58559f17dc353de648c4256df6 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:11:43 +0300 Subject: [PATCH 15/37] tests/spec: assert expectEvents The headline is not the delta, it is that `pass` changed meaning. 354 of the 487 cases declare `expectEvents` and until now the runner read none of them, so a case could send the wrong command entirely and still be counted a pass as long as the *result* came back right. The old column was an upper bound by construction. 193/99/195 becomes 159/133/195, and the two numbers are not comparable. Two rules decide how far the assertion reaches, both taken from the spec rather than from what would be convenient: - `command` and `reply` match as *root* documents (unified-test-format.md:1020-1022, :1037-1039). The driver hangs `lsid`, `$db` and `maxTimeMS` off nearly everything it sends; as nested documents essentially the whole corpus would fail on keys no expectation was ever written to mention, and the number would say nothing. - the event list is exact in number and order, not a prefix (unified-test-format.md:3088-3091). 23 cases expect an empty list and a prefix rule would pass every one of them without looking. The assertion runs after the operations, so a wrong result is still reported as a wrong result rather than being masked, and after the listeners are disabled, so the teardown's own commands cannot reach the buffer. `cmap` and `sdam` event types, `ignoreExtraEvents`, and any event field beyond `command`/`reply`/`commandName`/`databaseName` are reported unsupported at the point of assertion. None occurs in this corpus -- all 354 blocks are `eventType: command`, carrying 349 `commandStartedEvent` and 6 `commandSucceededEvent` -- so nothing is being quietly waived. All 34 newly-failing cases, triaged. Not one is a wrong answer from the engine; every one is a command the driver never sent: - 22x `command.writeConcern: missing` -- runner gap, and the sharpest thing this commit found. `buildEntities` drops `collectionOptions` on the floor, so `writeConcern: {w: 0}` never reached the driver and every "unacknowledged write" case in the corpus has been running an acknowledged write. They passed because the results of the two agree. This is precisely the class of error the instrument was built to find, and it was invisible to the result column. - 5x `command.sort.: missing` -- runner gap. The driver holds a sort as a JS `Map` (lib/sort.js), so `Object.keys` on it is empty and the matcher reports every expected key as absent. Measured, not guessed: EJSON prints a `Map` exactly like a document, which is why the dump looks correct. - 4x `command.bypassDocumentValidation: missing` -- unclassified. The option is absent from the wire for the `false` cases; the driver only forwards it when true on some paths (lib/operations/find_and_modify.js:19), and whether the runner also drops it has not been established. - 2x `command.comment: missing` on getMore -- server gap, most likely. The driver gates it on `maxWireVersion >= 9` (lib/operations/get_more.js:43) and this engine advertises 8 while reporting itself as 4.4.0, which is wire 9. The inconsistency is ours. - 1x `command.maxTimeMS: expected 6000, got 10000` -- the CSOT rewrite, dealt with in the next commit. Each of those gets its own commit, and none of them is fixed here: a check and the fix for what the check caught do not belong in one change. --- tests/spec/run.js | 54 ++++++++++++++++++++++++++++ tests/spec/scorecard.txt | 78 ++++++++++++++++++++++++++++------------ 2 files changed, 110 insertions(+), 22 deletions(-) diff --git a/tests/spec/run.js b/tests/spec/run.js index e62988d..b4f96f4 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -606,6 +606,59 @@ async function buildEntities(url, createEntities, clients, events) { return { map, clients, events }; } +// Two rules decide how much of the corpus this assertion can reach, and both +// come from the spec rather than from taste: +// +// - `command` and `reply` are matched as *root* documents +// (unified-test-format.md:1020-1022 and :1037-1039). The driver puts +// `lsid`, `$db` and `$readPreference` on nearly everything it sends; +// matched as nested documents, almost every case in the corpus would fail +// on keys its expectation was never written to mention. +// - the event list is exact in number and order, not a prefix +// (unified-test-format.md:3088-3091). 23 cases here expect an empty list, +// and a prefix rule would pass every one of them without looking. +function verifyEvents(expectEvents, entities) { + if (!expectEvents) return; + for (const spec of expectEvents) { + const type = spec.eventType || 'command'; + if (type !== 'command') throw new Unsupported(`${type} events`); + if (spec.ignoreExtraEvents) throw new Unsupported('ignoreExtraEvents'); + const buf = entities.events.get(spec.client); + // Not treated as an empty list: a client that never subscribed and a + // client that saw nothing are the same thing to a comparison and very + // different things to a runner, and the second one is a runner bug + // that would quietly pass the 23 empty-list cases. + if (!buf) fail(`events ${spec.client}`, 'the client entity is not observing command events'); + const expected = spec.events || []; + if (expected.length !== buf.length) { + fail(`events ${spec.client}`, `expected ${expected.length} events, observed ${buf.length}` + + ` [${buf.map((e) => `${e.kind}:${e.ev.commandName}`).join(', ')}]`); + } + expected.forEach((e, i) => matchEvent(e, buf[i], entities, `events ${spec.client}[${i}]`)); + } +} + +function matchEvent(expected, actual, entities, pathStr) { + const [name, body] = Object.entries(expected)[0]; + if (!(name in COMMAND_EVENTS)) throw new Unsupported(`event ${name}`); + const kind = name.replace(/Event$/, ''); + if (actual.kind !== kind) fail(pathStr, `expected ${kind}, got ${actual.kind} of ${actual.ev.commandName}`); + for (const [k, v] of Object.entries(body || {})) { + switch (k) { + case 'command': + case 'reply': + match(v, actual.ev[k], entities, `${pathStr}.${k}`, true); + break; + case 'commandName': + case 'databaseName': + match(v, actual.ev[k], entities, `${pathStr}.${k}`, false); + break; + default: + throw new Unsupported(`event assertion ${name}.${k}`); + } + } +} + async function verifyOutcome(outcome, entities) { if (!outcome) return; for (const spec of outcome) { @@ -649,6 +702,7 @@ async function runFile(file, url, server) { const entities = await buildEntities(url, doc.createEntities, clients, events); for (const op of test.operations) await runOne(op, entities); disableEvents(events); + verifyEvents(test.expectEvents, entities); await verifyOutcome(test.outcome, entities); }, CASE_TIMEOUT_MS, `case exceeded ${CASE_TIMEOUT_MS} ms`); out.pass++; diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index 1973557..4ef782b 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -13,7 +13,7 @@ # semantics; ignoring them makes some cases pass that a full runner would # fail, so treat `pass` as an upper bound until M1 wires events up. -total 193 pass 99 fail 195 skip 175 files 0 errored +total 159 pass 133 fail 195 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -25,40 +25,40 @@ aggregate-out-readConcern.json 0 0 4 aggregate-out.json 0 2 0 aggregate-rawdata.json 1 0 1 aggregate-write-readPreference.json 0 0 4 -aggregate.json 5 0 2 +aggregate.json 4 1 2 bulkWrite-arrayFilters.json 0 3 0 bulkWrite-collation.json 0 2 0 bulkWrite-comment.json 2 0 1 bulkWrite-delete-hint-serverError.json 0 0 2 bulkWrite-delete-hint.json 2 0 0 -bulkWrite-deleteMany-hint-unacknowledged.json 2 0 2 +bulkWrite-deleteMany-hint-unacknowledged.json 0 2 2 bulkWrite-deleteMany-let.json 0 1 1 bulkWrite-deleteMany-rawdata.json 1 0 1 -bulkWrite-deleteOne-hint-unacknowledged.json 2 0 2 +bulkWrite-deleteOne-hint-unacknowledged.json 0 2 2 bulkWrite-deleteOne-let.json 0 1 1 bulkWrite-deleteOne-rawdata.json 1 0 1 bulkWrite-errorResponse.json 0 0 1 bulkWrite-insertOne-dots_and_dollars.json 3 1 1 bulkWrite-replaceOne-dots_and_dollars.json 2 1 1 -bulkWrite-replaceOne-hint-unacknowledged.json 2 0 0 +bulkWrite-replaceOne-hint-unacknowledged.json 0 2 0 bulkWrite-replaceOne-let.json 0 1 1 bulkWrite-replaceOne-rawdata.json 1 0 1 -bulkWrite-replaceOne-sort.json 1 0 1 +bulkWrite-replaceOne-sort.json 0 1 1 bulkWrite-update-hint.json 3 0 0 bulkWrite-update-validation.json 3 0 0 bulkWrite-updateMany-dots_and_dollars.json 0 0 4 -bulkWrite-updateMany-hint-unacknowledged.json 2 0 0 +bulkWrite-updateMany-hint-unacknowledged.json 0 2 0 bulkWrite-updateMany-let.json 0 1 1 bulkWrite-updateMany-pipeline.json 0 1 0 bulkWrite-updateMany-rawdata.json 0 1 1 bulkWrite-updateOne-dots_and_dollars.json 0 0 4 -bulkWrite-updateOne-hint-unacknowledged.json 2 0 0 +bulkWrite-updateOne-hint-unacknowledged.json 0 2 0 bulkWrite-updateOne-let.json 0 1 1 bulkWrite-updateOne-pipeline.json 0 1 0 bulkWrite-updateOne-rawdata.json 0 1 1 -bulkWrite-updateOne-sort.json 1 0 1 +bulkWrite-updateOne-sort.json 0 1 1 bulkWrite.json 10 0 0 -bypassDocumentValidation.json 8 1 0 +bypassDocumentValidation.json 4 5 0 client-bulkWrite-delete-options.json 0 0 2 client-bulkWrite-delete-rawdata.json 0 0 2 client-bulkWrite-errorResponse.json 0 0 1 @@ -88,7 +88,7 @@ db-aggregate.json 0 2 0 deleteMany-collation.json 0 1 0 deleteMany-comment.json 2 0 1 deleteMany-hint-serverError.json 0 0 2 -deleteMany-hint-unacknowledged.json 2 0 2 +deleteMany-hint-unacknowledged.json 0 2 2 deleteMany-hint.json 2 0 0 deleteMany-let.json 0 1 1 deleteMany-rawdata.json 1 0 1 @@ -97,7 +97,7 @@ deleteOne-collation.json 0 1 0 deleteOne-comment.json 2 0 1 deleteOne-errorResponse.json 0 0 1 deleteOne-hint-serverError.json 0 0 2 -deleteOne-hint-unacknowledged.json 2 0 2 +deleteOne-hint-unacknowledged.json 0 2 2 deleteOne-hint.json 2 0 0 deleteOne-let.json 0 1 1 deleteOne-rawdata.json 1 0 1 @@ -109,15 +109,15 @@ distinct-rawdata.json 0 1 1 distinct.json 0 2 0 estimatedDocumentCount-comment.json 1 1 1 estimatedDocumentCount-rawdata.json 1 0 1 -estimatedDocumentCount.json 3 1 2 +estimatedDocumentCount.json 2 2 2 find-allowdiskuse-serverError.json 0 0 2 find-allowdiskuse.json 3 0 0 find-collation.json 0 1 0 -find-comment.json 1 2 2 +find-comment.json 0 3 2 find-let.json 0 1 1 find-rawdata.json 1 0 1 find.json 5 0 0 -findOne.json 2 0 0 +findOne.json 1 1 0 findOneAndDelete-collation.json 0 1 0 findOneAndDelete-comment.json 2 0 1 findOneAndDelete-hint-serverError.json 0 0 2 @@ -153,25 +153,25 @@ insertMany-dots_and_dollars.json 3 1 1 insertMany-rawdata.json 1 0 1 insertMany.json 3 0 0 insertOne-comment.json 2 0 1 -insertOne-dots_and_dollars.json 6 2 1 +insertOne-dots_and_dollars.json 5 3 1 insertOne-errorResponse.json 0 0 1 insertOne-rawdata.json 1 0 1 insertOne.json 1 0 0 replaceOne-collation.json 0 1 0 replaceOne-comment.json 2 0 1 -replaceOne-dots_and_dollars.json 3 1 1 -replaceOne-hint-unacknowledged.json 2 0 0 +replaceOne-dots_and_dollars.json 2 2 1 +replaceOne-hint-unacknowledged.json 0 2 0 replaceOne-hint.json 2 0 0 replaceOne-let.json 0 1 1 replaceOne-rawdata.json 1 0 1 -replaceOne-sort.json 1 0 1 +replaceOne-sort.json 0 1 1 replaceOne-validation.json 1 0 0 replaceOne.json 5 0 0 updateMany-arrayFilters.json 0 3 0 updateMany-collation.json 0 1 0 updateMany-comment.json 2 0 1 updateMany-dots_and_dollars.json 0 0 4 -updateMany-hint-unacknowledged.json 2 0 0 +updateMany-hint-unacknowledged.json 0 2 0 updateMany-hint.json 2 0 0 updateMany-let.json 0 1 1 updateMany-pipeline.json 0 1 0 @@ -183,12 +183,12 @@ updateOne-collation.json 0 1 0 updateOne-comment.json 2 0 1 updateOne-dots_and_dollars.json 0 0 4 updateOne-errorResponse.json 0 0 1 -updateOne-hint-unacknowledged.json 2 0 0 +updateOne-hint-unacknowledged.json 0 2 0 updateOne-hint.json 2 0 0 updateOne-let.json 0 1 1 updateOne-pipeline.json 0 1 0 updateOne-rawdata.json 1 0 1 -updateOne-sort.json 1 0 1 +updateOne-sort.json 0 1 1 updateOne-validation.json 1 0 0 updateOne.json 4 0 0 @@ -210,6 +210,7 @@ aggregate-out.json FAIL Aggregate with $out and batch size of 0 MongoServerError aggregate-rawdata.json SKIP Aggregate with rawData option needs server >= 8.2.0 aggregate-write-readPreference.json SKIP * needs topology replicaset/sharded/load-balanced aggregate.json SKIP aggregate with a document comment - pre 4.4 needs server <= 4.2.99 +aggregate.json FAIL aggregate with comment sets comment on getMore events client0[1].command.comment: missing from actual aggregate.json SKIP aggregate with comment does not set comment on getMore - pre 4.4 needs server <= 4.3.99 bulkWrite-arrayFilters.json FAIL BulkWrite updateOne with arrayFilters outcome crud-tests.test[0].y: expected an array, got {"$[i]":{"b":2}} bulkWrite-arrayFilters.json FAIL BulkWrite updateMany with arrayFilters outcome crud-tests.test[0].y: expected an array, got {"$[i]":{"b":2}} @@ -220,11 +221,15 @@ bulkWrite-comment.json SKIP BulkWrite with comment - pre 4.4 needs server <= 4.2 bulkWrite-delete-hint-serverError.json SKIP * needs server <= 4.3.3 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 +bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual bulkWrite-deleteMany-let.json SKIP BulkWrite deleteMany with let option needs server >= 5.0 bulkWrite-deleteMany-let.json FAIL BulkWrite deleteMany with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteMany-rawdata.json SKIP BulkWrite deleteMany with rawData option needs server >= 8.2.0 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 +bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual bulkWrite-deleteOne-let.json SKIP BulkWrite deleteOne with let option needs server >= 5.0 bulkWrite-deleteOne-let.json FAIL BulkWrite deleteOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteOne-rawdata.json SKIP BulkWrite deleteOne with rawData option needs server >= 8.2.0 @@ -233,14 +238,19 @@ bulkWrite-insertOne-dots_and_dollars.json SKIP Inserting document with top-level bulkWrite-insertOne-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-dots_and_dollars.json SKIP Replacing document with dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-replaceOne-dots_and_dollars.json FAIL Replacing document with dollar-prefixed key in embedded doc on pre-5.0 server yields server-side error bulkWrite: expected an error, the operation succeeded +bulkWrite-replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual bulkWrite-replaceOne-let.json SKIP BulkWrite replaceOne with let option needs server >= 5.0 bulkWrite-replaceOne-let.json FAIL BulkWrite replaceOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-rawdata.json SKIP BulkWrite replaceOne with rawData option needs server >= 8.2.0 bulkWrite-replaceOne-sort.json SKIP BulkWrite replaceOne with sort option needs server >= 8.0 +bulkWrite-replaceOne-sort.json FAIL BulkWrite replaceOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-level dollar-prefixed key on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 +bulkWrite-updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual bulkWrite-updateMany-let.json SKIP BulkWrite updateMany with let option needs server >= 5.0 bulkWrite-updateMany-let.json FAIL BulkWrite updateMany with let option unsupported (server-side error) bulkWrite: error message "update spec requires u" does not contain "'update.let' is an unknown field" bulkWrite-updateMany-pipeline.json FAIL UpdateMany in bulk write using pipelines MongoBulkWriteError: update spec requires u @@ -250,13 +260,20 @@ bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set top-leve bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 +bulkWrite-updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual bulkWrite-updateOne-let.json SKIP BulkWrite updateOne with let option needs server >= 5.0 bulkWrite-updateOne-let.json FAIL BulkWrite updateOne with let option unsupported (server-side error) bulkWrite: error message "update spec requires u" does not contain "'update.let' is an unknown field" bulkWrite-updateOne-pipeline.json FAIL UpdateOne in bulk write using pipelines MongoBulkWriteError: update spec requires u bulkWrite-updateOne-rawdata.json SKIP BulkWrite updateOne with rawData option needs server >= 8.2.0 bulkWrite-updateOne-rawdata.json FAIL BulkWrite updateOne with rawData option on less than 8.2.0 - ignore argument MongoBulkWriteError: update spec requires u bulkWrite-updateOne-sort.json SKIP BulkWrite updateOne with sort option needs server >= 8.0 +bulkWrite-updateOne-sort.json FAIL BulkWrite updateOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual bypassDocumentValidation.json FAIL Aggregate with $out passes bypassDocumentValidation: false MongoServerError: Unrecognized pipeline stage name: '$out' +bypassDocumentValidation.json FAIL BulkWrite passes bypassDocumentValidation: false events client0[0].command.bypassDocumentValidation: missing from actual +bypassDocumentValidation.json FAIL FindOneAndReplace passes bypassDocumentValidation: false events client0[0].command.bypassDocumentValidation: missing from actual +bypassDocumentValidation.json FAIL FindOneAndUpdate passes bypassDocumentValidation: false events client0[0].command.bypassDocumentValidation: missing from actual +bypassDocumentValidation.json FAIL InsertMany passes bypassDocumentValidation: false events client0[0].command.bypassDocumentValidation: missing from actual client-bulkWrite-delete-options.json SKIP * needs server >= 8.0 client-bulkWrite-delete-rawdata.json SKIP client bulk write delete with rawData option needs server >= 8.2.0 client-bulkWrite-delete-rawdata.json SKIP client bulk write delete with rawData option on less than 8.2.0 - ignore argument needs server >= 8.0 @@ -304,6 +321,8 @@ deleteMany-comment.json SKIP deleteMany with comment - pre 4.4 needs server <= 4 deleteMany-hint-serverError.json SKIP * needs server <= 4.3.3 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 +deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual +deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual deleteMany-let.json SKIP deleteMany with let option needs server >= 5.0 deleteMany-let.json FAIL deleteMany with let option unsupported (server-side error) deleteMany: expected an error, the operation succeeded deleteMany-rawdata.json SKIP deleteMany with rawData option needs server >= 8.2.0 @@ -313,6 +332,8 @@ deleteOne-errorResponse.json SKIP delete operations support errorResponse assert deleteOne-hint-serverError.json SKIP * needs server <= 4.3.3 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 +deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual +deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual deleteOne-let.json SKIP deleteOne with let option needs server >= 5.0 deleteOne-let.json FAIL deleteOne with let option unsupported (server-side error) deleteOne: expected an error, the operation succeeded deleteOne-rawdata.json SKIP deleteOne with rawData option needs server >= 8.2.0 @@ -327,6 +348,7 @@ distinct.json FAIL Distinct with a filter MongoServerError: no such command: 'di estimatedDocumentCount-comment.json SKIP estimatedDocumentCount with document comment needs server >= 4.4.14 estimatedDocumentCount-comment.json FAIL estimatedDocumentCount with document comment - pre 4.4.14, server error estimatedDocumentCount: expected an error, the operation succeeded estimatedDocumentCount-rawdata.json SKIP Estimated document count with rawData option needs server >= 8.2.0 +estimatedDocumentCount.json FAIL estimatedDocumentCount with maxTimeMS events client0[0].command.maxTimeMS: expected 6000, got 10000 estimatedDocumentCount.json SKIP estimatedDocumentCount errors correctly--command error runner: failPoint estimatedDocumentCount.json SKIP estimatedDocumentCount errors correctly--socket error runner: failPoint estimatedDocumentCount.json FAIL estimatedDocumentCount works correctly on views estimatedDocumentCount: expected 2, got 0 @@ -335,10 +357,12 @@ find-collation.json FAIL Find with a collation find: expected 1 elements, got 0 find-comment.json FAIL find with string comment find[0]: unexpected extra keys ["x"] find-comment.json FAIL find with document comment find[0]: unexpected extra keys ["x"] find-comment.json SKIP find with document comment - pre 4.4 needs server <= 4.2.99 +find-comment.json FAIL find with comment sets comment on getMore events client0[1].command.comment: missing from actual find-comment.json SKIP find with comment does not set comment on getMore - pre 4.4 needs server <= 4.3.99 find-let.json SKIP Find with let option needs server >= 5.0 find-let.json FAIL Find with let option unsupported (server-side error) find: expected an error, the operation succeeded find-rawdata.json SKIP Find with rawData option needs server >= 8.2.0 +findOne.json FAIL FindOne with filter, sort, and skip events client0[0].command.sort._id: missing from actual findOneAndDelete-collation.json FAIL FindOneAndDelete when one document matches with collation findOneAndDelete: expected a document, got null findOneAndDelete-comment.json SKIP findOneAndDelete with comment - pre 4.4 needs server <= 4.2.99 findOneAndDelete-hint-serverError.json SKIP * needs server <= 4.3.3 @@ -399,16 +423,21 @@ insertOne-comment.json SKIP insertOne with comment - pre 4.4 needs server <= 4.2 insertOne-dots_and_dollars.json SKIP Inserting document with top-level dollar-prefixed key on 5.0+ server needs server >= 5.0 insertOne-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error insertOne: expected an error, the operation succeeded insertOne-dots_and_dollars.json FAIL Inserting document with dollar-prefixed key in _id yields server-side error insertOne: expected an error, the operation succeeded +insertOne-dots_and_dollars.json FAIL Unacknowledged write using dollar-prefixed or dotted keys may be silently rejected on pre-5.0 server events client0[0].command.writeConcern: missing from actual insertOne-errorResponse.json SKIP insert operations support errorResponse assertions runner: failPoint insertOne-rawdata.json SKIP insertOne with rawData option needs server >= 8.2.0 replaceOne-collation.json FAIL ReplaceOne when one document matches with collation replaceOne.matchedCount: expected 1, got 0 replaceOne-comment.json SKIP ReplaceOne with comment - pre 4.4 needs server <= 4.2.99 replaceOne-dots_and_dollars.json SKIP Replacing document with dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 replaceOne-dots_and_dollars.json FAIL Replacing document with dollar-prefixed key in embedded doc on pre-5.0 server yields server-side error replaceOne: expected an error, the operation succeeded +replaceOne-dots_and_dollars.json FAIL Unacknowledged write using dollar-prefixed or dotted keys may be silently rejected on pre-5.0 server events client0[0].command.writeConcern: missing from actual +replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual +replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual replaceOne-let.json SKIP ReplaceOne with let option needs server >= 5.0 replaceOne-let.json FAIL ReplaceOne with let option unsupported (server-side error) replaceOne: expected an error, the operation succeeded replaceOne-rawdata.json SKIP ReplaceOne with rawData option needs server >= 8.2.0 replaceOne-sort.json SKIP ReplaceOne with sort option needs server >= 8.0 +replaceOne-sort.json FAIL replaceOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual updateMany-arrayFilters.json FAIL UpdateMany when no documents match arrayFilters updateMany.modifiedCount: expected 0, got 2 updateMany-arrayFilters.json FAIL UpdateMany when one document matches arrayFilters updateMany.modifiedCount: expected 1, got 2 updateMany-arrayFilters.json FAIL UpdateMany when multiple documents match arrayFilters outcome crud-v1.coll[0].y: expected an array, got {"$[i]":{"b":2}} @@ -418,6 +447,8 @@ updateMany-dots_and_dollars.json SKIP Updating document to set top-level dollar- updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 updateMany-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 +updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual +updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual updateMany-let.json SKIP updateMany with let option needs server >= 5.0 updateMany-let.json FAIL updateMany with let option unsupported (server-side error) updateMany: error message "update spec requires u" does not contain "'update.let' is an unknown field" updateMany-pipeline.json FAIL UpdateMany using pipelines MongoServerError: update spec requires u @@ -434,8 +465,11 @@ updateOne-dots_and_dollars.json SKIP Updating document to set top-level dotted k updateOne-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 updateOne-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 updateOne-errorResponse.json SKIP update operations support errorResponse assertions runner: failPoint +updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual +updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual updateOne-let.json SKIP UpdateOne with let option needs server >= 5.0 updateOne-let.json FAIL UpdateOne with let option unsupported (server-side error) updateOne: error message "update spec requires u" does not contain "'update.let' is an unknown field" updateOne-pipeline.json FAIL UpdateOne using pipelines MongoServerError: update spec requires u updateOne-rawdata.json SKIP UpdateOne with rawData option needs server >= 8.2.0 updateOne-sort.json SKIP UpdateOne with sort option needs server >= 8.0 +updateOne-sort.json FAIL updateOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual -- 2.39.5 From 54ad124c1885fce8fb54e13e2ff9097508ab83fb Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:13:38 +0300 Subject: [PATCH 16/37] tests/spec: a CSOT-rewritten maxTimeMS cannot be asserted Every client entity is built with CSOT `timeoutMS` (OP_TIMEOUT_MS, 10 s), and CSOT overwrites `maxTimeMS` on each command with what is left of that budget. An expectation of `maxTimeMS: 6000` therefore meets the harness's 10000, and no amount of engine correctness would change it. Reported unsupported rather than failed: a FAIL is a claim about the engine, and this is a claim about the runner. Refused unconditionally when an expected command mentions `maxTimeMS`, not only when the two values differ, so it can never become a pass by coincidence. Exactly one case in the corpus asserts it -- estimatedDocumentCount.json, "estimatedDocumentCount with maxTimeMS" -- so the whole cost of the hatch is one case, which is why it is worth taking instead of dropping `timeoutMS`. That option is not open anyway: `timeoutMS` is what replaced the outer race that once turned ~190 good cases into phantom timeout FAILs. This is the only escape hatch in the runner. Everything else is either an honest FAIL or an enumerated unsupported feature. 159/133/195 becomes 159/132/196: one case, fail to skip, and nothing else moves. --- tests/spec/run.js | 15 +++++++++++++++ tests/spec/scorecard.txt | 6 +++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/spec/run.js b/tests/spec/run.js index b4f96f4..021b193 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -646,6 +646,21 @@ function matchEvent(expected, actual, entities, pathStr) { for (const [k, v] of Object.entries(body || {})) { switch (k) { case 'command': + // The one assertion this runner cannot make, and the only + // escape hatch in it. Every client entity carries CSOT + // `timeoutMS` (see OP_TIMEOUT_MS), and CSOT overwrites + // `maxTimeMS` on every command with what is left of that + // budget -- so the value on the wire is the harness's, not the + // test's. Dropping `timeoutMS` is not the alternative: it is + // what replaced the outer race that produced ~190 phantom + // timeout FAILs, and it would cost far more than one case. + // Refused unconditionally rather than only when the values + // differ, so this can never turn into a pass by coincidence. + if (isPlainDoc(v) && Object.prototype.hasOwnProperty.call(v, 'maxTimeMS')) { + throw new Unsupported('maxTimeMS in an expected command (CSOT rewrites it)'); + } + match(v, actual.ev[k], entities, `${pathStr}.${k}`, true); + break; case 'reply': match(v, actual.ev[k], entities, `${pathStr}.${k}`, true); break; diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index 4ef782b..28b048a 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -13,7 +13,7 @@ # semantics; ignoring them makes some cases pass that a full runner would # fail, so treat `pass` as an upper bound until M1 wires events up. -total 159 pass 133 fail 195 skip 175 files 0 errored +total 159 pass 132 fail 196 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -109,7 +109,7 @@ distinct-rawdata.json 0 1 1 distinct.json 0 2 0 estimatedDocumentCount-comment.json 1 1 1 estimatedDocumentCount-rawdata.json 1 0 1 -estimatedDocumentCount.json 2 2 2 +estimatedDocumentCount.json 2 1 3 find-allowdiskuse-serverError.json 0 0 2 find-allowdiskuse.json 3 0 0 find-collation.json 0 1 0 @@ -348,7 +348,7 @@ distinct.json FAIL Distinct with a filter MongoServerError: no such command: 'di estimatedDocumentCount-comment.json SKIP estimatedDocumentCount with document comment needs server >= 4.4.14 estimatedDocumentCount-comment.json FAIL estimatedDocumentCount with document comment - pre 4.4.14, server error estimatedDocumentCount: expected an error, the operation succeeded estimatedDocumentCount-rawdata.json SKIP Estimated document count with rawData option needs server >= 8.2.0 -estimatedDocumentCount.json FAIL estimatedDocumentCount with maxTimeMS events client0[0].command.maxTimeMS: expected 6000, got 10000 +estimatedDocumentCount.json SKIP estimatedDocumentCount with maxTimeMS runner: maxTimeMS in an expected command (CSOT rewrites it) estimatedDocumentCount.json SKIP estimatedDocumentCount errors correctly--command error runner: failPoint estimatedDocumentCount.json SKIP estimatedDocumentCount errors correctly--socket error runner: failPoint estimatedDocumentCount.json FAIL estimatedDocumentCount works correctly on views estimatedDocumentCount: expected 2, got 0 -- 2.39.5 From bb8cdd964b16d5659d28b35e0c6bb22a55c1383d Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:14:58 +0300 Subject: [PATCH 17/37] tests/spec: the scorecard no longer disclaims expectEvents The disclaimer was accurate for as long as it stood -- events were not read, so `pass` was an upper bound and saying otherwise would have been a lie about the number. It is now a lie in the other direction, so it goes, replaced by what is actually true: events are compared exactly, in number and in order, which is what makes a pass mean the engine answered correctly *and* was asked the right question. The header says plainly that scorecards recorded before this are not comparable, and enumerates what is still skipped inside events rather than leaving "asserted" to be read as "asserted completely". Two facts in the docs had gone stale and are corrected here because this is the commit that rereads them: - README said `--op-timeout-ms` defaults to 3 s. It has been 10 s since the commit that explains, at length and directly above the constant, why 3 s was wrong. A stale number in exactly the place that warns against tightening it is worse than no number. - `MAX_SCHEMA` is [1, 24]; the comment above it still claimed 1.0-1.9. Totals unchanged at 159/132/196 -- this commit only rewrites prose, and the scorecard is re-recorded so its header matches the runner that produced it. --- tests/spec/README.md | 36 +++++++++++++++++++++++++----------- tests/spec/run.js | 18 ++++++++++++------ tests/spec/scorecard.txt | 13 +++++++++---- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/tests/spec/README.md b/tests/spec/README.md index f2905bf..3f4cc85 100644 --- a/tests/spec/README.md +++ b/tests/spec/README.md @@ -41,29 +41,43 @@ result is a real pass: Supported: `client`/`database`/`collection` entities, `initialData`, `outcome`, `expectError` (code, codeName, contains, labels, errorResponse), -`saveResultAsEntity`, `runOnRequirements` gating, and the `$$type`, -`$$exists`, `$$unsetOrMatches`, `$$matchesEntity`, `$$matchesHexBytes` -operators. +`expectEvents`, `saveResultAsEntity`, `runOnRequirements` gating, and the +`$$type`, `$$exists`, `$$unsetOrMatches`, `$$matchesEntity`, +`$$matchesHexBytes` operators. -**Not asserted yet: `expectEvents`** (command monitoring). Those assertions are -about the command shape the driver emits rather than result semantics. Ignoring -them lets some cases pass that a complete runner would fail, so **treat `pass` -as an upper bound** until M1 wires events up. This is stated again at the top of -`scorecard.txt` so the number is never read out of context. +`expectEvents` compares the commands the driver actually sent against the +expectation, **exact in number and in order**, with `command` and `reply` +matched as root documents so the driver's own additions (`lsid`, `$db`) are +allowed. It is what makes a pass mean the engine answered correctly *and* was +asked the right question — 354 of the 487 cases declare events, and before this +was asserted a case could send the wrong command and still be counted a pass. +**Scorecards recorded before it landed are not comparable**; there, `pass` was +an upper bound by construction. + +Still unasserted within events, each reported as SKIP at the point of +assertion: `cmap` and `sdam` event types, `ignoreExtraEvents`, `hasServiceId`, +`hasServerConnectionId`, and `maxTimeMS` in an expected command — the runner +puts CSOT `timeoutMS` on every client, and CSOT overwrites `maxTimeMS` with +what is left of that budget, so the value on the wire is the harness's. That is +the only assertion this runner declines to make; one case in the corpus is +affected. Not supported, each reported as SKIP with a reason and never as PASS: session and bucket entities (M4 / GridFS), `failPoint`, client-side encryption, `testRunner` operations, and any operation or matcher the runner does not know. +`MFDB_DUMP_EVENTS=1` prints each case's observed command stream, which is the +fastest way to tell a wrong answer from a command the driver never sent. + ## Reading the scorecard `scorecard.txt` records the totals, a per-file breakdown, and every non-passing case with its reason. The distinction that matters: - **FAIL** — the engine answered, and answered differently from the spec. Real - work. An operation that never answered inside `--op-timeout-ms` (default 3 s, - enforced by the driver itself via CSOT `timeoutMS`) is also a FAIL, because - "no answer" is a result. There is a second, much longer `--case-timeout-ms` + work. An operation that never answered inside `--op-timeout-ms` (default + 10 s, enforced by the driver itself via CSOT `timeoutMS`) is also a FAIL, + because "no answer" is a result. There is a second, much longer `--case-timeout-ms` backstop for a hang the driver cannot see; if it ever fires, treat the run with suspicion — see the trap below. - **SKIP** — nobody claims anything. Either the suite needs a feature whose diff --git a/tests/spec/run.js b/tests/spec/run.js index 021b193..a630bd9 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -31,8 +31,9 @@ const SUITE_DIR = path.join(__dirname, 'specifications', 'source', 'crud', 'test const SCORECARD = path.join(__dirname, 'scorecard.txt'); const REPO = path.join(__dirname, '..', '..'); -// The runner implements schema 1.0-1.9 features that CRUD tests actually use. -// A file declaring more than this is skipped whole rather than half-run. +// The runner implements the schema features that CRUD tests actually use, up +// to this version. A file declaring more than this is skipped whole rather +// than half-run. const MAX_SCHEMA = [1, 24]; const argv = process.argv.slice(2); @@ -897,10 +898,15 @@ function scorecardText(results, tot, errored, server, nfiles) { L.push('# - the suite needs a feature whose milestone has not landed (sessions M4,'); L.push('# auth M7, failPoints, gridfs) -- counted as skip, never as pass;'); L.push('# - or the runner itself does not implement the operation/matcher yet.'); - L.push('# Deliberately NOT asserted yet: expectEvents (command monitoring). Those'); - L.push('# assertions are about driver-visible command shape rather than result'); - L.push('# semantics; ignoring them makes some cases pass that a full runner would'); - L.push('# fail, so treat `pass` as an upper bound until M1 wires events up.'); + L.push('# expectEvents IS asserted: the commands the driver sent are compared to the'); + L.push('# expectation exactly, in number and in order, with `command` and `reply`'); + L.push('# matched as root documents. A pass therefore means the engine answered'); + L.push('# correctly *and* was asked the right question. Not comparable to any'); + L.push('# scorecard recorded before that landed, where `pass` was an upper bound.'); + L.push('# Still unasserted, each reported as skip where it is asserted, never as'); + L.push('# pass: cmap and sdam events, ignoreExtraEvents, hasServiceId,'); + L.push('# hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites'); + L.push('# it -- the only assertion this runner declines to make).'); L.push(''); L.push(`total\t${tot.pass} pass\t${tot.fail} fail\t${tot.skip} skip\t${nfiles} files\t${errored} errored`); L.push(''); diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index 28b048a..5556ea8 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -8,10 +8,15 @@ # - the suite needs a feature whose milestone has not landed (sessions M4, # auth M7, failPoints, gridfs) -- counted as skip, never as pass; # - or the runner itself does not implement the operation/matcher yet. -# Deliberately NOT asserted yet: expectEvents (command monitoring). Those -# assertions are about driver-visible command shape rather than result -# semantics; ignoring them makes some cases pass that a full runner would -# fail, so treat `pass` as an upper bound until M1 wires events up. +# expectEvents IS asserted: the commands the driver sent are compared to the +# expectation exactly, in number and in order, with `command` and `reply` +# matched as root documents. A pass therefore means the engine answered +# correctly *and* was asked the right question. Not comparable to any +# scorecard recorded before that landed, where `pass` was an upper bound. +# Still unasserted, each reported as skip where it is asserted, never as +# pass: cmap and sdam events, ignoreExtraEvents, hasServiceId, +# hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites +# it -- the only assertion this runner declines to make). total 159 pass 132 fail 196 skip 175 files 0 errored -- 2.39.5 From 7f426cdd333a29c7f268fa33c91c9d0613f4cbc5 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:17:03 +0300 Subject: [PATCH 18/37] tests/spec: a collection entity gets the options it was declared with `buildEntities` built every collection as `db.collection(name)` and every database as `client.db(name)`, dropping `collectionOptions` and `databaseOptions` on the floor. 15 collection entities declare a `writeConcern`, 7 a `readConcern`, one a `readPreference` -- and the 15 are all `{w: 0}`, so every "unacknowledged write" case in this corpus has been running an acknowledged write against a driver that was never told otherwise. They passed anyway, because an acknowledged and an unacknowledged write of the same document produce results a `$$unsetOrMatches` expectation accepts either way. Only the command on the wire distinguished them, and nothing was reading the command until the previous commits. This is the first thing the event assertions found, and it is a fair answer to what they cost. Option documents are unwrapped from their BSON types on the way to the driver. The suites are parsed with `relaxed: false`, so `{w: 0}` arrives as an Int32 and the driver gates `writeConcern.w` on `typeof w === 'number'` -- the same trap NUMERIC_OPTIONS already documents for operation options, and a silent one: the option would simply not apply. Wholesale unwrapping is safe here in a way it is not there, since these are settings the driver consumes rather than values an assertion compares. An option key outside the spec's `collectionOrDatabaseOptions` set is reported unsupported rather than ignored, which is the lesson of the bug itself. 159/132/196 becomes 173/118/196. 14 cases fixed, none broken. The other 10 unacknowledged cases now fail differently, and that is progress of a sort: with `w: 0` actually applied, the driver refuses client-side to send `hint` on a delete or findAndModify to a server older than 4.4. This engine reports itself as 4.4.0 with maxWireVersion 8, and 4.4 is wire 9. That inconsistency is ours, it is the same one behind the `comment`-on-getMore failures, and it gets the next commit. --- tests/spec/run.js | 34 +++++++++++++++++++++-- tests/spec/scorecard.txt | 60 +++++++++++++++------------------------- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/tests/spec/run.js b/tests/spec/run.js index a630bd9..b54fa26 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -542,6 +542,32 @@ function disableEvents(events) { for (const buf of events.values()) buf.enabled = false; } +// `collectionOrDatabaseOptions` (unified-test-format.md, entity definitions). +// Anything outside this set is refused rather than silently ignored -- that is +// the whole lesson of the bug this function exists to fix. +const ENTITY_OPTIONS = new Set(['readConcern', 'readPreference', 'writeConcern']); + +// These documents are parsed with `relaxed: false` like everything else in the +// file, so a plain JSON `0` arrives as a BSON Int32 -- and the driver gates +// `writeConcern.w` on `typeof w === 'number'`, which is exactly the trap +// NUMERIC_OPTIONS documents for operation options. Unwrapping wholesale is safe +// here in a way it is not there: these are settings the driver consumes, not +// values any assertion ever compares. +function plainOptions(spec, what) { + for (const k of Object.keys(spec || {})) { + if (!ENTITY_OPTIONS.has(k)) throw new Unsupported(`${what} ${k}`); + } + const walk = (v) => { + if (Array.isArray(v)) return v.map(walk); + if (typeof v === 'object' && v !== null && numeric(v) !== null) return numeric(v); + if (!isPlainDoc(v)) return v; + const out = {}; + for (const [k, x] of Object.entries(v)) out[k] = walk(x); + return out; + }; + return walk(spec || {}); +} + // `clients` is supplied by the caller so that entities created before a // failure are still closed: returning them only on success is what leaked. // `events` is supplied for the same reason -- a case that dies partway still @@ -596,8 +622,12 @@ async function buildEntities(url, createEntities, clients, events) { map[def.id] = c; break; } - case 'database': map[def.id] = map[def.client].db(def.databaseName); break; - case 'collection': map[def.id] = map[def.database].collection(def.collectionName); break; + case 'database': + map[def.id] = map[def.client].db(def.databaseName, plainOptions(def.databaseOptions, 'databaseOptions')); + break; + case 'collection': + map[def.id] = map[def.database].collection(def.collectionName, plainOptions(def.collectionOptions, 'collectionOptions')); + break; case 'session': throw new Unsupported('session entities (M4)'); case 'bucket': throw new Unsupported('gridfs bucket entities'); case 'clientEncryption': throw new Unsupported('client-side encryption'); diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index 5556ea8..0b17127 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -18,7 +18,7 @@ # hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites # it -- the only assertion this runner declines to make). -total 159 pass 132 fail 196 skip 175 files 0 errored +total 173 pass 118 fail 196 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -45,19 +45,19 @@ bulkWrite-deleteOne-rawdata.json 1 0 1 bulkWrite-errorResponse.json 0 0 1 bulkWrite-insertOne-dots_and_dollars.json 3 1 1 bulkWrite-replaceOne-dots_and_dollars.json 2 1 1 -bulkWrite-replaceOne-hint-unacknowledged.json 0 2 0 +bulkWrite-replaceOne-hint-unacknowledged.json 2 0 0 bulkWrite-replaceOne-let.json 0 1 1 bulkWrite-replaceOne-rawdata.json 1 0 1 bulkWrite-replaceOne-sort.json 0 1 1 bulkWrite-update-hint.json 3 0 0 bulkWrite-update-validation.json 3 0 0 bulkWrite-updateMany-dots_and_dollars.json 0 0 4 -bulkWrite-updateMany-hint-unacknowledged.json 0 2 0 +bulkWrite-updateMany-hint-unacknowledged.json 2 0 0 bulkWrite-updateMany-let.json 0 1 1 bulkWrite-updateMany-pipeline.json 0 1 0 bulkWrite-updateMany-rawdata.json 0 1 1 bulkWrite-updateOne-dots_and_dollars.json 0 0 4 -bulkWrite-updateOne-hint-unacknowledged.json 0 2 0 +bulkWrite-updateOne-hint-unacknowledged.json 2 0 0 bulkWrite-updateOne-let.json 0 1 1 bulkWrite-updateOne-pipeline.json 0 1 0 bulkWrite-updateOne-rawdata.json 0 1 1 @@ -158,14 +158,14 @@ insertMany-dots_and_dollars.json 3 1 1 insertMany-rawdata.json 1 0 1 insertMany.json 3 0 0 insertOne-comment.json 2 0 1 -insertOne-dots_and_dollars.json 5 3 1 +insertOne-dots_and_dollars.json 6 2 1 insertOne-errorResponse.json 0 0 1 insertOne-rawdata.json 1 0 1 insertOne.json 1 0 0 replaceOne-collation.json 0 1 0 replaceOne-comment.json 2 0 1 -replaceOne-dots_and_dollars.json 2 2 1 -replaceOne-hint-unacknowledged.json 0 2 0 +replaceOne-dots_and_dollars.json 3 1 1 +replaceOne-hint-unacknowledged.json 2 0 0 replaceOne-hint.json 2 0 0 replaceOne-let.json 0 1 1 replaceOne-rawdata.json 1 0 1 @@ -176,7 +176,7 @@ updateMany-arrayFilters.json 0 3 0 updateMany-collation.json 0 1 0 updateMany-comment.json 2 0 1 updateMany-dots_and_dollars.json 0 0 4 -updateMany-hint-unacknowledged.json 0 2 0 +updateMany-hint-unacknowledged.json 2 0 0 updateMany-hint.json 2 0 0 updateMany-let.json 0 1 1 updateMany-pipeline.json 0 1 0 @@ -188,7 +188,7 @@ updateOne-collation.json 0 1 0 updateOne-comment.json 2 0 1 updateOne-dots_and_dollars.json 0 0 4 updateOne-errorResponse.json 0 0 1 -updateOne-hint-unacknowledged.json 0 2 0 +updateOne-hint-unacknowledged.json 2 0 0 updateOne-hint.json 2 0 0 updateOne-let.json 0 1 1 updateOne-pipeline.json 0 1 0 @@ -226,15 +226,15 @@ bulkWrite-comment.json SKIP BulkWrite with comment - pre 4.4 needs server <= 4.2 bulkWrite-delete-hint-serverError.json SKIP * needs server <= 4.3.3 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual -bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ +bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ bulkWrite-deleteMany-let.json SKIP BulkWrite deleteMany with let option needs server >= 5.0 bulkWrite-deleteMany-let.json FAIL BulkWrite deleteMany with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteMany-rawdata.json SKIP BulkWrite deleteMany with rawData option needs server >= 8.2.0 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual -bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual +bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ +bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ bulkWrite-deleteOne-let.json SKIP BulkWrite deleteOne with let option needs server >= 5.0 bulkWrite-deleteOne-let.json FAIL BulkWrite deleteOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteOne-rawdata.json SKIP BulkWrite deleteOne with rawData option needs server >= 8.2.0 @@ -243,8 +243,6 @@ bulkWrite-insertOne-dots_and_dollars.json SKIP Inserting document with top-level bulkWrite-insertOne-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-dots_and_dollars.json SKIP Replacing document with dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-replaceOne-dots_and_dollars.json FAIL Replacing document with dollar-prefixed key in embedded doc on pre-5.0 server yields server-side error bulkWrite: expected an error, the operation succeeded -bulkWrite-replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual -bulkWrite-replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual bulkWrite-replaceOne-let.json SKIP BulkWrite replaceOne with let option needs server >= 5.0 bulkWrite-replaceOne-let.json FAIL BulkWrite replaceOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-rawdata.json SKIP BulkWrite replaceOne with rawData option needs server >= 8.2.0 @@ -254,8 +252,6 @@ bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-lev bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 -bulkWrite-updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual -bulkWrite-updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual bulkWrite-updateMany-let.json SKIP BulkWrite updateMany with let option needs server >= 5.0 bulkWrite-updateMany-let.json FAIL BulkWrite updateMany with let option unsupported (server-side error) bulkWrite: error message "update spec requires u" does not contain "'update.let' is an unknown field" bulkWrite-updateMany-pipeline.json FAIL UpdateMany in bulk write using pipelines MongoBulkWriteError: update spec requires u @@ -265,8 +261,6 @@ bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set top-leve bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 bulkWrite-updateOne-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 -bulkWrite-updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual -bulkWrite-updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual bulkWrite-updateOne-let.json SKIP BulkWrite updateOne with let option needs server >= 5.0 bulkWrite-updateOne-let.json FAIL BulkWrite updateOne with let option unsupported (server-side error) bulkWrite: error message "update spec requires u" does not contain "'update.let' is an unknown field" bulkWrite-updateOne-pipeline.json FAIL UpdateOne in bulk write using pipelines MongoBulkWriteError: update spec requires u @@ -326,8 +320,8 @@ deleteMany-comment.json SKIP deleteMany with comment - pre 4.4 needs server <= 4 deleteMany-hint-serverError.json SKIP * needs server <= 4.3.3 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual -deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual +deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ +deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ deleteMany-let.json SKIP deleteMany with let option needs server >= 5.0 deleteMany-let.json FAIL deleteMany with let option unsupported (server-side error) deleteMany: expected an error, the operation succeeded deleteMany-rawdata.json SKIP deleteMany with rawData option needs server >= 8.2.0 @@ -337,8 +331,8 @@ deleteOne-errorResponse.json SKIP delete operations support errorResponse assert deleteOne-hint-serverError.json SKIP * needs server <= 4.3.3 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server events client0[0].command.writeConcern: missing from actual -deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server events client0[0].command.writeConcern: missing from actual +deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ +deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ deleteOne-let.json SKIP deleteOne with let option needs server >= 5.0 deleteOne-let.json FAIL deleteOne with let option unsupported (server-side error) deleteOne: expected an error, the operation succeeded deleteOne-rawdata.json SKIP deleteOne with rawData option needs server >= 8.2.0 @@ -373,8 +367,8 @@ findOneAndDelete-comment.json SKIP findOneAndDelete with comment - pre 4.4 needs findOneAndDelete-hint-serverError.json SKIP * needs server <= 4.3.3 findOneAndDelete-hint-unacknowledged.json SKIP Unacknowledged findOneAndDelete with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 findOneAndDelete-hint-unacknowledged.json SKIP Unacknowledged findOneAndDelete with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -findOneAndDelete-hint-unacknowledged.json FAIL Unacknowledged findOneAndDelete with hint string on 4.4+ server findOneAndDelete: expected null, got {"_id":2,"x":22} -findOneAndDelete-hint-unacknowledged.json FAIL Unacknowledged findOneAndDelete with hint document on 4.4+ server findOneAndDelete: expected null, got {"_id":2,"x":22} +findOneAndDelete-hint-unacknowledged.json FAIL Unacknowledged findOneAndDelete with hint string on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ +findOneAndDelete-hint-unacknowledged.json FAIL Unacknowledged findOneAndDelete with hint document on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ findOneAndDelete-let.json SKIP findOneAndDelete with let option needs server >= 5.0 findOneAndDelete-let.json FAIL findOneAndDelete with let option unsupported (server-side error) findOneAndDelete: expected an error, the operation succeeded findOneAndDelete-rawdata.json SKIP findOneAndDelete with rawData option needs server >= 8.2.0 @@ -385,8 +379,8 @@ findOneAndReplace-dots_and_dollars.json FAIL Replacing document with dollar-pref findOneAndReplace-hint-serverError.json SKIP * needs server <= 4.3.0 findOneAndReplace-hint-unacknowledged.json SKIP Unacknowledged findOneAndReplace with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 findOneAndReplace-hint-unacknowledged.json SKIP Unacknowledged findOneAndReplace with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -findOneAndReplace-hint-unacknowledged.json FAIL Unacknowledged findOneAndReplace with hint string on 4.4+ server findOneAndReplace: expected null, got {"_id":2,"x":22} -findOneAndReplace-hint-unacknowledged.json FAIL Unacknowledged findOneAndReplace with hint document on 4.4+ server findOneAndReplace: expected null, got {"_id":2,"x":22} +findOneAndReplace-hint-unacknowledged.json FAIL Unacknowledged findOneAndReplace with hint string on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ +findOneAndReplace-hint-unacknowledged.json FAIL Unacknowledged findOneAndReplace with hint document on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ findOneAndReplace-let.json SKIP findOneAndReplace with let option needs server >= 5.0 findOneAndReplace-let.json FAIL findOneAndReplace with let option unsupported (server-side error) findOneAndReplace: expected an error, the operation succeeded findOneAndReplace-rawdata.json SKIP findOneAndReplace with rawData option needs server >= 8.2.0 @@ -410,8 +404,8 @@ findOneAndUpdate-errorResponse.json SKIP findOneAndUpdate document validation er findOneAndUpdate-hint-serverError.json SKIP * needs server <= 4.3.0 findOneAndUpdate-hint-unacknowledged.json SKIP Unacknowledged findOneAndUpdate with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 findOneAndUpdate-hint-unacknowledged.json SKIP Unacknowledged findOneAndUpdate with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -findOneAndUpdate-hint-unacknowledged.json FAIL Unacknowledged findOneAndUpdate with hint string on 4.4+ server findOneAndUpdate: expected null, got {"_id":2,"x":22} -findOneAndUpdate-hint-unacknowledged.json FAIL Unacknowledged findOneAndUpdate with hint document on 4.4+ server findOneAndUpdate: expected null, got {"_id":2,"x":22} +findOneAndUpdate-hint-unacknowledged.json FAIL Unacknowledged findOneAndUpdate with hint string on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ +findOneAndUpdate-hint-unacknowledged.json FAIL Unacknowledged findOneAndUpdate with hint document on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ findOneAndUpdate-let.json SKIP findOneAndUpdate with let option needs server >= 5.0 findOneAndUpdate-let.json FAIL findOneAndUpdate with let option unsupported (server-side error) findOneAndUpdate: expected an error, the operation succeeded findOneAndUpdate-pipeline.json FAIL FindOneAndUpdate using pipelines MongoServerError: update must be a document @@ -428,16 +422,12 @@ insertOne-comment.json SKIP insertOne with comment - pre 4.4 needs server <= 4.2 insertOne-dots_and_dollars.json SKIP Inserting document with top-level dollar-prefixed key on 5.0+ server needs server >= 5.0 insertOne-dots_and_dollars.json FAIL Inserting document with top-level dollar-prefixed key on pre-5.0 server yields server-side error insertOne: expected an error, the operation succeeded insertOne-dots_and_dollars.json FAIL Inserting document with dollar-prefixed key in _id yields server-side error insertOne: expected an error, the operation succeeded -insertOne-dots_and_dollars.json FAIL Unacknowledged write using dollar-prefixed or dotted keys may be silently rejected on pre-5.0 server events client0[0].command.writeConcern: missing from actual insertOne-errorResponse.json SKIP insert operations support errorResponse assertions runner: failPoint insertOne-rawdata.json SKIP insertOne with rawData option needs server >= 8.2.0 replaceOne-collation.json FAIL ReplaceOne when one document matches with collation replaceOne.matchedCount: expected 1, got 0 replaceOne-comment.json SKIP ReplaceOne with comment - pre 4.4 needs server <= 4.2.99 replaceOne-dots_and_dollars.json SKIP Replacing document with dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 replaceOne-dots_and_dollars.json FAIL Replacing document with dollar-prefixed key in embedded doc on pre-5.0 server yields server-side error replaceOne: expected an error, the operation succeeded -replaceOne-dots_and_dollars.json FAIL Unacknowledged write using dollar-prefixed or dotted keys may be silently rejected on pre-5.0 server events client0[0].command.writeConcern: missing from actual -replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual -replaceOne-hint-unacknowledged.json FAIL Unacknowledged replaceOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual replaceOne-let.json SKIP ReplaceOne with let option needs server >= 5.0 replaceOne-let.json FAIL ReplaceOne with let option unsupported (server-side error) replaceOne: expected an error, the operation succeeded replaceOne-rawdata.json SKIP ReplaceOne with rawData option needs server >= 8.2.0 @@ -452,8 +442,6 @@ updateMany-dots_and_dollars.json SKIP Updating document to set top-level dollar- updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 updateMany-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 -updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual -updateMany-hint-unacknowledged.json FAIL Unacknowledged updateMany with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual updateMany-let.json SKIP updateMany with let option needs server >= 5.0 updateMany-let.json FAIL updateMany with let option unsupported (server-side error) updateMany: error message "update spec requires u" does not contain "'update.let' is an unknown field" updateMany-pipeline.json FAIL UpdateMany using pipelines MongoServerError: update spec requires u @@ -470,8 +458,6 @@ updateOne-dots_and_dollars.json SKIP Updating document to set top-level dotted k updateOne-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 updateOne-dots_and_dollars.json SKIP Updating document to set dotted key in embedded doc on 5.0+ server needs server >= 5.0 updateOne-errorResponse.json SKIP update operations support errorResponse assertions runner: failPoint -updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint string on 4.2+ server events client0[0].command.writeConcern: missing from actual -updateOne-hint-unacknowledged.json FAIL Unacknowledged updateOne with hint document on 4.2+ server events client0[0].command.writeConcern: missing from actual updateOne-let.json SKIP UpdateOne with let option needs server >= 5.0 updateOne-let.json FAIL UpdateOne with let option unsupported (server-side error) updateOne: error message "update spec requires u" does not contain "'update.let' is an unknown field" updateOne-pipeline.json FAIL UpdateOne using pipelines MongoServerError: update spec requires u -- 2.39.5 From 548c882d47a569422898aa2c79aa8f0851c142e2 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:23:58 +0300 Subject: [PATCH 19/37] commands: the wire version says what the version string says `buildInfo` has always reported 4.4.0 and the handshake has always reported maxWireVersion 8, which is 4.2. A driver believes the wire version: it refused client-side to send `hint` on an unacknowledged delete or findAndModify (the error is "only supported on MongoDB 4.4+", raised without a round trip), and withheld `comment` from getMore, listCollections and listDatabases (lib/operations/get_more.js:43 and its neighbours). Both are things this engine handles -- the acknowledged hint suites pass, and getMore ignores fields it does not know -- so the effect was purely the number disagreeing with itself. The 8 was not arbitrary. The comment above it tied it to omitting `topologyVersion`, which is what keeps a driver off the streaming hello protocol we do not implement -- a real bug, once visible as Compass reconnecting every heartbeat. Checked before touching it, in the driver rather than from memory: `useStreamingProtocol` (lib/sdam/monitor.js:154) returns false whenever `topologyVersion` is absent and never looks at the wire version at all. The omission is the whole mechanism; the wire version was a second line of defence that never existed. The comment now says so. A test asserts the two agree, so they cannot drift apart again silently, which is the actual defect here -- not the value. Spec suites 173/118/196 -> 189/102/196: 16 cases fixed, none broken. Ten are the unacknowledged-hint cases the previous commit uncovered, six are `comment` forwarding. 166/166 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz, e2e 49, e2e2 concurrent 2 + crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/commands.zig | 50 ++++++++++++++++++++++++++++++++++++---- tests/spec/scorecard.txt | 38 +++++++++--------------------- 2 files changed, 57 insertions(+), 31 deletions(-) diff --git a/src/commands.zig b/src/commands.zig index 4f92d5b..3f7c05b 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -257,7 +257,13 @@ fn add_server_info(ctx: *Context, reply: *wire.Reply) !void { try reply.put("logicalSessionTimeoutMinutes", .{ .int32 = 30 }); try reply.put("connectionId", .{ .int32 = @intCast(ctx.connection_id) }); try reply.put("minWireVersion", .{ .int32 = 0 }); - try reply.put("maxWireVersion", .{ .int32 = 8 }); + // 9, because this server calls itself 4.4.0 in `buildInfo` and 4.4 is + // wire 9. Reporting 8 was reporting 4.2, and a driver believes the wire + // version over the string: it refused client-side to send `hint` on an + // unacknowledged delete or findAndModify (ten spec cases), and withheld + // `comment` from getMore, listCollections and listDatabases. Both are + // things this engine handles. + try reply.put("maxWireVersion", .{ .int32 = 9 }); try reply.put("readOnly", .{ .bool = false }); // Deliberately no `topologyVersion`. A driver treats its presence as @@ -269,8 +275,13 @@ fn add_server_info(ctx: *Context, reply: *wire.Reply) !void { // fails the heartbeat ("Server ended moreToCome unexpectedly"), drops // the connection and resets its pool — a connect/disconnect loop once // per heartbeat, which is what MongoDB Compass showed. Omitting the - // field keeps monitoring on the polling path, which we do implement, - // and matches maxWireVersion 8: streaming hello arrived in wire 9. + // field keeps monitoring on the polling path, which we do implement. + // + // This is the whole of the mechanism, and it does not depend on the wire + // version: `useStreamingProtocol` (driver lib/sdam/monitor.js:154) polls + // whenever `topologyVersion` is absent, whatever else the handshake said. + // Checked when maxWireVersion went to 9 above, since the old comment here + // leaned on 8 as a second line of defence that never existed. } fn cmd_hello(ctx: *Context, _: *wire.Message, reply: *wire.Reply) !void { @@ -2718,7 +2729,38 @@ test "ping and hello replies parse" { try testing.expectEqual(@as(f64, 1.0), ok.double); const primary = bson.get_pair(reply2.pairs.items, "isWritablePrimary").?; try testing.expect(primary.bool); - try testing.expectEqual(@as(i32, 8), bson.get_pair(reply2.pairs.items, "maxWireVersion").?.int32); + try testing.expectEqual(@as(i32, 9), bson.get_pair(reply2.pairs.items, "maxWireVersion").?.int32); +} + +test "the wire version agrees with the version the server calls itself" { + // These two are read by different parts of a driver -- the handshake picks + // features off the wire version, `runOnRequirements` in the spec suites + // reads the string -- and when they disagreed the driver believed the wire + // version and withheld commands the version string promised. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + var hello = wire.Reply.init(testing.allocator); + defer hello.deinit(); + var hello_msg = try parse_fake_msg("hello", .null, &.{}); + defer hello_msg.deinit(); + try dispatch(&ctx, &hello_msg, &hello); + + var info = wire.Reply.init(testing.allocator); + defer info.deinit(); + var info_msg = try parse_fake_msg("buildInfo", .null, &.{}); + defer info_msg.deinit(); + try dispatch(&ctx, &info_msg, &info); + + // The mapping is the server's own: 4.2 is wire 8, 4.4 is wire 9. + const wire_version = bson.get_pair(hello.pairs.items, "maxWireVersion").?.int32; + const version = bson.get_pair(info.pairs.items, "version").?.string; + const expected: i32 = if (std.mem.startsWith(u8, version, "4.4.")) 9 else if (std.mem.startsWith(u8, version, "4.2.")) 8 else -1; + try testing.expectEqual(expected, wire_version); } test "handshake does not advertise the streaming hello protocol" { diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index 0b17127..6d1e8a6 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -1,7 +1,7 @@ # MongoDB spec-test scorecard (PLAN D2) -- crud + aggregate, unified format # specs: mongodb/specifications @ 615e0f9 (pinned in tests/spec/fetch.sh) # driver: mongodb@7.5.0 (pinned in tests/e2e/package-lock.json) -# server: MultiforaDB reporting version 4.4.0, maxWireVersion 8 +# server: MultiforaDB reporting version 4.4.0, maxWireVersion 9 # reproduce: bash tests/spec/fetch.sh && node tests/spec/run.js --scorecard # # What SKIP means here, so the totals are not read as better than they are: @@ -18,7 +18,7 @@ # hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites # it -- the only assertion this runner declines to make). -total 173 pass 118 fail 196 skip 175 files 0 errored +total 189 pass 102 fail 196 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -30,16 +30,16 @@ aggregate-out-readConcern.json 0 0 4 aggregate-out.json 0 2 0 aggregate-rawdata.json 1 0 1 aggregate-write-readPreference.json 0 0 4 -aggregate.json 4 1 2 +aggregate.json 5 0 2 bulkWrite-arrayFilters.json 0 3 0 bulkWrite-collation.json 0 2 0 bulkWrite-comment.json 2 0 1 bulkWrite-delete-hint-serverError.json 0 0 2 bulkWrite-delete-hint.json 2 0 0 -bulkWrite-deleteMany-hint-unacknowledged.json 0 2 2 +bulkWrite-deleteMany-hint-unacknowledged.json 2 0 2 bulkWrite-deleteMany-let.json 0 1 1 bulkWrite-deleteMany-rawdata.json 1 0 1 -bulkWrite-deleteOne-hint-unacknowledged.json 0 2 2 +bulkWrite-deleteOne-hint-unacknowledged.json 2 0 2 bulkWrite-deleteOne-let.json 0 1 1 bulkWrite-deleteOne-rawdata.json 1 0 1 bulkWrite-errorResponse.json 0 0 1 @@ -93,7 +93,7 @@ db-aggregate.json 0 2 0 deleteMany-collation.json 0 1 0 deleteMany-comment.json 2 0 1 deleteMany-hint-serverError.json 0 0 2 -deleteMany-hint-unacknowledged.json 0 2 2 +deleteMany-hint-unacknowledged.json 2 0 2 deleteMany-hint.json 2 0 0 deleteMany-let.json 0 1 1 deleteMany-rawdata.json 1 0 1 @@ -102,7 +102,7 @@ deleteOne-collation.json 0 1 0 deleteOne-comment.json 2 0 1 deleteOne-errorResponse.json 0 0 1 deleteOne-hint-serverError.json 0 0 2 -deleteOne-hint-unacknowledged.json 0 2 2 +deleteOne-hint-unacknowledged.json 2 0 2 deleteOne-hint.json 2 0 0 deleteOne-let.json 0 1 1 deleteOne-rawdata.json 1 0 1 @@ -118,7 +118,7 @@ estimatedDocumentCount.json 2 1 3 find-allowdiskuse-serverError.json 0 0 2 find-allowdiskuse.json 3 0 0 find-collation.json 0 1 0 -find-comment.json 0 3 2 +find-comment.json 1 2 2 find-let.json 0 1 1 find-rawdata.json 1 0 1 find.json 5 0 0 @@ -126,7 +126,7 @@ findOne.json 1 1 0 findOneAndDelete-collation.json 0 1 0 findOneAndDelete-comment.json 2 0 1 findOneAndDelete-hint-serverError.json 0 0 2 -findOneAndDelete-hint-unacknowledged.json 0 2 2 +findOneAndDelete-hint-unacknowledged.json 2 0 2 findOneAndDelete-hint.json 2 0 0 findOneAndDelete-let.json 0 1 1 findOneAndDelete-rawdata.json 1 0 1 @@ -135,7 +135,7 @@ findOneAndReplace-collation.json 0 1 0 findOneAndReplace-comment.json 2 0 1 findOneAndReplace-dots_and_dollars.json 2 1 1 findOneAndReplace-hint-serverError.json 0 0 2 -findOneAndReplace-hint-unacknowledged.json 0 2 2 +findOneAndReplace-hint-unacknowledged.json 2 0 2 findOneAndReplace-hint.json 2 0 0 findOneAndReplace-let.json 0 1 1 findOneAndReplace-rawdata.json 1 0 1 @@ -147,7 +147,7 @@ findOneAndUpdate-comment.json 0 2 1 findOneAndUpdate-dots_and_dollars.json 0 0 4 findOneAndUpdate-errorResponse.json 0 1 1 findOneAndUpdate-hint-serverError.json 0 0 2 -findOneAndUpdate-hint-unacknowledged.json 0 2 2 +findOneAndUpdate-hint-unacknowledged.json 2 0 2 findOneAndUpdate-hint.json 2 0 0 findOneAndUpdate-let.json 0 1 1 findOneAndUpdate-pipeline.json 0 1 0 @@ -215,7 +215,6 @@ aggregate-out.json FAIL Aggregate with $out and batch size of 0 MongoServerError aggregate-rawdata.json SKIP Aggregate with rawData option needs server >= 8.2.0 aggregate-write-readPreference.json SKIP * needs topology replicaset/sharded/load-balanced aggregate.json SKIP aggregate with a document comment - pre 4.4 needs server <= 4.2.99 -aggregate.json FAIL aggregate with comment sets comment on getMore events client0[1].command.comment: missing from actual aggregate.json SKIP aggregate with comment does not set comment on getMore - pre 4.4 needs server <= 4.3.99 bulkWrite-arrayFilters.json FAIL BulkWrite updateOne with arrayFilters outcome crud-tests.test[0].y: expected an array, got {"$[i]":{"b":2}} bulkWrite-arrayFilters.json FAIL BulkWrite updateMany with arrayFilters outcome crud-tests.test[0].y: expected an array, got {"$[i]":{"b":2}} @@ -226,15 +225,11 @@ bulkWrite-comment.json SKIP BulkWrite with comment - pre 4.4 needs server <= 4.2 bulkWrite-delete-hint-serverError.json SKIP * needs server <= 4.3.3 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ -bulkWrite-deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ bulkWrite-deleteMany-let.json SKIP BulkWrite deleteMany with let option needs server >= 5.0 bulkWrite-deleteMany-let.json FAIL BulkWrite deleteMany with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteMany-rawdata.json SKIP BulkWrite deleteMany with rawData option needs server >= 8.2.0 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 bulkWrite-deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ -bulkWrite-deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server MongoBulkWriteError: hint for the delete command is only supported on MongoDB 4.4+ bulkWrite-deleteOne-let.json SKIP BulkWrite deleteOne with let option needs server >= 5.0 bulkWrite-deleteOne-let.json FAIL BulkWrite deleteOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-deleteOne-rawdata.json SKIP BulkWrite deleteOne with rawData option needs server >= 8.2.0 @@ -320,8 +315,6 @@ deleteMany-comment.json SKIP deleteMany with comment - pre 4.4 needs server <= 4 deleteMany-hint-serverError.json SKIP * needs server <= 4.3.3 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteMany-hint-unacknowledged.json SKIP Unacknowledged deleteMany with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint string on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ -deleteMany-hint-unacknowledged.json FAIL Unacknowledged deleteMany with hint document on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ deleteMany-let.json SKIP deleteMany with let option needs server >= 5.0 deleteMany-let.json FAIL deleteMany with let option unsupported (server-side error) deleteMany: expected an error, the operation succeeded deleteMany-rawdata.json SKIP deleteMany with rawData option needs server >= 8.2.0 @@ -331,8 +324,6 @@ deleteOne-errorResponse.json SKIP delete operations support errorResponse assert deleteOne-hint-serverError.json SKIP * needs server <= 4.3.3 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 deleteOne-hint-unacknowledged.json SKIP Unacknowledged deleteOne with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint string on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ -deleteOne-hint-unacknowledged.json FAIL Unacknowledged deleteOne with hint document on 4.4+ server MongoCompatibilityError: hint for the delete command is only supported on MongoDB 4.4+ deleteOne-let.json SKIP deleteOne with let option needs server >= 5.0 deleteOne-let.json FAIL deleteOne with let option unsupported (server-side error) deleteOne: expected an error, the operation succeeded deleteOne-rawdata.json SKIP deleteOne with rawData option needs server >= 8.2.0 @@ -356,7 +347,6 @@ find-collation.json FAIL Find with a collation find: expected 1 elements, got 0 find-comment.json FAIL find with string comment find[0]: unexpected extra keys ["x"] find-comment.json FAIL find with document comment find[0]: unexpected extra keys ["x"] find-comment.json SKIP find with document comment - pre 4.4 needs server <= 4.2.99 -find-comment.json FAIL find with comment sets comment on getMore events client0[1].command.comment: missing from actual find-comment.json SKIP find with comment does not set comment on getMore - pre 4.4 needs server <= 4.3.99 find-let.json SKIP Find with let option needs server >= 5.0 find-let.json FAIL Find with let option unsupported (server-side error) find: expected an error, the operation succeeded @@ -367,8 +357,6 @@ findOneAndDelete-comment.json SKIP findOneAndDelete with comment - pre 4.4 needs findOneAndDelete-hint-serverError.json SKIP * needs server <= 4.3.3 findOneAndDelete-hint-unacknowledged.json SKIP Unacknowledged findOneAndDelete with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 findOneAndDelete-hint-unacknowledged.json SKIP Unacknowledged findOneAndDelete with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -findOneAndDelete-hint-unacknowledged.json FAIL Unacknowledged findOneAndDelete with hint string on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ -findOneAndDelete-hint-unacknowledged.json FAIL Unacknowledged findOneAndDelete with hint document on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ findOneAndDelete-let.json SKIP findOneAndDelete with let option needs server >= 5.0 findOneAndDelete-let.json FAIL findOneAndDelete with let option unsupported (server-side error) findOneAndDelete: expected an error, the operation succeeded findOneAndDelete-rawdata.json SKIP findOneAndDelete with rawData option needs server >= 8.2.0 @@ -379,8 +367,6 @@ findOneAndReplace-dots_and_dollars.json FAIL Replacing document with dollar-pref findOneAndReplace-hint-serverError.json SKIP * needs server <= 4.3.0 findOneAndReplace-hint-unacknowledged.json SKIP Unacknowledged findOneAndReplace with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 findOneAndReplace-hint-unacknowledged.json SKIP Unacknowledged findOneAndReplace with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -findOneAndReplace-hint-unacknowledged.json FAIL Unacknowledged findOneAndReplace with hint string on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ -findOneAndReplace-hint-unacknowledged.json FAIL Unacknowledged findOneAndReplace with hint document on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ findOneAndReplace-let.json SKIP findOneAndReplace with let option needs server >= 5.0 findOneAndReplace-let.json FAIL findOneAndReplace with let option unsupported (server-side error) findOneAndReplace: expected an error, the operation succeeded findOneAndReplace-rawdata.json SKIP findOneAndReplace with rawData option needs server >= 8.2.0 @@ -404,8 +390,6 @@ findOneAndUpdate-errorResponse.json SKIP findOneAndUpdate document validation er findOneAndUpdate-hint-serverError.json SKIP * needs server <= 4.3.0 findOneAndUpdate-hint-unacknowledged.json SKIP Unacknowledged findOneAndUpdate with hint string fails with client-side error on pre-4.4 server needs server <= 4.2.99 findOneAndUpdate-hint-unacknowledged.json SKIP Unacknowledged findOneAndUpdate with hint document fails with client-side error on pre-4.4 server needs server <= 4.2.99 -findOneAndUpdate-hint-unacknowledged.json FAIL Unacknowledged findOneAndUpdate with hint string on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ -findOneAndUpdate-hint-unacknowledged.json FAIL Unacknowledged findOneAndUpdate with hint document on 4.4+ server MongoCompatibilityError: hint for the findAndModify command is only supported on MongoDB 4.4+ findOneAndUpdate-let.json SKIP findOneAndUpdate with let option needs server >= 5.0 findOneAndUpdate-let.json FAIL findOneAndUpdate with let option unsupported (server-side error) findOneAndUpdate: expected an error, the operation succeeded findOneAndUpdate-pipeline.json FAIL FindOneAndUpdate using pipelines MongoServerError: update must be a document -- 2.39.5 From 6d5c860e11a79ae4873ca20e0d58d4ae9c1b9ce8 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:25:28 +0300 Subject: [PATCH 20/37] tests/spec: an event's command is compared in the shape it was sent A command-monitoring event hands over the command as the driver holds it in memory, and that is not always the shape it puts on the wire: a sort is a JS `Map` (driver lib/sort.js). `Object.keys` on a Map is empty, so the matcher reported every key of an expected sort as missing from a command that in fact carried it -- five cases, all of them the runner's fault and none the engine's. This one is worth the paragraph because of how well it hides. EJSON serializes a Map exactly like a document, so `MFDB_DUMP_EVENTS` prints `"sort":{"_id":1}` next to a failure that says `sort._id` is missing, and the dump -- the tool built for exactly this triage in the commit that added the buffers -- reads as evidence that the matcher is wrong about something else. It took `Object.keys(formatSort({_id: 1}))` returning `[]` to see it. Converted for the comparison only, and at every depth, since a sort also appears inside `updates[i]`. `match` stays a plain reading of the spec's Evaluating Matches with no driver knowledge in it. Mutation-checked: pass the event's own value through and findOne.json "FindOne with filter, sort, and skip" goes red again with the original message. 189/102/196 becomes 194/97/196. --- tests/spec/run.js | 25 +++++++++++++++++++++++-- tests/spec/scorecard.txt | 17 ++++++----------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/spec/run.js b/tests/spec/run.js index b54fa26..406267a 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -669,6 +669,27 @@ function verifyEvents(expectEvents, entities) { } } +// A monitored event hands over the command as the driver holds it in memory, +// which is not always the shape it puts on the wire: a sort is a JS `Map` +// (driver lib/sort.js). `Object.keys` on a Map is empty, so the matcher +// reported every key of an expected sort as missing from a command that in +// fact carried it. EJSON serializes a Map exactly like a document, which is +// why a dump of the event looks perfectly correct and this had to be measured +// rather than read. Converted only for the comparison, and at every depth, +// since a sort also appears inside `updates[i]`. +function wireShape(v) { + if (v instanceof Map) { + const out = {}; + for (const [k, x] of v) out[k] = wireShape(x); + return out; + } + if (Array.isArray(v)) return v.map(wireShape); + if (!isPlainDoc(v)) return v; + const out = {}; + for (const [k, x] of Object.entries(v)) out[k] = wireShape(x); + return out; +} + function matchEvent(expected, actual, entities, pathStr) { const [name, body] = Object.entries(expected)[0]; if (!(name in COMMAND_EVENTS)) throw new Unsupported(`event ${name}`); @@ -690,10 +711,10 @@ function matchEvent(expected, actual, entities, pathStr) { if (isPlainDoc(v) && Object.prototype.hasOwnProperty.call(v, 'maxTimeMS')) { throw new Unsupported('maxTimeMS in an expected command (CSOT rewrites it)'); } - match(v, actual.ev[k], entities, `${pathStr}.${k}`, true); + match(v, wireShape(actual.ev[k]), entities, `${pathStr}.${k}`, true); break; case 'reply': - match(v, actual.ev[k], entities, `${pathStr}.${k}`, true); + match(v, wireShape(actual.ev[k]), entities, `${pathStr}.${k}`, true); break; case 'commandName': case 'databaseName': diff --git a/tests/spec/scorecard.txt b/tests/spec/scorecard.txt index 6d1e8a6..e690acb 100644 --- a/tests/spec/scorecard.txt +++ b/tests/spec/scorecard.txt @@ -18,7 +18,7 @@ # hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites # it -- the only assertion this runner declines to make). -total 189 pass 102 fail 196 skip 175 files 0 errored +total 194 pass 97 fail 196 skip 175 files 0 errored # per-file: name pass fail skip aggregate-allowdiskuse.json 3 0 0 @@ -48,7 +48,7 @@ bulkWrite-replaceOne-dots_and_dollars.json 2 1 1 bulkWrite-replaceOne-hint-unacknowledged.json 2 0 0 bulkWrite-replaceOne-let.json 0 1 1 bulkWrite-replaceOne-rawdata.json 1 0 1 -bulkWrite-replaceOne-sort.json 0 1 1 +bulkWrite-replaceOne-sort.json 1 0 1 bulkWrite-update-hint.json 3 0 0 bulkWrite-update-validation.json 3 0 0 bulkWrite-updateMany-dots_and_dollars.json 0 0 4 @@ -61,7 +61,7 @@ bulkWrite-updateOne-hint-unacknowledged.json 2 0 0 bulkWrite-updateOne-let.json 0 1 1 bulkWrite-updateOne-pipeline.json 0 1 0 bulkWrite-updateOne-rawdata.json 0 1 1 -bulkWrite-updateOne-sort.json 0 1 1 +bulkWrite-updateOne-sort.json 1 0 1 bulkWrite.json 10 0 0 bypassDocumentValidation.json 4 5 0 client-bulkWrite-delete-options.json 0 0 2 @@ -122,7 +122,7 @@ find-comment.json 1 2 2 find-let.json 0 1 1 find-rawdata.json 1 0 1 find.json 5 0 0 -findOne.json 1 1 0 +findOne.json 2 0 0 findOneAndDelete-collation.json 0 1 0 findOneAndDelete-comment.json 2 0 1 findOneAndDelete-hint-serverError.json 0 0 2 @@ -169,7 +169,7 @@ replaceOne-hint-unacknowledged.json 2 0 0 replaceOne-hint.json 2 0 0 replaceOne-let.json 0 1 1 replaceOne-rawdata.json 1 0 1 -replaceOne-sort.json 0 1 1 +replaceOne-sort.json 1 0 1 replaceOne-validation.json 1 0 0 replaceOne.json 5 0 0 updateMany-arrayFilters.json 0 3 0 @@ -193,7 +193,7 @@ updateOne-hint.json 2 0 0 updateOne-let.json 0 1 1 updateOne-pipeline.json 0 1 0 updateOne-rawdata.json 1 0 1 -updateOne-sort.json 0 1 1 +updateOne-sort.json 1 0 1 updateOne-validation.json 1 0 0 updateOne.json 4 0 0 @@ -242,7 +242,6 @@ bulkWrite-replaceOne-let.json SKIP BulkWrite replaceOne with let option needs se bulkWrite-replaceOne-let.json FAIL BulkWrite replaceOne with let option unsupported (server-side error) bulkWrite: expected an error, the operation succeeded bulkWrite-replaceOne-rawdata.json SKIP BulkWrite replaceOne with rawData option needs server >= 8.2.0 bulkWrite-replaceOne-sort.json SKIP BulkWrite replaceOne with sort option needs server >= 8.0 -bulkWrite-replaceOne-sort.json FAIL BulkWrite replaceOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-level dollar-prefixed key on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set top-level dotted key on 5.0+ server needs server >= 5.0 bulkWrite-updateMany-dots_and_dollars.json SKIP Updating document to set dollar-prefixed key in embedded doc on 5.0+ server needs server >= 5.0 @@ -262,7 +261,6 @@ bulkWrite-updateOne-pipeline.json FAIL UpdateOne in bulk write using pipelines M bulkWrite-updateOne-rawdata.json SKIP BulkWrite updateOne with rawData option needs server >= 8.2.0 bulkWrite-updateOne-rawdata.json FAIL BulkWrite updateOne with rawData option on less than 8.2.0 - ignore argument MongoBulkWriteError: update spec requires u bulkWrite-updateOne-sort.json SKIP BulkWrite updateOne with sort option needs server >= 8.0 -bulkWrite-updateOne-sort.json FAIL BulkWrite updateOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual bypassDocumentValidation.json FAIL Aggregate with $out passes bypassDocumentValidation: false MongoServerError: Unrecognized pipeline stage name: '$out' bypassDocumentValidation.json FAIL BulkWrite passes bypassDocumentValidation: false events client0[0].command.bypassDocumentValidation: missing from actual bypassDocumentValidation.json FAIL FindOneAndReplace passes bypassDocumentValidation: false events client0[0].command.bypassDocumentValidation: missing from actual @@ -351,7 +349,6 @@ find-comment.json SKIP find with comment does not set comment on getMore - pre 4 find-let.json SKIP Find with let option needs server >= 5.0 find-let.json FAIL Find with let option unsupported (server-side error) find: expected an error, the operation succeeded find-rawdata.json SKIP Find with rawData option needs server >= 8.2.0 -findOne.json FAIL FindOne with filter, sort, and skip events client0[0].command.sort._id: missing from actual findOneAndDelete-collation.json FAIL FindOneAndDelete when one document matches with collation findOneAndDelete: expected a document, got null findOneAndDelete-comment.json SKIP findOneAndDelete with comment - pre 4.4 needs server <= 4.2.99 findOneAndDelete-hint-serverError.json SKIP * needs server <= 4.3.3 @@ -416,7 +413,6 @@ replaceOne-let.json SKIP ReplaceOne with let option needs server >= 5.0 replaceOne-let.json FAIL ReplaceOne with let option unsupported (server-side error) replaceOne: expected an error, the operation succeeded replaceOne-rawdata.json SKIP ReplaceOne with rawData option needs server >= 8.2.0 replaceOne-sort.json SKIP ReplaceOne with sort option needs server >= 8.0 -replaceOne-sort.json FAIL replaceOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual updateMany-arrayFilters.json FAIL UpdateMany when no documents match arrayFilters updateMany.modifiedCount: expected 0, got 2 updateMany-arrayFilters.json FAIL UpdateMany when one document matches arrayFilters updateMany.modifiedCount: expected 1, got 2 updateMany-arrayFilters.json FAIL UpdateMany when multiple documents match arrayFilters outcome crud-v1.coll[0].y: expected an array, got {"$[i]":{"b":2}} @@ -447,4 +443,3 @@ updateOne-let.json FAIL UpdateOne with let option unsupported (server-side error updateOne-pipeline.json FAIL UpdateOne using pipelines MongoServerError: update spec requires u updateOne-rawdata.json SKIP UpdateOne with rawData option needs server >= 8.2.0 updateOne-sort.json SKIP UpdateOne with sort option needs server >= 8.0 -updateOne-sort.json FAIL updateOne with sort option unsupported (server-side error) events client0[0].command.updates[0].sort._id: missing from actual -- 2.39.5 From e65da2740ee41e10ce27abd7044e583b2990dc9c Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 13:28:39 +0300 Subject: [PATCH 21/37] plan: what the event assertions found Four defects and one deliberate non-fix, recorded where the milestone can see them rather than only in five commit messages. The one worth carrying forward: the runner dropped `collectionOptions`, so every "unacknowledged write" case in the corpus had been running an acknowledged write and passing, because the two produce results the expectation accepts either way. Only the wire distinguished them and nothing read the wire. Also states plainly that scorecards from before this are not comparable with ones after, and why `bypassDocumentValidation` is left failing: both available refusals are worse than four undeserved entries in the fail column, and one of them is the shape that turns a scorecard into flattery. --- PLAN.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/PLAN.md b/PLAN.md index 713fd7c..92d54ab 100644 --- a/PLAN.md +++ b/PLAN.md @@ -721,6 +721,70 @@ between `compact` and `checkpoint`; it is out of this scope because it wants its own design pass, and because the free list must not add a second instance of the same shape. +### The spec runner starts reading `expectEvents` + +354 of the 487 cases declare `expectEvents` and the runner read none of them, +so a case could send the wrong command entirely and still be counted a pass as +long as the result came back right. The old `pass` column was an upper bound by +construction and said so; it is now an assertion that the engine answered +correctly **and** was asked the right question. **Scorecards recorded before +this are not comparable with ones recorded after.** + +The totals moved 168/124/195 → 194/97/196 across the commits, but the path +matters more than the endpoints: turning the assertion on cost 34 passes, and +every one of them was a defect the result column could not see. + +What it found, in the order it found them: + +1. **The runner dropped `collectionOptions`.** Every collection entity was + built as `db.collection(name)`, so the 15 entities declaring + `writeConcern: {w: 0}` never got it — **every "unacknowledged write" case in + the corpus was running an acknowledged write.** They passed because the two + produce results a `$$unsetOrMatches` expectation accepts either way. Only + the command on the wire distinguished them, and nothing read the command. +2. **The wire version disagreed with the version string.** `buildInfo` said + 4.4.0, the handshake said maxWireVersion 8, which is 4.2. A driver believes + the wire version: it refused *client-side* to send `hint` on an + unacknowledged delete or findAndModify, and withheld `comment` from + getMore, listCollections and listDatabases. 16 cases. The 8 was not + arbitrary — it was tied to keeping drivers off the streaming hello protocol + — but that turned out to rest entirely on omitting `topologyVersion`, which + is checked in the driver and is the whole mechanism. A test now asserts the + two numbers agree, since drifting apart silently was the actual defect. +3. **`$$unsetOrMatches` was changing root-ness.** The operator wraps a value, + it does not reposition it; the runner matched what stood behind it as a + nested document. 25 cases, all of them results the engine had right. +4. **An event's command is not the shape the driver sends.** A sort is held as + a JS `Map`, so `Object.keys` on it is empty and every expected key read as + missing. It hides well: EJSON prints a Map exactly like a document, so the + event dump reads as evidence the matcher is wrong about something else. + +Two assertions are declined, both enumerated in the runner and in +`scorecard.txt`, and neither can hide anything the engine did: + +- **`maxTimeMS`** — the harness's own doing. Every client carries CSOT + `timeoutMS`, which overwrites `maxTimeMS` with the remaining budget, so the + value on the wire is ours. Refused unconditionally rather than only when it + would fail, so it cannot become a pass by coincidence. One case, and dropping + `timeoutMS` instead would cost far more — it is what replaced the outer race + that once produced ~190 phantom timeout FAILs. +- **`cmap`/`sdam` event types, `ignoreExtraEvents`, `hasServiceId`, + `hasServerConnectionId`** — none occurs in this corpus; reported unsupported + where asserted rather than waived. + +**Left failing on purpose: `bypassDocumentValidation: false`, 4 cases.** +mongodb@7.5.0 strips the field unless it is exactly `true` on the bulk and +findAndModify paths (`lib/bulk/common.js:292`, +`lib/operations/find_and_modify.js:19`) while sending it correctly for single +-document operations, so 4 sibling cases pass and 4 fail on a difference that +is entirely the driver's. The field is built client-side and never reaches the +engine. A refusal was written and thrown away: made unconditional it also +skipped the 4 that legitimately pass, and made conditional it would be a +skip-when-it-would-fail rule, which is the shape that turns a scorecard into +flattery. Four undeserved entries in the fail column is the cheaper error, and +this note is the correction. Revisit when the driver is bumped — which already +has to be its own commit with its own re-recorded scorecard. + --- ## 6. Deferred designs (grill each at its milestone) @@ -759,10 +823,11 @@ of the same shape. the anchor rewritten — resuming at it returned updated documents twice, caught by draining a collection being updated underneath. - Still open in M1: the doc-level free list, sessions plumbing (`lsid` - accepted), and command-monitoring (`expectEvents`) in the spec runner. The - eight reclamation bugs above were cleared first, as preconditions for the - free list rather than as work of their own. + Still open in M1: the doc-level free list and sessions plumbing (`lsid` + accepted). The eight reclamation bugs above were cleared first, as + preconditions for the free list rather than as work of their own; + command-monitoring (`expectEvents`) landed next, so that the free list and + sessions are measured by an instrument that is no longer known to overstate. **A prerequisite the free list must honour**, recorded here while it is still being designed: *an offset that was ever a record start must remain a record start.* `doc_bytes` reads a `u32` length prefix in place, so an -- 2.39.5 From 0034da4293ae344fcfb96dc0a22e049a11c10636 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 15:00:06 +0300 Subject: [PATCH 22/37] wire: parse a logical session id out of a command body An accessor over the command envelope, next to `db_name` and the same shape: called from dispatch, never from `parse`, because a malformed session id is a command that gets an error reply, not a connection that gets torn down. It returns the 16 bytes or names what is wrong; the codes stay in the command layer, where they were measured. The tolerated fields were measured against mongod 8.3.7 rather than recalled, and the measurement contradicted the assumption this was designed on. The design said unknown fields inside `lsid` would be tolerated, on the reasoning that the server tolerates unknown fields everywhere and pinpoint strictness would be inconsistent. mongod answers IDLUnknownField (40415) -- it is strict here and the reasoning was simply wrong. It also accepts `uid`, the hash of the credentials owning the session, which a driver starts sending the moment authentication is on; rejecting that would have broken every command in M7, and the test says so where a future reader will meet it. `txnNumber` and `txnUUID` inside `lsid` are refused. They are not the retryable-write `txnNumber` that sits outside it: together they name an *internal* session, one that runs a transaction on another session's behalf. mongod refuses them on a standalone too. Also `bson.Value.type_name`, which is mongod's name for a type rather than Zig's -- a TypeMismatch message quotes it, and a driver that matches on the text is matching on these. 170/170 unit tests. --- src/bson.zig | 29 +++++++++ src/wire.zig | 163 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) diff --git a/src/bson.zig b/src/bson.zig index 5dfa35e..7532e4e 100644 --- a/src/bson.zig +++ b/src/bson.zig @@ -74,6 +74,35 @@ pub const Value = union(enum) { }; } + /// The name mongod uses for this type in a TypeMismatch message ("is the + /// wrong type 'int', expected type 'object'"). Its own names, not Zig's: + /// a driver that matches on the text is matching on these. + pub fn type_name(self: Value) []const u8 { + return switch (self) { + .double => "double", + .string => "string", + .doc => "object", + .array => "array", + .binary => "binData", + .object_id => "objectId", + .bool => "bool", + .datetime => "date", + .null => "null", + .regex => "regex", + .code => "javascript", + .symbol => "symbol", + .int32 => "int", + .timestamp => "timestamp", + .int64 => "long", + .decimal128 => "decimal", + .min_key => "minKey", + .max_key => "maxKey", + // An unparsed value keeps only its tag byte, and the tags this + // union does not name are the ones nothing here inspects. + .opaque_val => "unknown", + }; + } + pub fn is_number(self: Value) bool { return switch (self) { .double, .int32, .int64 => true, diff --git a/src/wire.zig b/src/wire.zig index df6cc9d..e87559a 100644 --- a/src/wire.zig +++ b/src/wire.zig @@ -185,6 +185,81 @@ pub const Message = struct { }; } + /// A logical session id: a UUID, so 16 bytes of binary subtype 4. + pub const SessionId = [16]u8; + + /// Every way an `lsid` can be malformed, named after what is wrong rather + /// than after the error the caller will send: the codes belong to the + /// command layer, which is where they were measured. + pub const LsidError = error{ + LsidNotDocument, + LsidUnknownField, + LsidIdMissing, + LsidIdNotBinary, + LsidIdNotUuid, + LsidIdWrongLength, + LsidInternalSession, + LsidTxnNumberWithoutTxnUuid, + }; + + /// The logical session id, or null when the command carries no `lsid`. + /// + /// An accessor over the command envelope, like `db_name`: called from + /// dispatch, never from `parse`, because a malformed session id is a + /// command that gets an error reply and not a connection that gets torn + /// down. + /// + /// The fields it tolerates were measured against mongod 8.3.7, not + /// recalled, and the measurement contradicted the assumption it was + /// written on. mongod does *not* ignore unknown fields inside `lsid` -- + /// it answers IDLUnknownField -- and it does accept `uid`, the hash of + /// the credentials that own the session, which a driver sends as soon as + /// authentication is on. Both matter to us: the first because tolerating + /// what the server rejects is the kind of divergence that only shows up + /// under a driver nobody tested, the second because M7 would otherwise + /// break every command. + pub fn lsid(self: *const Message) LsidError!?SessionId { + const v = bson.get_pair(self.body.pairs, "lsid") orelse return null; + const doc = switch (v) { + .doc => |d| d, + else => return error.LsidNotDocument, + }; + + var id: ?bson.Binary = null; + var has_txn_number = false; + var has_txn_uuid = false; + for (doc) |pair| { + if (std.mem.eql(u8, pair.key, "id")) { + id = switch (pair.value) { + .binary => |b| b, + else => return error.LsidIdNotBinary, + }; + } else if (std.mem.eql(u8, pair.key, "uid")) { + // Accepted and ignored: it identifies the user a session + // belongs to, and this server has exactly one. + } else if (std.mem.eql(u8, pair.key, "txnNumber")) { + has_txn_number = true; + } else if (std.mem.eql(u8, pair.key, "txnUUID")) { + has_txn_uuid = true; + } else { + return error.LsidUnknownField; + } + } + + // A `txnNumber` inside the session id is not the retryable-write one + // outside it: together with `txnUUID` the two name an *internal* + // session, which only exists to run a transaction on another + // session's behalf. Neither can mean anything here, and mongod + // refuses them on a standalone too. + if (has_txn_number and !has_txn_uuid) return error.LsidTxnNumberWithoutTxnUuid; + if (has_txn_uuid) return error.LsidInternalSession; + + const bin = id orelse return error.LsidIdMissing; + if (bin.subtype != 4) return error.LsidIdNotUuid; + if (bin.data.len != 16) return error.LsidIdWrongLength; + return bin.data[0..16].*; + } + /// Documents of a batch argument (`documents`, `updates`, `deletes`). /// Drivers send them either as an OP_MSG document sequence or as an array /// inside the command body; callers should not have to care which. The @@ -378,6 +453,94 @@ test "reply serializes to a parseable message" { try testing.expectEqual(@as(usize, 0), msg.seqs.len); } +/// An OP_MSG carrying one body section, for tests that care about the command +/// envelope rather than about framing. +fn fake_op_msg(gpa: std.mem.Allocator, pairs: []const bson.Pair) !Message { + var doc: std.ArrayListUnmanaged(u8) = .empty; + defer doc.deinit(gpa); + try bson.write_doc(pairs, gpa, &doc); + + var buf: std.ArrayListUnmanaged(u8) = .empty; + defer buf.deinit(gpa); + try buf.appendSlice(gpa, &[_]u8{ 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0xDD, 0x07, 0, 0, 0, 0, 0, 0 }); + try buf.append(gpa, 0x00); + try buf.appendSlice(gpa, doc.items); + std.mem.writeInt(u32, buf.items[0..4], @intCast(buf.items.len), .little); + return Message.parse(gpa, buf.items); +} + +test "a session id is read out of a command" { + const uuid = [_]u8{0xAB} ** 16; + var msg = try fake_op_msg(testing.allocator, &.{ + .{ .key = "ping", .value = .{ .int32 = 1 } }, + .{ .key = "lsid", .value = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + } } }, + }); + defer msg.deinit(); + try testing.expectEqual(uuid, (try msg.lsid()).?); +} + +test "a command with no lsid has no session" { + var msg = try fake_op_msg(testing.allocator, &.{.{ .key = "ping", .value = .{ .int32 = 1 } }}); + defer msg.deinit(); + try testing.expect((try msg.lsid()) == null); +} + +test "a session id carrying a user hash is still a session id" { + // `uid` arrives as soon as authentication is on (M7). Rejecting it as an + // unknown field would break every command the moment that lands, which is + // exactly the kind of divergence a measurement against a real server is + // for -- mongod 8.3.7 answers ok:1 to this. + const uuid = [_]u8{0x11} ** 16; + var msg = try fake_op_msg(testing.allocator, &.{ + .{ .key = "ping", .value = .{ .int32 = 1 } }, + .{ .key = "lsid", .value = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + .{ .key = "uid", .value = .{ .binary = .{ .subtype = 0, .data = &[_]u8{0} ** 32 } } }, + } } }, + }); + defer msg.deinit(); + try testing.expectEqual(uuid, (try msg.lsid()).?); +} + +test "every malformed session id is named" { + const uuid = [_]u8{0x22} ** 16; + const cases = [_]struct { want: anyerror, lsid: bson.Value }{ + .{ .want = error.LsidNotDocument, .lsid = .{ .int32 = 5 } }, + .{ .want = error.LsidIdMissing, .lsid = .{ .doc = &.{} } }, + .{ .want = error.LsidIdNotBinary, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .string = "nope" } }, + } } }, + .{ .want = error.LsidIdNotUuid, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 0, .data = &uuid } } }, + } } }, + .{ .want = error.LsidIdWrongLength, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = uuid[0..15] } } }, + } } }, + .{ .want = error.LsidUnknownField, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + .{ .key = "bogus", .value = .{ .int32 = 1 } }, + } } }, + .{ .want = error.LsidTxnNumberWithoutTxnUuid, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + .{ .key = "txnNumber", .value = .{ .int64 = 1 } }, + } } }, + .{ .want = error.LsidInternalSession, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + .{ .key = "txnUUID", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + } } }, + }; + for (cases) |c| { + var msg = try fake_op_msg(testing.allocator, &.{ + .{ .key = "ping", .value = .{ .int32 = 1 } }, + .{ .key = "lsid", .value = c.lsid }, + }); + defer msg.deinit(); + try testing.expectError(c.want, msg.lsid()); + } +} + test "reject non-OP_MSG non-OP_QUERY opcodes" { var buf: [20]u8 = undefined; std.mem.writeInt(u32, buf[0..4], 20, .little); -- 2.39.5 From 54092f44bea870e3ab677528a47982a1d9d2b343 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 15:06:35 +0300 Subject: [PATCH 23/37] commands: accept a well-formed lsid, refuse what would be a lie A driver puts `lsid` on every acknowledged command already, because `add_server_info` advertises `logicalSessionTimeoutMinutes`. So this is not new plumbing, it is a decision about input that has been arriving all along and being ignored. Accepting it and doing nothing is honest: a session here would own nothing -- no transactions to scope, no retryable writes, cursors that outlive their connection for their own reasons. There is deliberately no session registry; it would be a mutex on the dispatch path guarding state nothing reads, and M4's transaction state machine is what should decide its shape. `txnNumber` is a different matter, and ignoring it would be the lie this commit exists to remove. A transactional write would run non-transactionally, answer `ok`, and become durable; the client would find out at `commitTransaction`, by which time the data is on disk. It is refused, with `startTransaction` and `autocommit` alongside it for the same reason. Every code and every message was measured against mongod 8.3.7 through a raw OP_MSG probe -- the driver overwrites `lsid` with its own session, so a malformed one cannot be sent through it and none of this was checkable the usual way. Three things the measurement settled that guessing would have got wrong: an unknown command with a malformed lsid answers CommandNotFound, so the lookup comes first and this check belongs exactly where it sits; the codes for a bad session id are IDL parser codes (40414, 40415) rather than anything resembling the rest of our table; and a bad UUID length is InvalidUUID 207 while a bad subtype is TypeMismatch 14, which no amount of reasoning would have produced. Three divergences from mongod, all one cause: it keeps a per-command table of which commands accept `txnNumber` at all, and answers Location50889 or OperationNotSupportedInTransaction 263 for those that do not, before reaching the standalone refusal. We have no such table and give the standalone answer uniformly. For every CRUD command -- everything a driver would actually send these fields on -- the replies are identical; they differ only on things like `ping`, where mongod is more specific rather than differently right. Checked before any lock is taken, and the test for that is the second half of each refusal: keep using the engine afterwards, with a write that has to take the catalog exclusive to create a collection. Mutation-checked by moving the call below `lock_catalog` -- the test run hangs, which is precisely how the nameless-command lock leak presented, since a leaked *shared* lock is invisible to every reader. 174/174 unit tests. --- src/commands.zig | 340 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) diff --git a/src/commands.zig b/src/commands.zig index 3f7c05b..47f0822 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -52,6 +52,16 @@ pub const ErrorCode = enum(i32) { unauthorized = 13, type_mismatch = 14, operation_failed = 96, + // Session and transaction codes, measured against mongod 8.3.7 with a raw + // OP_MSG probe -- the driver rewrites `lsid` with its own session, so a + // malformed one cannot be sent through it and none of this could have been + // checked the usual way. The two five-digit ones are IDL parser codes: they + // are what mongod's generated command parsers answer, not hand-written + // checks, which is why they look unlike the rest of the table. + illegal_operation = 20, + invalid_uuid = 207, + idl_failed_to_parse = 40414, + idl_unknown_field = 40415, }; /// Which lock (if any) a command needs on the engine. Contract: only @@ -151,6 +161,12 @@ pub fn dispatch(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void { return reply.put_error(@intFromEnum(ErrorCode.command_not_found), "CommandNotFound", errmsg); }; + // Session and transaction fields are checked here: after the command is + // known -- mongod answers CommandNotFound to an unknown command carrying a + // malformed lsid, measured, not assumed -- and before any lock is taken, + // for the reason the comment below records at length. + if (try reject_bad_session_fields(msg, reply, name)) return; + // Lock the catalog (shared for most commands, exclusive for DDL), then // the target collection, then run the handler. The collection lock is // taken while the catalog lock is held, so a concurrent drop can never @@ -381,6 +397,168 @@ fn cmd_server_status(ctx: *Context, _: *wire.Message, reply: *wire.Reply) !void try reply.put_ok(); } +// --------------------------------------------------------------------------- +// Sessions +// +// This server keeps no session registry, and that is a decision rather than an +// omission: a session here would own nothing. There are no transactions to +// scope, no cursors that outlive their connection differently because of one, +// and no retryable writes -- a driver disables those for a standalone. A +// registry would be a mutex on the dispatch path guarding state nothing reads. +// M4 gets to design it, when the transaction state machine says what it needs. +// +// What is *not* optional is telling the truth about the fields a driver sends +// anyway. `lsid` rides on every acknowledged command already, because +// `add_server_info` advertises `logicalSessionTimeoutMinutes`. Accepting it and +// doing nothing is honest -- there is nothing to do. Accepting `txnNumber` and +// doing nothing is not: it would run a transactional write non-transactionally +// and answer ok, and the client would only find out at `commitTransaction`, +// long after the data was on disk. +// --------------------------------------------------------------------------- + +/// mongod's name for a binary subtype, for the one message that quotes it. +fn subtype_name(subtype: u8) []const u8 { + return switch (subtype) { + 0x00 => "general", + 0x01 => "function", + 0x02 => "binary", + 0x03 => "uuid_old", + 0x04 => "UUID", + 0x05 => "MD5", + 0x06 => "encrypt", + else => "unknown", + }; +} + +/// The first `lsid` field this server does not know, for the message that has +/// to name it. Walked again on the error path only, so that `Message.lsid` +/// stays a yes-or-no answer. +fn unknown_lsid_field(msg: *wire.Message) []const u8 { + const doc = switch (msg.body.get("lsid") orelse return "?") { + .doc => |d| d, + else => return "?", + }; + for (doc) |pair| { + const known = std.mem.eql(u8, pair.key, "id") or std.mem.eql(u8, pair.key, "uid") or + std.mem.eql(u8, pair.key, "txnNumber") or std.mem.eql(u8, pair.key, "txnUUID"); + if (!known) return pair.key; + } + return "?"; +} + +/// Writes the reply for a malformed `lsid` and answers whether it did. +fn reject_bad_lsid(msg: *wire.Message, reply: *wire.Reply, cmd: []const u8) !bool { + _ = msg.lsid() catch |err| { + const arena = reply.arena_alloc(); + const text = switch (err) { + error.LsidNotDocument => try std.fmt.allocPrint( + arena, + "BSON field '{s}.lsid' is the wrong type '{s}', expected type 'object'", + .{ cmd, msg.body.get("lsid").?.type_name() }, + ), + error.LsidIdMissing => try std.fmt.allocPrint( + arena, + "BSON field '{s}.lsid.id' is missing but a required field", + .{cmd}, + ), + error.LsidIdNotBinary => try std.fmt.allocPrint( + arena, + "BSON field '{s}.lsid.id' is the wrong type '{s}', expected type 'binData'", + .{ cmd, bson.get_pair(msg.body.get("lsid").?.doc, "id").?.type_name() }, + ), + error.LsidIdNotUuid => try std.fmt.allocPrint( + arena, + "BSON field '{s}.lsid.id' is the wrong binData type '{s}', expected type 'UUID'", + .{ cmd, subtype_name(bson.get_pair(msg.body.get("lsid").?.doc, "id").?.binary.subtype) }, + ), + error.LsidUnknownField => try std.fmt.allocPrint( + arena, + "BSON field '{s}.lsid.{s}' is an unknown field.", + .{ cmd, unknown_lsid_field(msg) }, + ), + else => "", + }; + switch (err) { + error.LsidNotDocument, error.LsidIdNotBinary, error.LsidIdNotUuid => try reply.put_error( + @intFromEnum(ErrorCode.type_mismatch), + "TypeMismatch", + text, + ), + error.LsidIdMissing => try reply.put_error( + @intFromEnum(ErrorCode.idl_failed_to_parse), + "IDLFailedToParse", + text, + ), + error.LsidUnknownField => try reply.put_error( + @intFromEnum(ErrorCode.idl_unknown_field), + "IDLUnknownField", + text, + ), + error.LsidIdWrongLength => try reply.put_error( + @intFromEnum(ErrorCode.invalid_uuid), + "InvalidUUID", + "uuid must be a 16-byte binary field with UUID (4) subtype", + ), + error.LsidTxnNumberWithoutTxnUuid => try invalid_arg( + reply, + "Cannot specify txnNumber in lsid without specifying txnUUID", + ), + error.LsidInternalSession => try invalid_arg( + reply, + "Internal sessions are not supported outside of transactions", + ), + } + return true; + }; + return false; +} + +/// Writes the reply for a transaction field this server cannot honour, and +/// answers whether it did. +/// +/// Refusing rather than ignoring is the whole point. A driver that is told +/// `ok` for a write carrying a `txnNumber` has been told the write is part of +/// a transaction; it is not, it is already durable, and the first the client +/// hears of it is a failing `commitTransaction`. The order of the checks and +/// every message below are mongod 8.3.7's, measured. +fn reject_txn_fields(msg: *wire.Message, reply: *wire.Reply, cmd: []const u8) !bool { + const txn_number = msg.body.get("txnNumber"); + if (msg.body.get("startTransaction") != null and msg.body.get("autocommit") == null) { + try invalid_arg(reply, "'startTransaction' field requires 'autocommit' field to also be specified"); + return true; + } + if (msg.body.get("autocommit") != null and txn_number == null) { + try invalid_arg(reply, "'autocommit' field requires a transaction number to also be specified"); + return true; + } + const n = txn_number orelse return false; + if (n != .int64 and n != .int32) { + const text = try std.fmt.allocPrint( + reply.arena_alloc(), + "BSON field '{s}.txnNumber' is the wrong type '{s}', expected type 'long'", + .{ cmd, n.type_name() }, + ); + try reply.put_error(@intFromEnum(ErrorCode.type_mismatch), "TypeMismatch", text); + return true; + } + if (msg.body.get("lsid") == null) { + try invalid_arg(reply, "Transaction number requires a session ID to also be specified"); + return true; + } + try reply.put_error( + @intFromEnum(ErrorCode.illegal_operation), + "IllegalOperation", + "Transaction numbers are only allowed on a replica set member or mongos", + ); + return true; +} + +/// Answers whether an error reply was written, in which case dispatch is done. +fn reject_bad_session_fields(msg: *wire.Message, reply: *wire.Reply, cmd: []const u8) !bool { + if (try reject_bad_lsid(msg, reply, cmd)) return true; + return reject_txn_fields(msg, reply, cmd); +} + fn cmd_end_sessions(_: *Context, _: *wire.Message, reply: *wire.Reply) !void { try reply.put_ok(); } @@ -2869,6 +3047,168 @@ fn parse_fake_msg(name: []const u8, value: bson.Value, extra: []const bson.Pair) return wire.Message.parse(testing.allocator, msg.items); } +/// Runs one command and hands back its reply's `code`, or null on ok:1. +fn run_for_code(ctx: *Context, name: []const u8, value: bson.Value, extra: []const bson.Pair) !?i32 { + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg(name, value, extra); + defer msg.deinit(); + try dispatch(ctx, &msg, &reply); + if (bson.get_pair(reply.pairs.items, "code")) |c| return c.int32; + try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double); + return null; +} + +fn doc_count(ctx: *Context, coll: []const u8) !i64 { + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg("count", .{ .string = coll }, &.{}); + defer msg.deinit(); + try dispatch(ctx, &msg, &reply); + return bson.get_pair(reply.pairs.items, "n").?.int64; +} + +test "a well-formed session id changes nothing and is not echoed" { + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + const uuid = [_]u8{0x5A} ** 16; + const lsid = bson.Value{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + } }; + + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg("insert", .{ .string = "sess" }, &.{ + .{ .key = "documents", .value = .{ .array = &.{.{ .doc = &.{.{ .key = "a", .value = .{ .int32 = 1 } }} }} } }, + .{ .key = "lsid", .value = lsid }, + }); + defer msg.deinit(); + try dispatch(&ctx, &msg, &reply); + try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double); + // mongod answers a well-formed lsid with exactly `{ok: 1}` and no echo, + // measured; a driver reads only `$clusterTime` and `operationTime` back. + try testing.expect(bson.get_pair(reply.pairs.items, "lsid") == null); + try testing.expectEqual(@as(i64, 1), try doc_count(&ctx, "sess")); +} + +test "a malformed session id is refused without leaking a lock" { + // The second half is the point. `reject_bad_session_fields` runs before any + // lock is taken, and the way to show it is to keep using the engine after + // each refusal: a leaked *shared* catalog lock is invisible to readers and + // only stops the next write that needs it exclusively. That is exactly how + // the nameless-command bug hid for a milestone. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + const uuid = [_]u8{0x7C} ** 16; + const good = bson.Value{ .binary = .{ .subtype = 4, .data = &uuid } }; + // Every code here was measured against mongod 8.3.7, not recalled. + const cases = [_]struct { code: i32, lsid: bson.Value }{ + .{ .code = 14, .lsid = .{ .int32 = 5 } }, + .{ .code = 40414, .lsid = .{ .doc = &.{} } }, + .{ .code = 14, .lsid = .{ .doc = &.{.{ .key = "id", .value = .{ .string = "nope" } }} } }, + .{ .code = 14, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 0, .data = &uuid } } }, + } } }, + .{ .code = 207, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = uuid[0..15] } } }, + } } }, + .{ .code = 40415, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = good }, + .{ .key = "bogus", .value = .{ .int32 = 1 } }, + } } }, + .{ .code = 72, .lsid = .{ .doc = &.{ + .{ .key = "id", .value = good }, + .{ .key = "txnUUID", .value = good }, + } } }, + }; + + for (cases, 0..) |c, i| { + const insert_doc = bson.Value{ .array = &.{.{ .doc = &.{.{ .key = "a", .value = .{ .int32 = 1 } }} }} }; + const code = try run_for_code(&ctx, "insert", .{ .string = "leaky" }, &.{ + .{ .key = "documents", .value = insert_doc }, + .{ .key = "lsid", .value = c.lsid }, + }); + try testing.expectEqual(c.code, code.?); + // A write that needs the catalog exclusive to create a collection: the + // one operation a leaked shared lock would block. + var name_buf: [16]u8 = undefined; + const fresh = try std.fmt.bufPrint(&name_buf, "after{d}", .{i}); + try testing.expectEqual(@as(?i32, null), try run_for_code(&ctx, "insert", .{ .string = fresh }, &.{ + .{ .key = "documents", .value = insert_doc }, + })); + } +} + +test "a transactional write is refused rather than applied" { + // The count is the assertion. Ignoring `txnNumber` would answer ok, write + // the document, and leave the client to discover at commitTransaction that + // its transaction was never one -- long after the data was durable. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + const uuid = [_]u8{0x3E} ** 16; + const lsid = bson.Value{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = &uuid } } }, + } }; + const docs = bson.Value{ .array = &.{.{ .doc = &.{.{ .key = "a", .value = .{ .int32 = 1 } }} }} }; + + try testing.expectEqual(@as(i32, 20), (try run_for_code(&ctx, "insert", .{ .string = "txn" }, &.{ + .{ .key = "documents", .value = docs }, + .{ .key = "lsid", .value = lsid }, + .{ .key = "txnNumber", .value = .{ .int64 = 1 } }, + .{ .key = "startTransaction", .value = .{ .bool = true } }, + .{ .key = "autocommit", .value = .{ .bool = false } }, + })).?); + try testing.expectEqual(@as(i64, 0), try doc_count(&ctx, "txn")); + + // The order the checks fire in is mongod's, measured: a missing session id + // is reported before the standalone refusal, and a bad type before both. + try testing.expectEqual(@as(i32, 72), (try run_for_code(&ctx, "insert", .{ .string = "txn" }, &.{ + .{ .key = "documents", .value = docs }, + .{ .key = "txnNumber", .value = .{ .int64 = 1 } }, + })).?); + try testing.expectEqual(@as(i32, 14), (try run_for_code(&ctx, "insert", .{ .string = "txn" }, &.{ + .{ .key = "documents", .value = docs }, + .{ .key = "lsid", .value = lsid }, + .{ .key = "txnNumber", .value = .{ .string = "nope" } }, + })).?); + try testing.expectEqual(@as(i32, 72), (try run_for_code(&ctx, "insert", .{ .string = "txn" }, &.{ + .{ .key = "documents", .value = docs }, + .{ .key = "lsid", .value = lsid }, + .{ .key = "startTransaction", .value = .{ .bool = true } }, + })).?); + try testing.expectEqual(@as(i64, 0), try doc_count(&ctx, "txn")); +} + +test "an unknown command is reported before its session id is judged" { + // Measured: mongod answers CommandNotFound to `{nosuchcmd: 1, lsid: 5}`, + // so the lookup comes first and this is where the check belongs. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + try testing.expectEqual(@as(i32, 59), (try run_for_code(&ctx, "nosuchcmd", .{ .int32 = 1 }, &.{ + .{ .key = "lsid", .value = .{ .int32 = 5 } }, + })).?); +} + test "concurrent insert/find commands on a threaded Io" { // Exercises dispatch's lock classification end-to-end: writer fibers run // `insert` under the exclusive lock, reader fibers run `count`/`find` -- 2.39.5 From f5c827e581c4c431ebe4e9555caeb78f8ea171b5 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 15:08:54 +0300 Subject: [PATCH 24/37] commands: endSessions judges the array it is handed It stays a no-op -- there is nothing to end -- but stops being a blind `ok`. This is the one session command that arrives in normal operation: a driver sends it on close for every session it handed out. Validating an argument that is then discarded looks like ceremony, and is not: `ok: 1` to a malformed `endSessions` says the server understood something it never read, which is the same class of answer the previous commit removed for `txnNumber`. Codes and messages measured against mongod 8.3.7, including the field path `endSessions.endSessionsFromClient` -- an IDL artefact, since the command's own field is `endSessions` and the parsed argument carries a different name. It is reproduced rather than tidied: a path that differs from the real server's is worse than an odd one. With this the raw probe's replies are identical to mongod's for every lsid and endSessions shape. What is left is five deliberate divergences, all recorded: three because mongod knows per command whether `txnNumber` is even accepted and answers Location50889 or 263 before reaching the standalone refusal, and two because `startSession` and `refreshSessions` are not implemented -- a driver calls neither, generating session ids locally, so CommandNotFound is the honest answer. `commitTransaction` joins them, and it can only be reached by a client whose write this server already refused. 175/175 unit tests. --- src/commands.zig | 141 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/src/commands.zig b/src/commands.zig index 47f0822..2bdd392 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -559,10 +559,108 @@ fn reject_bad_session_fields(msg: *wire.Message, reply: *wire.Reply, cmd: []cons return reject_txn_fields(msg, reply, cmd); } -fn cmd_end_sessions(_: *Context, _: *wire.Message, reply: *wire.Reply) !void { +/// Still a no-op -- there is nothing to end -- but no longer a blind `ok`. +/// +/// A driver sends this on close for every session it handed out, so it is the +/// one session command that arrives in normal operation. Validating an array +/// we then discard looks like ceremony; it is not. `ok: 1` to a malformed +/// `endSessions` is the same class of answer as `ok: 1` to a transactional +/// write: the client is told the server understood, and it did not. mongod's +/// field path is `endSessions.endSessionsFromClient`, an IDL artefact -- the +/// command's own field is `endSessions` and the parsed argument has a +/// different name -- and it is reproduced rather than tidied, because a name +/// that differs from the real server's is worse than an odd one. +fn cmd_end_sessions(_: *Context, msg: *wire.Message, reply: *wire.Reply) !void { + const arena = reply.arena_alloc(); + const sessions = switch (msg.body.get("endSessions").?) { + .array => |a| a, + else => |v| { + const text = try std.fmt.allocPrint( + arena, + "BSON field 'endSessions.endSessions' is the wrong type '{s}', expected type 'array'", + .{v.type_name()}, + ); + return reply.put_error(@intFromEnum(ErrorCode.type_mismatch), "TypeMismatch", text); + }, + }; + + for (sessions, 0..) |entry, i| { + if (try reject_bad_session_entry(entry, i, reply)) return; + } try reply.put_ok(); } +/// One element of `endSessions`. Answers whether an error reply was written. +fn reject_bad_session_entry(entry: bson.Value, i: usize, reply: *wire.Reply) !bool { + const arena = reply.arena_alloc(); + const prefix = "BSON field 'endSessions.endSessionsFromClient"; + const doc = switch (entry) { + .doc => |d| d, + else => { + const text = try std.fmt.allocPrint( + arena, + "{s}.{d}' is the wrong type '{s}', expected type 'object'", + .{ prefix, i, entry.type_name() }, + ); + try reply.put_error(@intFromEnum(ErrorCode.type_mismatch), "TypeMismatch", text); + return true; + }, + }; + + var seen_id = false; + for (doc) |pair| { + if (std.mem.eql(u8, pair.key, "uid")) continue; + if (!std.mem.eql(u8, pair.key, "id")) { + const text = try std.fmt.allocPrint( + arena, + "{s}.{s}' is an unknown field.", + .{ prefix, pair.key }, + ); + try reply.put_error(@intFromEnum(ErrorCode.idl_unknown_field), "IDLUnknownField", text); + return true; + } + seen_id = true; + const bin = switch (pair.value) { + .binary => |b| b, + else => { + const text = try std.fmt.allocPrint( + arena, + "{s}.id' is the wrong type '{s}', expected type 'binData'", + .{ prefix, pair.value.type_name() }, + ); + try reply.put_error(@intFromEnum(ErrorCode.type_mismatch), "TypeMismatch", text); + return true; + }, + }; + if (bin.subtype != 4) { + const text = try std.fmt.allocPrint( + arena, + "{s}.id' is the wrong binData type '{s}', expected type 'UUID'", + .{ prefix, subtype_name(bin.subtype) }, + ); + try reply.put_error(@intFromEnum(ErrorCode.type_mismatch), "TypeMismatch", text); + return true; + } + if (bin.data.len != 16) { + try reply.put_error( + @intFromEnum(ErrorCode.invalid_uuid), + "InvalidUUID", + "uuid must be a 16-byte binary field with UUID (4) subtype", + ); + return true; + } + } + if (!seen_id) { + try reply.put_error( + @intFromEnum(ErrorCode.idl_failed_to_parse), + "IDLFailedToParse", + "BSON field 'endSessions.endSessionsFromClient.id' is missing but a required field", + ); + return true; + } + return false; +} + fn cmd_connection_status(_: *Context, _: *wire.Message, reply: *wire.Reply) !void { const auth_info = try reply.arena_alloc().alloc(bson.Pair, 2); auth_info[0] = .{ .key = "authenticatedUsers", .value = .{ .array = &.{} } }; @@ -3194,6 +3292,47 @@ test "a transactional write is refused rather than applied" { try testing.expectEqual(@as(i64, 0), try doc_count(&ctx, "txn")); } +test "endSessions judges the array it is handed" { + // Still a no-op, and that is not what is being tested. The command arrives + // in normal operation -- a driver sends it on close for every session it + // handed out -- so a blind `ok` here tells a client the server understood + // something it never looked at. Codes measured against mongod 8.3.7. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + const uuid = [_]u8{0x6B} ** 16; + const good = bson.Value{ .binary = .{ .subtype = 4, .data = &uuid } }; + const one = bson.Value{ .doc = &.{.{ .key = "id", .value = good }} }; + + try testing.expectEqual(@as(?i32, null), try run_for_code(&ctx, "endSessions", .{ .array = &.{} }, &.{})); + try testing.expectEqual(@as(?i32, null), try run_for_code(&ctx, "endSessions", .{ .array = &.{one} }, &.{})); + // `uid` rides along once authentication is on, exactly as it does in lsid. + try testing.expectEqual(@as(?i32, null), try run_for_code(&ctx, "endSessions", .{ .array = &.{.{ .doc = &.{ + .{ .key = "id", .value = good }, + .{ .key = "uid", .value = .{ .binary = .{ .subtype = 0, .data = &[_]u8{0} ** 32 } } }, + } }} }, &.{})); + + try testing.expectEqual(@as(i32, 14), (try run_for_code(&ctx, "endSessions", .{ .string = "nope" }, &.{})).?); + try testing.expectEqual(@as(i32, 14), (try run_for_code(&ctx, "endSessions", .{ .array = &.{.{ .int32 = 5 }} }, &.{})).?); + try testing.expectEqual(@as(i32, 14), (try run_for_code(&ctx, "endSessions", .{ .array = &.{.{ .doc = &.{ + .{ .key = "id", .value = .{ .string = "nope" } }, + } }} }, &.{})).?); + try testing.expectEqual(@as(i32, 207), (try run_for_code(&ctx, "endSessions", .{ .array = &.{.{ .doc = &.{ + .{ .key = "id", .value = .{ .binary = .{ .subtype = 4, .data = uuid[0..15] } } }, + } }} }, &.{})).?); + try testing.expectEqual(@as(i32, 40414), (try run_for_code(&ctx, "endSessions", .{ .array = &.{.{ .doc = &.{} }} }, &.{})).?); + try testing.expectEqual(@as(i32, 40415), (try run_for_code(&ctx, "endSessions", .{ .array = &.{.{ .doc = &.{ + .{ .key = "id", .value = good }, + .{ .key = "bogus", .value = .{ .int32 = 1 } }, + } }} }, &.{})).?); + // A bad entry after a good one is still a bad entry. + try testing.expectEqual(@as(i32, 14), (try run_for_code(&ctx, "endSessions", .{ .array = &.{ one, .{ .int32 = 5 } } }, &.{})).?); +} + test "an unknown command is reported before its session id is judged" { // Measured: mongod answers CommandNotFound to `{nosuchcmd: 1, lsid: 5}`, // so the lookup comes first and this is where the check belongs. -- 2.39.5 From 3e2d0ab1340ba26f7076fac58601e76e23aa829b Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 15:11:28 +0300 Subject: [PATCH 25/37] commands: a count reply is an int32 Two of the tests added alongside endSessions read the count reply's `n` as int64. ReleaseFast does not check the union tag, so both passed there and aborted in ReleaseSafe -- which is the whole argument for running the suite in both, and the reason the plan makes ReleaseSafe non-optional. 175/175 in ReleaseFast and ReleaseSafe. --- src/commands.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands.zig b/src/commands.zig index 2bdd392..75ee4d5 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -3157,13 +3157,13 @@ fn run_for_code(ctx: *Context, name: []const u8, value: bson.Value, extra: []con return null; } -fn doc_count(ctx: *Context, coll: []const u8) !i64 { +fn doc_count(ctx: *Context, coll: []const u8) !i32 { var reply = wire.Reply.init(testing.allocator); defer reply.deinit(); var msg = try parse_fake_msg("count", .{ .string = coll }, &.{}); defer msg.deinit(); try dispatch(ctx, &msg, &reply); - return bson.get_pair(reply.pairs.items, "n").?.int64; + return bson.get_pair(reply.pairs.items, "n").?.int32; } test "a well-formed session id changes nothing and is not echoed" { @@ -3191,7 +3191,7 @@ test "a well-formed session id changes nothing and is not echoed" { // mongod answers a well-formed lsid with exactly `{ok: 1}` and no echo, // measured; a driver reads only `$clusterTime` and `operationTime` back. try testing.expect(bson.get_pair(reply.pairs.items, "lsid") == null); - try testing.expectEqual(@as(i64, 1), try doc_count(&ctx, "sess")); + try testing.expectEqual(@as(i32, 1), try doc_count(&ctx, "sess")); } test "a malformed session id is refused without leaking a lock" { @@ -3271,7 +3271,7 @@ test "a transactional write is refused rather than applied" { .{ .key = "startTransaction", .value = .{ .bool = true } }, .{ .key = "autocommit", .value = .{ .bool = false } }, })).?); - try testing.expectEqual(@as(i64, 0), try doc_count(&ctx, "txn")); + try testing.expectEqual(@as(i32, 0), try doc_count(&ctx, "txn")); // The order the checks fire in is mongod's, measured: a missing session id // is reported before the standalone refusal, and a bad type before both. @@ -3289,7 +3289,7 @@ test "a transactional write is refused rather than applied" { .{ .key = "lsid", .value = lsid }, .{ .key = "startTransaction", .value = .{ .bool = true } }, })).?); - try testing.expectEqual(@as(i64, 0), try doc_count(&ctx, "txn")); + try testing.expectEqual(@as(i32, 0), try doc_count(&ctx, "txn")); } test "endSessions judges the array it is handed" { -- 2.39.5 From e416ad179aaebafe1e36d67154887bd4f075acc1 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 15:14:07 +0300 Subject: [PATCH 26/37] plan: sessions, and the three measurements that corrected it Records what `lsid` support is and what it deliberately is not, so M4 inherits the decisions rather than the questions. The part worth keeping is not the design but its corrections: three of the assumptions this stage was planned on turned out to be wrong when measured against a real mongod, and one of them -- that unknown fields inside `lsid` are tolerated -- would have shipped a divergence nothing in the test corpus could have caught. Also states that the scorecard did not move, 194/97/196 either side, which was the prediction rather than a surprise: the corpus has no session entities at all. What is new is that the prediction is now checkable -- after the event assertions landed, a command wrongly refused here would show up as a changed event stream instead of silently. --- PLAN.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/PLAN.md b/PLAN.md index 92d54ab..7186101 100644 --- a/PLAN.md +++ b/PLAN.md @@ -823,16 +823,62 @@ has to be its own commit with its own re-recorded scorecard. the anchor rewritten — resuming at it returned updated documents twice, caught by draining a collection being updated underneath. - Still open in M1: the doc-level free list and sessions plumbing (`lsid` - accepted). The eight reclamation bugs above were cleared first, as - preconditions for the free list rather than as work of their own; - command-monitoring (`expectEvents`) landed next, so that the free list and - sessions are measured by an instrument that is no longer known to overstate. + Still open in M1: the doc-level free list. The eight reclamation bugs above + were cleared first, as preconditions for the free list rather than as work of + their own; command-monitoring (`expectEvents`) landed next, so that what + followed is measured by an instrument no longer known to overstate. **A prerequisite the free list must honour**, recorded here while it is still being designed: *an offset that was ever a record start must remain a record start.* `doc_bytes` reads a `u32` length prefix in place, so an offset landing mid-record after a re-split is a garbage-length read rather than a wrong answer — and an offsets cursor holds exactly such offsets. +- **M1 sessions** — *settled and implemented.* `lsid` is parsed, validated and + deliberately acted on in no way; `txnNumber`, `startTransaction` and + `autocommit` are refused; `endSessions` validates the array it discards. + Every code and message was **measured against mongod 8.3.7** with a raw + OP_MSG probe, because the driver overwrites `lsid` with its own session and a + malformed one cannot be sent through it. Three measurements contradicted the + design they were checking: + - **Unknown fields inside `lsid` are rejected** (IDLUnknownField 40415). The + design said to tolerate them, reasoning that the server tolerates unknown + fields everywhere. It does not, here. + - **`uid` is accepted** — the hash of the credentials owning the session, + which a driver sends as soon as authentication is on. Rejecting it would + have broken every command in M7. + - **An unknown command with a malformed `lsid` answers CommandNotFound**, so + command lookup precedes session validation, which is where the check sits. + + The refusals, so M4 does not reopen them: + - **No session registry.** A session here would own nothing: no transactions + to scope, no retryable writes (a driver disables them for a standalone), + and cursors that outlive their connection for reasons of their own. It + would be a mutex on the dispatch path guarding state nothing reads. M4's + transaction state machine gets to say what shape it needs. + - **`lsid` is not echoed.** Measured: mongod answers a well-formed one with + exactly `{ok: 1}`. A driver reads only `$clusterTime` and `operationTime` + back, and a standalone sends neither — correctly, since without + `operationTime` there is nothing for `afterClusterTime` to attach to and + causal consistency stays off. + - **`startSession` and `refreshSessions` are not implemented.** Both are real + mongod commands, but a driver calls neither — it generates session ids + locally — so CommandNotFound is the honest answer. Candidates for M4. + + Five divergences from mongod remain, all deliberate. Three share one cause: + mongod keeps a per-command table of which commands accept `txnNumber` at all + and answers Location50889 or OperationNotSupportedInTransaction 263 for those + that do not, *before* reaching the standalone refusal. We have no such table + and give the standalone answer uniformly, so replies are identical for every + CRUD command — everything a driver would send these fields on — and differ + only on things like `ping`, where mongod is more specific rather than + differently right. The other two are `startSession`/`refreshSessions` above, + with `commitTransaction` alongside them, reachable only by a client whose + write this server has already refused. + + **Effect on the scorecard: exactly zero, and that was the prediction.** + 194/97/196 before and after. The corpus has no `session` entity, no operation + taking a `session` argument, and no `lsid` assertion. The value here is + protocol hygiene, not a number — and after Stage 1 a wrongly-refused command + would have shown up as a changed event stream rather than silently. - **M2 aggregation**: stage/expression tiers, which spec-test files are the gate, whether $lookup/$unwind/facet make the first cut. - **M4 transactions**: snapshot isolation over mmap (COW vs undo), read -- 2.39.5 From 23b283ff449873c53bad22140ceecdd7f565d853 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 16:33:00 +0300 Subject: [PATCH 27/37] db: the slab knows where its dead bytes are Inert on its own: nothing is reclaimed yet and no behaviour changes. What changes is that a collection can now answer *where* its garbage is, which is the precondition for handing any of it back. A slab extent becomes a `SlabRun`: the same two u32s plus a dense array of dead-byte counters, one per `map_align` window. The window is the unit because it is the smallest thing that can be given back at all -- `mark_appendable` refuses an unaligned start and `protect_stable` rounds outwards -- so a counter never exceeds `map_align` and its width follows from that. Two bytes per window is the entire memory cost: 2.7 MB for a 21 GB slab on 16 KiB pages. The shapes that track dead *documents* instead (an interval set, a free-run list) cost gigabytes at the 200-byte document scale of D7.3, and would make `evict_doc` allocate after the write is already committed, which is a failure with nowhere to go. `mark_dead` is therefore infallible, and is called from the two places slab dies: `evict_doc`, after every index entry naming the bytes is gone, and `note_skip`, for what the appender writes off at a checkpoint or when it abandons the tail of an extent. Ordering `mark_dead` last in `evict_doc` is what will make reclamation by counting alone sound -- a window reaches `map_align` dead only once every document touching it has been through there. The run list is now sorted by page number rather than allocation order. That was free while an extent could only be appended to; a recycled run arrives *below* one the collection already owns, and `run_of` is a binary search. Sortedness and non-overlap are asserted at the single point runs enter. Two counters accompany it. `dead_unlocated` holds garbage that has no window: the head and tail of a run outside its whole windows, and -- the larger share -- everything that died before the last restart. It exists so one identity stays exact: sum of window counters + dead_unlocated == slab_used - live_bytes Left side is where, right side is how much; reclamation reads the first and the compaction trigger reads the second, and a drift between them is either a rebuild firing on a clean database or a window handed back with a live document in it. `reclaimed_bytes` is inert here and exists for the churn gate, which cannot otherwise tell "the ratio improved because reclamation worked" from "the ratio improved for another reason". The catalog is byte-identical: still `u32 count, (u32 first, u32 pages)*`, so `catalog_version` stays 1 and there is no second read path. The window map is deliberately not persisted -- an open puts the whole amount into `dead_unlocated` instead. The consequence runs one way: a forgotten dead byte is a window that is not handed back, never a live window that is. Reading inserts sorted rather than appending, so a catalog written before this commit loads into an ordered list. Five tests. The accounting identity across both kinds of death; run edges counted but not placed, driven against `mark_dead` directly since the alignment of a real extent is the allocator's business; documents never straddling a run, over a collection with an oversized document in a run of its own; a run recycled to a lower address keeping the list ordered and findable; and a restart forgetting where the garbage is but not how much. Mutations, each red on its own: drop `mark_dead` from `note_skip` (30824 vs 0), from `evict_doc` (151304 vs 30824), the run-tail branch (16384 vs 12288), the run-head branch (26 tests crash on the underflowed window index), the sorted insert (overlap assert fires), and the `dead_unlocated` line in `read_catalog` (40240 vs 0). 180/180 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz, e2e 49, e2e2 concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/db.zig | 564 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 539 insertions(+), 25 deletions(-) diff --git a/src/db.zig b/src/db.zig index e4808c9..c595778 100644 --- a/src/db.zig +++ b/src/db.zig @@ -36,6 +36,67 @@ const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size; const LogKind = enum { upsert, delete, index_create, index_drop }; +/// Dead bytes in one `map_align` window. The window is the unit of reclamation +/// -- a whole system page is the smallest thing `mark_appendable` and +/// `protect_stable` can hand back -- so a counter never exceeds `map_align`, +/// and the width follows from that rather than being chosen. +/// +/// Two bytes per window on the usual platforms. That is the entire memory cost +/// of knowing where a collection's garbage is: 2.7 MB for a 21 GB slab on +/// 16 KiB pages, 10.6 MB on 4 KiB ones. The alternative shapes -- an interval +/// set, a free-run list -- cost memory proportional to the number of *dead +/// documents*, which for 200-byte documents at that scale is gigabytes, and +/// would make `evict_doc` allocate after the write is already committed. +const WindowDead = if (pgr.map_align <= std.math.maxInt(u16)) u16 else u32; + +/// A run of pages a collection's slab owns, plus where its garbage is. +/// +/// This replaced a bare `pgr.Extent` because an extent can only be given back +/// whole, and a churning collection almost never empties one. A run is split +/// instead: the windows inside it with nothing live left go to the pager and +/// the run becomes two shorter ones. So the list is kept sorted by page number, +/// which makes `run_of` a binary search and the sortedness itself an assert -- +/// allocation order stopped being meaningful once a run could be recycled to a +/// *lower* address than one already owned. +const SlabRun = struct { + first: u32, + pages: u32, + /// The run's start, rounded up to `map_align`: the first offset that begins + /// a whole window. `alloc_pages` works in 4 KiB pages, so a run need not + /// start on a system page. + window_first: u64, + /// Dead bytes per window, `dead[i]` covering + /// `[window_first + i*map_align, +map_align)`. `map_align` means the window + /// holds nothing live and can be handed back. + dead: []WindowDead, + + /// Windows wholly inside the pages `[first, first+pages)`. The bytes + /// outside them -- below `window_first`, and the tail after the last whole + /// window -- are real slab that documents do live in; their garbage is + /// counted in `Collection.dead_unlocated` instead, because it can never be + /// reclaimed on its own. + fn window_count(first: u32, pages: u32) usize { + const from = @as(u64, first) << pgr.page_shift; + const to = from + (@as(u64, pages) << pgr.page_shift); + const wf = std.mem.alignForward(u64, from, pgr.map_align); + const we = std.mem.alignBackward(u64, to, pgr.map_align); + return if (we > wf) @intCast((we - wf) / pgr.map_align) else 0; + } + + fn start(self: SlabRun) u64 { + return @as(u64, self.first) << pgr.page_shift; + } + + fn end(self: SlabRun) u64 { + return (@as(u64, self.first) + self.pages) << pgr.page_shift; + } + + /// One past the last byte covered by a window counter. + fn window_end(self: SlabRun) u64 { + return self.window_first + self.dead.len * pgr.map_align; + } +}; + pub const Collection = struct { /// Documents live as canonical BSON bytes in the data file, in extents this /// collection owns; the map holds each document's offset. Those are @@ -53,8 +114,9 @@ pub const Collection = struct { doc_count: u64, /// The data file this collection's documents live in. pager: *pgr.Pager, - /// Extents owned by this collection's slab, in allocation order. - slab_extents: std.ArrayListUnmanaged(pgr.Extent), + /// Page runs owned by this collection's slab, sorted by page number, each + /// carrying the map of where its dead bytes are. See `SlabRun`. + slab_runs: std.ArrayListUnmanaged(SlabRun), /// Absolute file offset of the next document write, and the end of the /// extent it falls in. slab_tail: u64, @@ -72,6 +134,25 @@ pub const Collection = struct { /// collection so dropping one can move the right amount from the engine's /// live total to its dead total. live_bytes: u64, + /// Garbage this collection knows it has but cannot place in a window: the + /// edges of a run that fall outside any whole window, and -- the larger + /// share -- everything that died before the last restart, since the window + /// map is not persisted. + /// + /// It exists to keep one identity exact: + /// + /// sum of every window counter + dead_unlocated == slab_used - live_bytes + /// + /// Without it the two halves of the accounting would drift apart at every + /// open, and there would be no assert that could tell drift from a lost + /// update. What it costs is only that garbage from before a restart is not + /// reclaimed window-wise; it still arms compaction like any other. + dead_unlocated: u64, + /// Slab handed back to the pager by window reclamation, cumulative for the + /// life of the process. Purely an observation: it is what distinguishes + /// "the ratio improved because reclamation worked" from "the ratio improved + /// for some other reason", which is the only way to read the churn gate. + reclaimed_bytes: u64, /// Secondary indexes (persisted through the log). Heap-allocated, so an /// `*Index` handed out by `find_index` or `create_index` stays valid when /// a sibling index is dropped. Held by value, `orderedRemove` memmoved the @@ -116,11 +197,13 @@ pub const Collection = struct { var self: Collection = .{ .doc_count = 0, .pager = pager, - .slab_extents = .empty, + .slab_runs = .empty, .slab_tail = 0, .slab_end = 0, .slab_used = 0, .live_bytes = 0, + .dead_unlocated = 0, + .reclaimed_bytes = 0, .hold = .{}, .indexes = .empty, .id_index = undefined, @@ -146,6 +229,117 @@ pub const Collection = struct { return null; } + /// Take ownership of a page run, keeping `slab_runs` sorted by page number. + /// The only place a run enters the list, so the sort order and the window + /// map are established together and cannot disagree. + fn insert_run(self: *Collection, gpa: std.mem.Allocator, first: u32, pages: u32) !void { + const dead = try gpa.alloc(WindowDead, SlabRun.window_count(first, pages)); + errdefer gpa.free(dead); + @memset(dead, 0); + var at: usize = 0; + while (at < self.slab_runs.items.len and self.slab_runs.items[at].first < first) at += 1; + // A recycled run must not overlap one this collection already owns: + // that would be the pager handing out pages twice, and the symptom + // would be a document quietly overwritten rather than anything failing. + if (at > 0) { + const prev = self.slab_runs.items[at - 1]; + assert_msg(prev.first + prev.pages <= first, "a slab run overlaps the one below it"); + } + if (at < self.slab_runs.items.len) { + assert_msg(first + pages <= self.slab_runs.items[at].first, "a slab run overlaps the one above it"); + } + try self.slab_runs.insert(gpa, at, .{ + .first = first, + .pages = pages, + .window_first = std.mem.alignForward(u64, @as(u64, first) << pgr.page_shift, pgr.map_align), + .dead = dead, + }); + } + + /// The run holding `off`, or null if no run does. Binary search, which the + /// sorted list is for: `mark_dead` runs once per evicted document, and a + /// collection with a fragmented slab can own thousands of runs. + fn run_of(self: *const Collection, off: u64) ?usize { + const page: u32 = @intCast(off >> pgr.page_shift); + var lo: usize = 0; + var hi: usize = self.slab_runs.items.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + const r = self.slab_runs.items[mid]; + if (page < r.first) { + hi = mid; + } else if (page >= r.first + r.pages) { + lo = mid + 1; + } else { + return mid; + } + } + return null; + } + + /// Record that `[off, off+len)` of slab is garbage. + /// + /// Infallible, and that is the constraint the whole representation was + /// chosen around: the two callers are `evict_doc`, which runs after the + /// log record is already durable, and the appender's skip accounting. An + /// allocation here would be a failure with nowhere to report it. + /// + /// Bytes that fall outside a whole window -- the head of a run before its + /// first window boundary, and the tail after its last -- go to + /// `dead_unlocated`. They are not lost, only unreclaimable on their own. + fn mark_dead(self: *Collection, off: u64, len: u64) void { + if (len == 0) return; + const ri = self.run_of(off) orelse { + assert_msg(false, "dead slab bytes fall outside every run the collection owns"); + unreachable; + }; + const r = &self.slab_runs.items[ri]; + const stop = off + len; + // A document is written inside one extent by construction + // (`slab_reserve` never lets an append cross `slab_end`), so a dead + // range that crosses a run boundary means an offset from a different + // layout -- a stale index entry, which is the failure the layout epoch + // exists to prevent. + assert_msg(stop <= r.end(), "a dead slab range crosses the end of the run holding it"); + var pos = off; + if (pos < r.window_first) { + const n = @min(stop, r.window_first) - pos; + self.dead_unlocated += n; + pos += n; + } + const win_end = r.window_end(); + while (pos < stop and pos < win_end) { + const w: usize = @intCast((pos - r.window_first) / pgr.map_align); + const w_end = r.window_first + (w + 1) * pgr.map_align; + const n = @min(stop, w_end) - pos; + // A window cannot hold more dead bytes than it has bytes. Tripping + // this means the same range was marked twice -- a double eviction, + // or a recycled offset marked against the previous owner's map. + assert_msg(r.dead[w] + n <= pgr.map_align, "a slab window holds more dead bytes than it has"); + r.dead[w] += @intCast(n); + pos += n; + } + if (pos < stop) self.dead_unlocated += stop - pos; + } + + /// Garbage this collection has placed in windows. Walks every window, so it + /// belongs to the reclamation scan and to tests, not to a hot path. + fn dead_located(self: *const Collection) u64 { + var sum: u64 = 0; + for (self.slab_runs.items) |r| { + for (r.dead) |d| sum += d; + } + return sum; + } + + /// Drop the window maps and the run list. The pages themselves are the + /// caller's business -- a drop hands them to the pager, a rebuild has + /// already done so. + fn free_runs(self: *Collection, gpa: std.mem.Allocator) void { + for (self.slab_runs.items) |r| gpa.free(r.dead); + self.slab_runs.clearRetainingCapacity(); + } + /// Append `bytes` to the slab, returning its flat offset. The last /// segment holds up to `slab_segment_size`; a full one starts the next. /// Make room for a document of `len` bytes, so the append that follows @@ -210,7 +404,7 @@ pub const Collection = struct { )); try self.pager.reserve_pages(&self.hold, want_pages); const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages); - try self.slab_extents.append(gpa, .{ .first = first, .pages = want_pages }); + try self.insert_run(gpa, first, want_pages); self.slab_tail = @as(u64, first) << pgr.page_shift; self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift); return skipped; @@ -230,8 +424,15 @@ pub const Collection = struct { /// Two collections churning against a checkpoint every 32 MiB skip up to a /// system page each per checkpoint, and an abandoned extent tail can be /// most of 8 MiB. Counted here, that garbage arms compaction like any other. + /// + /// Skipped slab always starts at the cursor -- all three callers write off + /// the bytes in front of it and then move it -- so this is also where the + /// window map learns about it. That matters more for the abandoned tail + /// than for the round-up: most of 8 MiB of a run is whole windows, dead on + /// arrival, and reclaiming them is free. fn note_skip(self: *Collection, bytes: u64) u64 { self.slab_used += bytes; + self.mark_dead(self.slab_tail, bytes); return bytes; } @@ -619,10 +820,11 @@ pub const Engine = struct { coll.indexes.deinit(self.gpa); // Give the slab's pages back. They become reusable two generations // later, so a fallback to the previous image still finds them intact. - for (coll.slab_extents.items) |e| { - self.pager.free_pages(e.first, e.pages) catch {}; + for (coll.slab_runs.items) |r| { + self.pager.free_pages(r.first, r.pages) catch {}; } - coll.slab_extents.deinit(self.gpa); + coll.free_runs(self.gpa); + coll.slab_runs.deinit(self.gpa); self.gpa.destroy(coll); } @@ -700,6 +902,11 @@ pub const Engine = struct { coll.doc_count -= 1; assert_msg(coll.live_bytes >= old_bytes.len, "evicting a document would underflow the collection's live bytes"); coll.live_bytes -= old_bytes.len; + // Last, and after every index entry naming these bytes is gone. That + // ordering is what makes window reclamation safe to do by counting + // alone: a window only reaches `map_align` dead once every document + // touching it has been through here, so nothing reachable is inside it. + coll.mark_dead(off, old_bytes.len); } /// `bson.encode_key` of a stored document's `_id`, owned by the caller. @@ -1582,16 +1789,21 @@ pub const Engine = struct { try coll.lock.lock(self.io); defer coll.lock.unlock(self.io); - const old_extents = try self.gpa.dupe(pgr.Extent, coll.slab_extents.items); + var old_extents = try self.gpa.alloc(pgr.Extent, coll.slab_runs.items.len); defer self.gpa.free(old_extents); + for (coll.slab_runs.items, 0..) |r, i| old_extents[i] = .{ .first = r.first, .pages = r.pages }; // Fresh slab. The old extents stay allocated until the free list // releases them, two generations on. - coll.slab_extents.clearRetainingCapacity(); + coll.free_runs(self.gpa); coll.slab_tail = 0; coll.slab_end = 0; coll.slab_used = 0; coll.live_bytes = 0; + // The rebuild is the one place the located and unlocated halves are + // both reset: every byte it copies is live, so a fresh slab has no + // garbage to place. Anything the copy skips is marked as it happens. + coll.dead_unlocated = 0; // Walk in _id order, which is also the order the new slab ends up in -- // so a later scan reads it sequentially. @@ -1848,7 +2060,7 @@ pub const Engine = struct { while (coll_it.next()) |ce| { const coll = ce.value_ptr.*; // Everything below this line is written by a collection's own - // writer under its own lock, and `slab_extents` is an ArrayList + // writer under its own lock, and `slab_runs` is an ArrayList // that `slab_reserve` appends to -- so reading it under only the // shared catalog lock could walk a slice a concurrent append had // already reallocated. Lock order is catalog then collection, @@ -1866,10 +2078,14 @@ pub const Engine = struct { "a collection cannot hold more live bytes than it ever appended", ); dead_sum += coll.slab_used - coll.live_bytes; - try put_u32(gpa, out, @intCast(coll.slab_extents.items.len)); - for (coll.slab_extents.items) |e| { - try put_u32(gpa, out, e.first); - try put_u32(gpa, out, e.pages); + // Runs, not extents, but the same two u32s: only the window map + // is new and it is deliberately not persisted (see + // `dead_unlocated`), so `catalog_version` stays 1 and there is + // no second read path to keep working. + try put_u32(gpa, out, @intCast(coll.slab_runs.items.len)); + for (coll.slab_runs.items) |r| { + try put_u32(gpa, out, r.first); + try put_u32(gpa, out, r.pages); } try put_u32(gpa, out, @intCast(coll.indexes.items.len + 1)); try write_index_catalog(gpa, out, &coll.id_index); @@ -1950,12 +2166,21 @@ pub const Engine = struct { if (coll.slab_used < coll.live_bytes) return error.CorruptCatalog; self.live_bytes += coll.live_bytes; self.dead_bytes += coll.slab_used - coll.live_bytes; + // An open knows *that* the collection has garbage but not + // *where*: the window map is rebuilt empty, and the whole + // amount starts out unlocated. The consequence runs one way -- + // a forgotten dead byte is a window that is not handed back, + // never a live window that is. + coll.dead_unlocated = coll.slab_used - coll.live_bytes; const nex = try r.read_u32(); var e: u32 = 0; while (e < nex) : (e += 1) { const first = try r.read_u32(); const pages = try r.read_u32(); - try coll.slab_extents.append(self.gpa, .{ .first = first, .pages = pages }); + // Sorted insert rather than append: a catalog written + // before runs were address-ordered holds them in + // allocation order, and `run_of` is a binary search. + try coll.insert_run(self.gpa, first, pages); } const nix = try r.read_u32(); // Index 0 is the implicit _id_, already created by @@ -2787,8 +3012,8 @@ test "an append after a checkpoint keeps its extent instead of abandoning it" { try engine.commit(); const coll = engine.get_collection("app", "users").?; - try testing.expectEqual(@as(usize, 1), coll.slab_extents.items.len); - const extent_start = @as(u64, coll.slab_extents.items[0].first) << pgr.page_shift; + try testing.expectEqual(@as(usize, 1), coll.slab_runs.items.len); + const extent_start = coll.slab_runs.items[0].start(); try engine.checkpoint(); const tail_at_checkpoint = coll.slab_tail; @@ -2801,7 +3026,7 @@ test "an append after a checkpoint keeps its extent instead of abandoning it" { try engine.insert("app", "users", &second, &env.gen); try engine.commit(); - try testing.expectEqual(@as(usize, 1), coll.slab_extents.items.len); + try testing.expectEqual(@as(usize, 1), coll.slab_runs.items.len); // A page or two for the tree's copy-on-write is expected; a whole slab // extent is the regression this guards against. try testing.expect(engine.pager.alloc_tail < tail_before + slab_extent_pages); @@ -2831,6 +3056,300 @@ test "an append after a checkpoint keeps its extent instead of abandoning it" { } } +/// A document of roughly `size` bytes, so a test can fill extents without +/// writing tens of thousands of records. +fn make_padded(gpa: std.mem.Allocator, id: i32, size: usize) !bson.Document { + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); + const a = arena.allocator(); + const pad = try a.alloc(u8, size); + @memset(pad, 'x'); + const pairs = try a.alloc(bson.Pair, 2); + pairs[0] = .{ .key = try a.dupe(u8, "_id"), .value = .{ .int32 = id } }; + pairs[1] = .{ .key = try a.dupe(u8, "pad"), .value = .{ .string = pad } }; + return .{ .arena = arena, .pairs = pairs }; +} + +/// Every dead byte the collection knows about, placed or not. +fn dead_total(coll: *const Collection) u64 { + return coll.dead_located() + coll.dead_unlocated; +} + +test "every dead slab byte is counted in exactly one place" { + // The identity the window map rests on: + // + // sum of window counters + dead_unlocated == slab_used - live_bytes + // + // The left side is where the garbage is, the right side is how much there + // is; reclamation reads the first and the compaction trigger reads the + // second, so a drift between them is a rebuild that fires on a clean + // database or a window that is handed back with a document in it. + // + // Both kinds of death are exercised: evicted documents, and the slab the + // appender writes off when a checkpoint freezes the page its cursor points + // into. + // + // Mutation check: drop the `mark_dead` call from `note_skip`, or the + // `mark_dead` call from `evict_doc` -- each removes one of the two ways + // slab dies and the sides part company by that amount. The run *edges* are + // covered separately, by the test below, because whether a run's start is + // `map_align`-aligned is up to the allocator and not something an + // engine-level test can arrange. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // no rebuild may intervene + try engine.lock(); + defer engine.unlock(); + + var i: i32 = 0; + while (i < 40) : (i += 1) { + var d = try make_padded(gpa, i, 6000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + // A checkpoint every few documents, so the appender keeps having to + // round its cursor up to a system page and skipping the bytes between. + if (@mod(i, 7) == 6) { + try engine.commit(); + try engine.checkpoint(); + } + } + try engine.commit(); + + const coll = engine.get_collection("app", "c").?; + // Some skipping must actually have happened, or the test proves only the + // easy half. 6000-byte documents never land flush against a page boundary. + try testing.expect(coll.slab_used > coll.live_bytes); + try testing.expectEqual(coll.slab_used - coll.live_bytes, dead_total(coll)); + + // Now the other kind: evictions. + i = 0; + while (i < 40) : (i += 2) { + try testing.expect(try engine.remove_by_id("app", "c", .{ .int32 = i })); + } + try engine.commit(); + try testing.expectEqual(coll.slab_used - coll.live_bytes, dead_total(coll)); + // And the evicted bytes are mostly placeable: 6000-byte documents are far + // smaller than a window, so they fall inside one rather than off its edge. + try testing.expect(coll.dead_located() > coll.dead_unlocated); +} + +test "dead bytes outside a whole window are counted but not placed" { + // A run is allocated in 4 KiB pages but reclaimed in `map_align` windows, + // so unless the allocator happens to hand back an aligned run there is a + // head below its first window boundary and a tail above its last. Bytes + // that die there can never be given back on their own -- but they are + // still garbage, and if they were simply dropped the amount of garbage the + // collection reports would fall short of the amount it has, which is a + // compaction that never fires. + // + // Driven against `mark_dead` directly: the alignment of a real slab extent + // is the allocator's business and an engine-level test cannot arrange for + // an unaligned one. + // + // Mutation checks, each red on its own: drop the trailing `if (pos < + // stop)` and the tail bytes go uncounted; drop the leading `if (pos < + // r.window_first)` and the window index underflows instead, which takes + // down half the suite. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, env.io, tmp.path); + defer engine.deinit(); + try engine.lock(); + defer engine.unlock(); + + var d = try make_doc(gpa, 1, "alice"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + const coll = engine.get_collection("app", "c").?; + + // A run deliberately starting one 4 KiB page past a window boundary, long + // enough to hold two whole windows plus a partial one at each end. + const pages_per_window: u32 = @intCast(pgr.map_align / pgr.page_size); + if (pages_per_window < 2) return error.SkipZigTest; // no edges to test + const owned = coll.slab_runs.items[0]; + const aligned = std.mem.alignForward(u32, owned.first + owned.pages + 8, pages_per_window); + try coll.insert_run(gpa, aligned + 1, 3 * pages_per_window); + const ri = coll.run_of(@as(u64, aligned + 1) << pgr.page_shift).?; + const r = coll.slab_runs.items[ri]; + try testing.expectEqual(@as(usize, 2), r.dead.len); + try testing.expect(r.window_first > r.start()); + try testing.expect(r.window_end() < r.end()); + + const before = coll.dead_unlocated; + // The head, one whole window, and the tail. + coll.mark_dead(r.start(), r.window_first - r.start()); + coll.mark_dead(r.window_first, pgr.map_align); + coll.mark_dead(r.window_end(), r.end() - r.window_end()); + + try testing.expectEqual(@as(u64, pgr.map_align), coll.dead_located()); + try testing.expectEqual( + before + (r.window_first - r.start()) + (r.end() - r.window_end()), + coll.dead_unlocated, + ); + // The whole window is full and the one beside it untouched: the head and + // tail bytes did not leak into a counter that would hand a window back. + try testing.expectEqual(@as(WindowDead, pgr.map_align), r.dead[0]); + try testing.expectEqual(@as(WindowDead, 0), r.dead[1]); +} + +test "a slab run holds whole documents" { + // `mark_dead` charges a document to the run holding its first byte and + // asserts the rest is in the same run. That is only sound because + // `slab_reserve` never lets an append cross `slab_end` -- so check it + // against a collection that owns several runs, including one taken for a + // single oversized document. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); + try engine.lock(); + defer engine.unlock(); + + var small = try make_padded(gpa, 1, 1000); + defer small.deinit(); + try engine.insert("app", "c", &small, &env.gen); + // Larger than the standard 8 MiB extent, so it gets a run of its own and + // the next document forces a third. + var huge = try make_padded(gpa, 2, 9 * 1024 * 1024); + defer huge.deinit(); + try engine.insert("app", "c", &huge, &env.gen); + var after = try make_padded(gpa, 3, 1000); + defer after.deinit(); + try engine.insert("app", "c", &after, &env.gen); + try engine.commit(); + + const coll = engine.get_collection("app", "c").?; + try testing.expect(coll.slab_runs.items.len >= 2); + // Sorted by page number, and non-overlapping. + for (coll.slab_runs.items[1..], 0..) |r, k| { + const prev = coll.slab_runs.items[k]; + try testing.expect(prev.first + prev.pages <= r.first); + } + var it = coll.id_index.iter(); + while (it.next()) |entry| { + const ri = coll.run_of(entry.off) orelse return error.TestUnexpectedResult; + const r = coll.slab_runs.items[ri]; + try testing.expect(entry.off + coll.doc_bytes(entry.off).len <= r.end()); + } +} + +test "a slab run recycled to a lower address keeps the list sorted" { + // Runs used to be held in allocation order, which was fine while an extent + // could only be appended. Reclamation makes a recycled run arrive at an + // address *below* one the collection already owns, and `run_of` is a binary + // search -- so the list has to be ordered by page, not by age. + // + // Exercised on the structure directly: producing a lower-addressed + // allocation through the engine needs the free list to be primed, which is + // 3.3's business, and this invariant should hold before then. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, env.io, tmp.path); + defer engine.deinit(); + try engine.lock(); + defer engine.unlock(); + + var d = try make_doc(gpa, 1, "alice"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + const coll = engine.get_collection("app", "c").?; + + // Three more runs, arriving out of order and clear of the one the insert + // took. They are never written to, so no pages need to exist. + const base: u32 = coll.slab_runs.items[0].first + coll.slab_runs.items[0].pages + 16; + try coll.insert_run(gpa, base + 200, 8); + try coll.insert_run(gpa, base, 8); + try coll.insert_run(gpa, base + 100, 8); + try testing.expectEqual(@as(usize, 4), coll.slab_runs.items.len); + for (coll.slab_runs.items[1..], 0..) |r, k| { + try testing.expect(coll.slab_runs.items[k].first < r.first); + } + // And every one of them is findable at its own address, which is the point + // of the ordering. + for ([_]u32{ base, base + 100, base + 200 }) |first| { + const off = @as(u64, first) << pgr.page_shift; + const ri = coll.run_of(off) orelse return error.TestUnexpectedResult; + try testing.expectEqual(first, coll.slab_runs.items[ri].first); + } + try testing.expect(coll.run_of(@as(u64, base + 8) << pgr.page_shift) == null); +} + +test "a restart forgets where the garbage is, not that there is any" { + // The window map is deliberately not persisted: it would be a new catalog + // field, a new version, and a second read path, to save re-deriving + // something the collection can live without. What must survive is the + // *amount*, because that is what arms compaction -- so an open puts the + // whole of it into `dead_unlocated` and the identity still holds with every + // window counter at zero. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var dead_before: u64 = 0; + { + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); + try engine.lock(); + defer engine.unlock(); + var i: i32 = 0; + while (i < 20) : (i += 1) { + var d = try make_padded(gpa, i, 4000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + i = 0; + while (i < 20) : (i += 2) { + _ = try engine.remove_by_id("app", "c", .{ .int32 = i }); + } + try engine.commit(); + const coll = engine.get_collection("app", "c").?; + try testing.expect(coll.dead_located() > 0); + dead_before = coll.slab_used - coll.live_bytes; + try engine.checkpoint(); + } + + var engine2 = try Engine.open(gpa, io, tmp.path); + defer engine2.deinit(); + const coll = engine2.get_collection("app", "c").?; + try testing.expectEqual(dead_before, coll.slab_used - coll.live_bytes); + try testing.expectEqual(@as(u64, 0), coll.dead_located()); + try testing.expectEqual(dead_before, coll.dead_unlocated); + try testing.expectEqual(dead_before, dead_total(coll)); + // The runs came back too, and in a shape `run_of` can use. + var it = coll.id_index.iter(); + while (it.next()) |entry| try testing.expect(coll.run_of(entry.off) != null); +} + test "a replace that changes nothing is not a write" { // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm. // Red on all three: the log grows, the document is superseded so the engine @@ -3401,7 +3920,7 @@ test "a checkpoint runs alongside writers on several collections" { // counters while holding only the *shared catalog* lock -- and a writer // holds that same lock shared, taking the collection's lock exclusively. // So the snapshot walked structures its owner was free to mutate, and - // `slab_extents` is an ArrayList a new extent appends to: a reallocation + // `slab_runs` is an ArrayList a new extent inserts into: a reallocation // mid-walk leaves the serializer reading freed memory. // // Several collections rather than one, because the interesting overlap is a @@ -4485,12 +5004,7 @@ fn id_key_for(gpa: std.mem.Allocator, v: bson.Value) ![]u8 { /// index from one still holding pre-rebuild offsets -- the old bytes are on the /// free list rather than overwritten, so reading them still succeeds. fn offset_in_extents(coll: *const Collection, off: u64) bool { - for (coll.slab_extents.items) |e| { - const first = @as(u64, e.first) << pgr.page_shift; - const end = first + (@as(u64, e.pages) << pgr.page_shift); - if (off >= first and off < end) return true; - } - return false; + return coll.run_of(off) != null; } /// Document bytes at an absolute file offset, without needing the Collection. -- 2.39.5 From afdb44fe906d66101887f549d6ad2be1339fc2b6 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 16:50:33 +0300 Subject: [PATCH 28/37] db: a slab window with nothing live in it goes back to the pager The reclamation itself. A checkpoint now begins by handing back every `map_align` window whose dead-byte counter has reached `map_align`, splitting the runs around what is kept. Counting is the entire liveness test, and that is what makes this cheap. `evict_doc` removes a document's index entries before marking its bytes dead, so a window reaches `map_align` only once every document with a byte in it is unreachable -- "no live bytes" and "no reference to these bytes" are the same statement, established without scanning anything. Inside `checkpoint` rather than a hook after it, because reclamation changes two things that must agree: `slab_runs`, which the catalog describes, and the pager's free list. One `publish` makes both durable. A crash before it leaves the old catalog and the old free list -- no reclamation happened -- and a crash after leaves both describing the new ownership. No new record type, no replay path, no ordering in between to get wrong. It also means the write path pays only for a counter update and the cadence of returning pages is the checkpoint threshold. `full_windows` counts windows that have reached `map_align`, so a collection with nothing to give back is not scanned at all. Without it every checkpoint would walk every window of every collection -- O(slab) regardless of workload, and the workload this design is known not to help (small documents on 16 KiB pages) is exactly the one that would pay it for nothing. A failure changes nothing: the replacement run list is built whole before the old one is touched, so out of memory means the garbage stays and the next checkpoint tries again. Past the last fallible step the list is swapped in first and the pages handed over second; a `free_pages` that fails there leaks the run, which costs space. The other order would leave pages owned twice. `slab_used` changes meaning from "bytes ever appended" to "slab consumed and not yet given back". That is what keeps `slab_used - live_bytes` equal to the garbage the collection still has, with no new persistent field -- both halves are already in the catalog. `layout_epoch` is bumped when, and only when, a collection actually gave something back. Reclamation does not move a live document, so a cursor's live offsets stay good; but a saved offset list can name a page now on the free list, and reading it would succeed and return plausible garbage rather than fail. `cursor_still_valid` kills such a cursor with QueryPlanKilled, as a rebuild already does. Not bumping it otherwise matters just as much: every cursor on a busy collection would die on the checkpoint cadence for nothing. Three tests. The survivor: 199 of 200 documents deleted, and the window holding the last one is not given back, still reads, and goes back only once it is empty too. The deferral: a reclaimed page is in `free_hold` after the publish that freed it and in `free_ready` only after the next one -- asserted on the pager's lists, since `free_ready_pages()` also moves for copy-on-write victims. And the epoch, added to the existing three-promise cursor test: unmoved by a checkpoint that reclaims nothing, moved by one that does. Two Stage 0.6 tests changed, in the direction that was the point. "the slab counts what the appender skips" asserted that an abandoned extent tail survives a checkpoint, because only a rebuild could reclaim it; now the checkpoint gives 4 MiB of it straight back and what stays is the edges. "a rebuild leaves behind what its own copying skipped" asserted 2-4 MiB left after a rebuild; the checkpoint inside `compact` reclaims most of that now, so it asserts the same thing about a smaller number -- the counter is not zeroed and equals what the collection has. Mutations: relax the fullness test to accept a window 2 KiB short of empty and the suite aborts on the append-cursor assertion (the appender's own window is the first thing a loosened test reaches); drop that assertion too and the survivor test goes red alone. Remove the `reclaim_slabs` call, 5 tests fail. Remove the epoch bump, 1. Invert the `full_windows` shortcut, 5. 182/182 unit tests in ReleaseFast and ReleaseSafe, 82/82 fuzz, e2e 49, e2e2 concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86 (the cursor suite, which the epoch bump was most likely to redden), crash-fuzz 60 cycles. --- src/db.zig | 382 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 372 insertions(+), 10 deletions(-) diff --git a/src/db.zig b/src/db.zig index c595778..3af904f 100644 --- a/src/db.zig +++ b/src/db.zig @@ -121,9 +121,16 @@ pub const Collection = struct { /// extent it falls in. slab_tail: u64, slab_end: u64, - /// Document bytes written into this collection's slab since the last - /// rebuild. `slab_tail` cannot answer that -- it is an absolute file offset, - /// so it jumps forward whenever a fresh extent is taken. + /// Slab this collection has consumed and not yet given back. `slab_tail` + /// cannot answer that -- it is an absolute file offset, so it jumps forward + /// whenever a fresh extent is taken. + /// + /// It used to mean "bytes ever appended since the last rebuild", which was + /// the same thing while a rebuild was the only way to get slab back. Window + /// reclamation subtracts from it, and that is what keeps + /// `slab_used - live_bytes` equal to the garbage the collection still has + /// -- with no new persistent field, since both halves are already in the + /// catalog. slab_used: u64, /// This collection's outstanding page promise, for the document slab. Per /// collection because concurrent writers must not release each other's -- @@ -148,6 +155,15 @@ pub const Collection = struct { /// update. What it costs is only that garbage from before a restart is not /// reclaimed window-wise; it still arms compaction like any other. dead_unlocated: u64, + /// Windows whose counter has reached `map_align`, i.e. how much there is + /// for the next checkpoint to give back. + /// + /// It exists so that a checkpoint costs nothing on a collection with + /// nothing to reclaim. Scanning would otherwise be O(slab) per checkpoint + /// whatever the workload -- and the workload this design is *known* not to + /// help, small documents on large system pages, is exactly the one that + /// would pay that for no return. + full_windows: u32, /// Slab handed back to the pager by window reclamation, cumulative for the /// life of the process. Purely an observation: it is what distinguishes /// "the ratio improved because reclamation worked" from "the ratio improved @@ -203,6 +219,7 @@ pub const Collection = struct { .slab_used = 0, .live_bytes = 0, .dead_unlocated = 0, + .full_windows = 0, .reclaimed_bytes = 0, .hold = .{}, .indexes = .empty, @@ -316,7 +333,9 @@ pub const Collection = struct { // this means the same range was marked twice -- a double eviction, // or a recycled offset marked against the previous owner's map. assert_msg(r.dead[w] + n <= pgr.map_align, "a slab window holds more dead bytes than it has"); + const was_full = r.dead[w] == pgr.map_align; r.dead[w] += @intCast(n); + if (!was_full and r.dead[w] == pgr.map_align) self.full_windows += 1; pos += n; } if (pos < stop) self.dead_unlocated += stop - pos; @@ -338,6 +357,122 @@ pub const Collection = struct { fn free_runs(self: *Collection, gpa: std.mem.Allocator) void { for (self.slab_runs.items) |r| gpa.free(r.dead); self.slab_runs.clearRetainingCapacity(); + self.full_windows = 0; + } + + /// One piece of a run that survives reclamation, with a window map of its + /// own copied out of the original. + /// + /// Every kept piece gets a fresh array, including a run nothing was taken + /// from. Moving the original array instead would save a copy and make the + /// failure path have to know which arrays it still owns -- the version that + /// tried it had a double free in the out-of-memory case, which is the one + /// case nothing exercises. + fn keep_piece( + out: *std.ArrayListUnmanaged(SlabRun), + gpa: std.mem.Allocator, + r: SlabRun, + p0: u32, + p1: u32, + ) !void { + const wf = std.mem.alignForward(u64, @as(u64, p0) << pgr.page_shift, pgr.map_align); + const we = std.mem.alignBackward(u64, @as(u64, p1) << pgr.page_shift, pgr.map_align); + const count: usize = if (we > wf) @intCast((we - wf) / pgr.map_align) else 0; + const dead = try gpa.alloc(WindowDead, count); + errdefer gpa.free(dead); + // A piece boundary is either the run's own start/end or a window + // boundary, so the piece's windows line up with a contiguous stretch of + // the original's and the counters can be copied rather than rebuilt. + const base: usize = @intCast((wf - r.window_first) / pgr.map_align); + @memcpy(dead, r.dead[base..][0..count]); + try out.append(gpa, .{ .first = p0, .pages = p1 - p0, .window_first = wf, .dead = dead }); + } + + /// Give back every window with nothing live left in it, splitting the runs + /// around what is kept. Returns the bytes handed to the pager. + /// + /// Counting is the whole test: a window reaches `map_align` dead only once + /// every document with a byte in it has been through `evict_doc`, which + /// removes its index entries before marking it. So "no live bytes" and "no + /// reference to these bytes" are the same statement, and nothing has to be + /// scanned to establish it. + /// + /// Fallible, and arranged so a failure changes nothing: the replacement + /// list is built whole before the old one is touched. The garbage simply + /// stays and the next checkpoint tries again. + fn reclaim_windows(self: *Collection, gpa: std.mem.Allocator) !u64 { + assert_msg( + self.slab_used >= self.live_bytes, + "a collection cannot hold more live bytes than it ever appended", + ); + // The identity, checked where every window is being walked anyway. + assert_msg( + self.dead_located() + self.dead_unlocated == self.slab_used - self.live_bytes, + "the collection's placed and unplaced garbage must add up to its garbage", + ); + var out: std.ArrayListUnmanaged(SlabRun) = .empty; + errdefer { + for (out.items) |p| gpa.free(p.dead); + out.deinit(gpa); + } + var give: std.ArrayListUnmanaged(pgr.Extent) = .empty; + defer give.deinit(gpa); + + var freed: u64 = 0; + for (self.slab_runs.items) |r| { + var keep_from = r.first; + var i: usize = 0; + while (i < r.dead.len) { + if (r.dead[i] != pgr.map_align) { + i += 1; + continue; + } + var j = i + 1; + while (j < r.dead.len and r.dead[j] == pgr.map_align) j += 1; + const from = r.window_first + i * pgr.map_align; + const to = r.window_first + j * pgr.map_align; + // The appender's own extent is off limits, and not by + // filtering: bytes above the cursor have never been written, so + // no window covering them can have reached `map_align` dead. + // Tripping this means a range was marked dead twice. + assert_msg( + to <= self.slab_tail or from >= self.slab_end, + "reclaiming a slab window the append cursor is still walking", + ); + const p_from: u32 = @intCast(from >> pgr.page_shift); + const p_to: u32 = @intCast(to >> pgr.page_shift); + if (p_from > keep_from) try keep_piece(&out, gpa, r, keep_from, p_from); + try give.append(gpa, .{ .first = p_from, .pages = p_to - p_from }); + freed += to - from; + keep_from = p_to; + i = j; + } + if (keep_from < r.first + r.pages) { + try keep_piece(&out, gpa, r, keep_from, r.first + r.pages); + } + } + if (freed == 0) { + for (out.items) |p| gpa.free(p.dead); + out.deinit(gpa); + // Every full window was given back or there were none, so nothing + // is left for the next checkpoint to find. + self.full_windows = 0; + return 0; + } + + // Past the last fallible step: swap the list in, then hand the pages + // over. A `free_pages` that fails here leaks the run -- it is no longer + // the collection's and not yet the pager's -- which costs space and + // nothing else. The other order would leave the same pages owned twice. + for (self.slab_runs.items) |r| gpa.free(r.dead); + self.slab_runs.deinit(gpa); + self.slab_runs = out; + self.full_windows = 0; + for (give.items) |e| self.pager.free_pages(e.first, e.pages) catch {}; + assert_msg(self.slab_used >= self.live_bytes + freed, "reclaiming more slab than the collection has"); + self.slab_used -= freed; + self.reclaimed_bytes += freed; + return freed; } /// Append `bytes` to the slab, returning its flat offset. The last @@ -2314,6 +2449,62 @@ pub const Engine = struct { self.committed_seq = 0; } + /// Hand back every slab window with nothing live left in it, across every + /// collection. The first phase of a checkpoint. + /// + /// Inside the checkpoint rather than a hook after it, and that placement is + /// the whole safety argument. Reclamation changes two things: it splits + /// `slab_runs`, which the catalog describes, and it calls + /// `pager.free_pages`, which the free list describes. The checkpoint's + /// single `publish` makes both durable together, so a crash before it + /// leaves the old catalog and the old free list -- no reclamation happened + /// -- and a crash after leaves both describing the new ownership. There is + /// no order in between to get wrong, and no new record type or replay path. + /// + /// Batched with the checkpoint for a second reason: the write path pays + /// only for a counter update, and the cadence of actually returning pages + /// is the checkpoint threshold rather than per-delete. + /// + /// Catalog shared, then each collection exclusive, one at a time -- the + /// order `compact` and `write_catalog` both use. + fn reclaim_slabs(self: *Engine) void { + self.catalog_lock.lockSharedUncancelable(self.io); + defer self.catalog_lock.unlockShared(self.io); + var db_it = self.dbs.iterator(); + while (db_it.next()) |db_entry| { + var coll_it = db_entry.value_ptr.collections.iterator(); + while (coll_it.next()) |ce| self.reclaim_collection(ce.value_ptr.*); + } + } + + fn reclaim_collection(self: *Engine, coll: *Collection) void { + coll.lock.lockUncancelable(self.io); + defer coll.lock.unlock(self.io); + // The common case, and the reason this is affordable at every + // checkpoint: a collection with no full window is not scanned at all. + if (coll.full_windows == 0) return; + // Out of memory here means the garbage stays where it is. Nothing is + // lost and the next checkpoint tries again. + const freed = coll.reclaim_windows(self.gpa) catch return; + if (freed == 0) return; + self.counter_lock.lockUncancelable(self.io); + assert_msg(self.dead_bytes >= freed, "reclaiming more slab than the engine counts as dead"); + self.dead_bytes -= freed; + self.counter_lock.unlock(self.io); + // A cursor holding slab offsets is now holding some that name pages + // this collection no longer owns -- and reading them would succeed, + // since the pages are only on the free list, so the answer would be + // plausible garbage rather than an error. `cursor_still_valid` compares + // this epoch and kills such a cursor with QueryPlanKilled, which is + // what a rebuild already does to it. + // + // Only when something was actually given back: a collection that + // reclaimed nothing must not have its cursors killed on the cadence of + // the checkpoint. + self.layout_epoch_seq += 1; + coll.layout_epoch = self.layout_epoch_seq; + } + /// Publish the current state as a checkpoint. /// /// The watermark equals the sequence the log has already made durable, never @@ -2323,6 +2514,7 @@ pub const Engine = struct { /// D6) reduced to an ordering. pub fn checkpoint(self: *Engine) !void { try self.commit(); + self.reclaim_slabs(); var buf: std.ArrayListUnmanaged(u8) = .empty; defer buf.deinit(self.gpa); @@ -3350,6 +3542,134 @@ test "a restart forgets where the garbage is, not that there is any" { while (it.next()) |entry| try testing.expect(coll.run_of(entry.off) != null); } +/// Whether `page` falls in one of the pager's free-list generations. +fn in_extents(list: []const pgr.Extent, page: u32) bool { + for (list) |e| if (page >= e.first and page < e.first + e.pages) return true; + return false; +} + +test "a slab window with one live document in it is never given back" { + // The load-bearing test of window reclamation. A window goes back when its + // counter reaches `map_align`, which is a statement about *bytes*, not + // about documents -- so the thing that must never happen is a window handed + // to the pager while a document still sits in it. The document would still + // read, because a freed page is only on a list, so the failure would be + // silent until the pages were handed out again and overwritten. + // + // Mutation check: relax the fullness test in `reclaim_windows` to + // `r.dead[i] + 2048 < pgr.map_align`, so a window 2 KiB short of empty + // qualifies. On its own that aborts the whole suite on the append-cursor + // assertion instead -- the appender's own window is the first thing a + // loosened test reaches, which is worth knowing. Drop that assertion too + // and this is the test that goes red, alone. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // no rebuild may intervene + try engine.lock(); + defer engine.unlock(); + + var i: i32 = 0; + while (i < 200) : (i += 1) { + var d = try make_padded(gpa, i, 2000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + const coll = engine.get_collection("app", "c").?; + + // Everything dies but one document, roughly in the middle of the slab. + i = 0; + while (i < 200) : (i += 1) { + if (i == 100) continue; + try testing.expect(try engine.remove_by_id("app", "c", .{ .int32 = i })); + } + try engine.commit(); + const survivor_enc = try id_key_for(gpa, bson.Value{ .int32 = 100 }); + defer gpa.free(survivor_enc); + const survivor = coll.id_index.lookup_exact(survivor_enc).?; + + try engine.checkpoint(); + // Most of the slab went back... + try testing.expect(coll.reclaimed_bytes > 100 * 2000); + // ...but not the window the survivor is in, and it still reads. + try testing.expect(coll.run_of(survivor) != null); + try testing.expect(std.mem.indexOf(u8, coll.doc_bytes(survivor), "xxxx") != null); + // The accounting followed the pages: what is left is what is left. + try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); + try testing.expectEqual( + coll.slab_used - coll.live_bytes, + coll.dead_located() + coll.dead_unlocated, + ); + + // And once the survivor is gone, its window goes too. + try testing.expect(try engine.remove_by_id("app", "c", .{ .int32 = 100 })); + try engine.commit(); + try engine.checkpoint(); + try testing.expect(coll.run_of(survivor) == null); +} + +test "a reclaimed slab window is not reusable until two publishes later" { + // Reclamation hands pages to `free_pages`, which withholds them for two + // generations -- and it has to, because the image one generation back is + // still the fallback a crash would open, and its catalog still claims them. + // Handing them straight out would let a write land on pages the recovery + // path is about to read as documents. + // + // Asserted on the pager's own lists rather than on `free_ready_pages()`, + // whose total also moves for copy-on-write victims and the catalog stream. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); + try engine.lock(); + defer engine.unlock(); + + var i: i32 = 0; + while (i < 200) : (i += 1) { + var d = try make_padded(gpa, i, 2000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + const coll = engine.get_collection("app", "c").?; + const owned_before = coll.slab_runs.items[0]; + + i = 0; + while (i < 200) : (i += 1) _ = try engine.remove_by_id("app", "c", .{ .int32 = i }); + try engine.commit(); + try engine.checkpoint(); + try testing.expect(coll.reclaimed_bytes > 0); + + // A page from the first window given back: the run's first window is all + // dead now, so its first page is no longer the collection's. + const gone = @as(u32, @intCast(owned_before.window_first >> pgr.page_shift)); + try testing.expect(coll.run_of(@as(u64, gone) << pgr.page_shift) == null); + try testing.expect(gone >= owned_before.first); + + // One publish has happened, so it is held, not ready. + try testing.expect(!in_extents(engine.pager.free_ready.items, gone)); + try testing.expect(in_extents(engine.pager.free_hold.items, gone)); + + // The second publish is what makes it allocatable. + try engine.checkpoint(); + try testing.expect(in_extents(engine.pager.free_ready.items, gone)); +} + test "a replace that changes nothing is not a write" { // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm. // Red on all three: the log grows, the document is superseded so the engine @@ -3656,9 +3976,19 @@ test "the slab counts what the appender skips" { try testing.expectEqual(gap + abandoned, engine.dead_bytes); try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); - // And it survives the round trip, because it is in `slab_used`. + // And the next checkpoint gives most of it straight back. An abandoned + // extent tail is whole windows with nothing live in them, which is exactly + // what window reclamation is for -- so counting it was not bookkeeping for + // its own sake, it is what made this reclaimable at all. + // + // What stays is the edges: the round-up gap, which shares its window with + // the live documents below it, and the bytes of the run outside any whole + // window. try engine.checkpoint(); - try testing.expectEqual(gap + abandoned, engine.dead_bytes); + try testing.expect(coll.reclaimed_bytes > 4 * 1024 * 1024); + try testing.expectEqual(gap + abandoned - coll.reclaimed_bytes, engine.dead_bytes); + try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); + try testing.expect(engine.dead_bytes < gap + 2 * pgr.map_align); } test "a rebuild leaves behind what its own copying skipped" { @@ -3706,11 +4036,16 @@ test "a rebuild leaves behind what its own copying skipped" { const coll = engine.get_collection("app", "c").?; try testing.expectEqual(@as(u64, 2), coll.doc_count); - // The deleted document is gone from the slab, but the gap the copy left - // between the two survivors is not -- and the engine says so. + // The deleted document is gone from the slab, and the gap the copy left + // between the two survivors is now mostly gone too -- `compact` ends in a + // checkpoint, and a checkpoint reclaims whole windows. What survives is the + // edges of that gap, which is a smaller number than this test used to + // assert but the same statement: the counter is *not* zeroed, and it equals + // what the collection actually has. try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); - try testing.expect(engine.dead_bytes > 2 * 1024 * 1024); - try testing.expect(engine.dead_bytes < 4 * 1024 * 1024); + try testing.expect(coll.reclaimed_bytes > 2 * 1024 * 1024); + try testing.expect(engine.dead_bytes > 0); + try testing.expect(engine.dead_bytes < 4 * pgr.map_align); } test "dropping a collection does not arm compaction" { @@ -5119,8 +5454,35 @@ test "the epochs that invalidate a cursor move exactly when they must" { try testing.expect(after_recreate != after_rebuild); try testing.expect(after_recreate != before); - // And the index-level token, which guards the position hint. + // Reclamation does not move a live document, so a cursor's *live* offsets + // stay good -- but the pages it gives back can be handed out again, and a + // cursor's saved offset list may name one of them. Same remedy, and the + // same token. + // + // Both halves matter. A checkpoint that reclaims nothing must leave the + // epoch alone, or every open cursor on a busy collection dies on the + // checkpoint cadence for nothing. try engine.lock(); + var j: i32 = 0; + while (j < 200) : (j += 1) { + var d = try make_padded(gpa, 1000 + j, 2000); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + const quiet_before = engine.get_collection("app", "c").?.layout_epoch; + try engine.checkpoint(); + try testing.expectEqual(quiet_before, engine.get_collection("app", "c").?.layout_epoch); + + j = 0; + while (j < 200) : (j += 1) _ = try engine.remove_by_id("app", "c", .{ .int32 = 1000 + j }); + try engine.commit(); + try engine.checkpoint(); + const after_reclaim = engine.get_collection("app", "c").?.layout_epoch; + try testing.expect(engine.get_collection("app", "c").?.reclaimed_bytes > 0); + try testing.expect(after_reclaim != quiet_before); + + // And the index-level token, which guards the position hint. const coll = engine.get_collection("app", "c").?; const index_before = coll.id_index.epoch; try coll.id_index.reset_tree(gpa); -- 2.39.5 From 7fe1009243b09917227332e798fde36b52becdea Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 17:03:42 +0300 Subject: [PATCH 29/37] db/pager: a slab extent comes off the free list when one fits Without this the previous commit is decorative. Windows go back, the free list fills up, and the file grows by the whole write volume anyway -- because nothing asks for the pages in the shape they arrive in. `take_free` cannot serve a slab extent from reclaimed windows, and that is on purpose. It is best fit precisely so the thousands of single-page copy-on-write requests per generation cannot dismantle the large runs; the consequence is that a 2048-page extent request never matches anything smaller, and reclamation hands back runs a few windows at a time. So `alloc_slab_run` is a second policy in the same allocator: at least `min_pages`, at most `max_pages`, longest available so the collection switches extents as rarely as possible, ties to the smallest source run so the big ones stay as whole as they can. It takes a partial run when it cannot have a whole one and it is allowed to trim a larger one -- there is no cannibalisation to fear when the request is itself at least 1 MiB, and what it leaves behind is a run rather than a hole. `take_free` is untouched and its pinned first-fit mutation test still passes. What it hands out is aligned to `map_align`, which is not cosmetic. That is the granularity writeback tears at and the granularity reclamation gives back at, so a run starting mid-system-page both wastes its first window and shares a kernel page with whatever holds the rest of it -- for a page still in the published image, exactly the tearing `mark_appendable` refuses to risk. The trimmed edges stay on the free list. The caller's floor is 1 MiB: a shorter extent is exhausted after a handful of documents and every exhaustion writes off what is left of the one before it. Measured, in the new engine-level test: delete-and-refill of 400 16 KiB documents per round, four rounds. The tail stands at 2068 pages after the first round and 2083 after three more of the same volume -- 15 pages of growth against 4800 pages written. That is the number the whole milestone is about, and it is the one Risk 5 in the plan says to check directly rather than inferring from a ratio. Three tests. The churn one above. The pager's policy: what it hands out starts and ends on a system-page boundary, comes out of the long run rather than the short one and past the long one's unaligned first page, and everything not handed out is still on the list; a run below the floor is left alone. Mutations: raise `slab_run_min_pages` to a whole extent and the churn test goes red (which is also the measurement saying the floor has to stay well under an extent); drop the alignment and the pager test goes red on an odd-page run; drop the `usable < min_pages` test and the short run is handed out. 184/184 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, e2e 49, e2e2 concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/db.zig | 98 +++++++++++++++++++++++++++++++--- src/pager.zig | 143 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 8 deletions(-) diff --git a/src/db.zig b/src/db.zig index 3af904f..d3a8717 100644 --- a/src/db.zig +++ b/src/db.zig @@ -34,6 +34,16 @@ const assert_msg = @import("assert.zig").assert_msg; /// Slack is bounded by one extent per collection. const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size; +/// Shortest run worth taking off the free list for a slab, 1 MiB. Below this +/// the extent is exhausted after a few documents, and every exhaustion writes +/// off whatever is left of the one before it. +const slab_run_min_pages: u32 = slab_extent_pages / 8; + +/// Pages a slab allocation of `len` bytes needs at minimum. +fn pages_for(len: usize) u32 { + return @intCast((len + pgr.page_size - 1) / pgr.page_size); +} + const LogKind = enum { upsert, delete, index_create, index_drop }; /// Dead bytes in one `map_align` window. The window is the unit of reclamation @@ -533,15 +543,22 @@ pub const Collection = struct { const skipped = self.note_skip(self.slab_end - self.slab_tail); // A document larger than the standard extent gets one of its own; BSON // reaches 16 MB and the extent is 8 MiB. - const want_pages: u32 = @intCast(@max( - slab_extent_pages, - (len + pgr.page_size - 1) / pgr.page_size, - )); + const want_pages: u32 = @max(slab_extent_pages, pages_for(len)); try self.pager.reserve_pages(&self.hold, want_pages); - const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages); - try self.insert_run(gpa, first, want_pages); - self.slab_tail = @as(u64, first) << pgr.page_shift; - self.slab_end = self.slab_tail + (@as(u64, want_pages) << pgr.page_shift); + // Off the free list first, or window reclamation is decorative: the + // pages come back, nothing asks for them in a shape they arrive in, and + // the file grows by the whole write volume anyway. A floor of 1 MiB, + // because a shorter extent is exhausted after a handful of documents + // and every exhaustion abandons what is left of it -- and because the + // floor is what makes trimming a larger run harmless. + const min_pages: u32 = @min(want_pages, @max(pages_for(len), slab_run_min_pages)); + const run = self.pager.alloc_slab_run(&self.hold, min_pages, want_pages) orelse pgr.Extent{ + .first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages), + .pages = want_pages, + }; + try self.insert_run(gpa, run.first, run.pages); + self.slab_tail = @as(u64, run.first) << pgr.page_shift; + self.slab_end = self.slab_tail + (@as(u64, run.pages) << pgr.page_shift); return skipped; } @@ -3670,6 +3687,71 @@ test "a reclaimed slab window is not reusable until two publishes later" { try testing.expect(in_extents(engine.pager.free_ready.items, gone)); } +test "a churning collection reuses its slab instead of growing the file" { + // The one that decides whether any of this was worth doing. Reclamation can + // be working perfectly -- windows counted, pages handed back, + // `reclaimed_bytes` climbing -- and the file still grow by the whole write + // volume, because nothing asks for the pages in the shape they come back + // in. That is what `alloc_slab_run` is for, and this is what says so. + // + // Delete-and-refill in rounds, with checkpoints per round so reclamation + // gets to run and what it frees becomes allocatable. The first round has to + // grow the file; the ones after it must not. + // + // Measured here: 2068 pages after the first round, 2083 after three more of + // the same volume -- 15 pages of growth against 4800 pages written. + // + // Mutation check: raise `slab_run_min_pages` to a whole extent, so no + // reclaimed run is ever long enough to qualify and every extent request + // bumps the tail. Red -- which is also the measurement that says the floor + // has to stay well below an extent to be any use. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // reuse, not rebuild + try engine.lock(); + defer engine.unlock(); + + // Documents a whole window wide, which is the case the design is built for + // -- see the note on small documents in the churn gate. + const doc_size = pgr.map_align; + const per_round = 400; + var round: i32 = 0; + var tail_after_first: u32 = 0; + while (round < 4) : (round += 1) { + var i: i32 = 0; + while (i < per_round) : (i += 1) { + var d = try make_padded(gpa, round * per_round + i, doc_size); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + i = 0; + while (i < per_round) : (i += 1) { + _ = try engine.remove_by_id("app", "c", .{ .int32 = round * per_round + i }); + } + try engine.commit(); + // Two, so what this round freed is allocatable in the next one. + try engine.checkpoint(); + try engine.checkpoint(); + if (round == 0) tail_after_first = engine.pager.alloc_tail; + } + + const coll = engine.get_collection("app", "c").?; + try testing.expect(coll.reclaimed_bytes > 0); + // Three more rounds of the same volume after the first. Anything left is + // fragmentation the windows could not cover, not the write volume. + const grew = engine.pager.alloc_tail - tail_after_first; + try testing.expect(grew < 3 * per_round * doc_size / pgr.page_size / 4); +} + test "a replace that changes nothing is not a write" { // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm. // Red on all three: the log grows, the document is superseded so the engine diff --git a/src/pager.zig b/src/pager.zig index 3642a35..aa960c6 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -704,6 +704,90 @@ pub const Pager = struct { return first; } + /// Take a run off the free list for a document slab: at least `min_pages`, + /// at most `max_pages`, starting and ending on a system-page boundary. + /// Returns null when nothing on the list qualifies, and the caller bumps the + /// tail instead. + /// + /// `take_free` cannot serve this, and that is the whole reason this exists. + /// It is deliberately best fit -- smallest sufficient run -- so that the + /// thousands of single-page copy-on-write requests per generation cannot + /// dismantle the large runs. A slab extent asks for 2048 pages, and window + /// reclamation gives back runs a few pages at a time, so with an exact-size + /// rule the free list could fill up with reclaimed slab that no slab request + /// would ever take: the pages come back, the file keeps growing, the ratio + /// does not move. That is the failure this whole milestone is measured + /// against. + /// + /// So this one takes a partial run when it cannot get a whole one, and is + /// allowed to trim a larger one. There is no cannibalisation to fear here: + /// the request is itself large (the caller's floor is 1 MiB), so what it + /// leaves behind is still a usable run rather than a hole. The one-page + /// requests still go through `take_free` unchanged, and its pinned mutation + /// test is untouched. + /// + /// The alignment is not cosmetic. `map_align` is the granularity writeback + /// works at and the granularity reclamation gives back at, so a run that + /// starts mid-system-page both wastes its first window and shares a kernel + /// page with whatever occupies the rest of it -- which for a page still in + /// the published image is the tearing `mark_appendable` refuses to risk. + pub fn alloc_slab_run(self: *Pager, hold: *Reservation, min_pages: u32, max_pages: u32) ?Extent { + assert(min_pages > 0); + assert(min_pages <= max_pages); + self.alloc_lock.lockUncancelable(self.io); + defer self.alloc_lock.unlock(self.io); + assert_msg(max_pages <= hold.pages, "a slab run request overran reserve_pages' promise"); + // A split can leave a piece at each end, so one entry may become two. + // Out of memory before anything is disturbed: the caller falls back to + // bumping the tail, which is what it would have done anyway. + self.free_ready.ensureUnusedCapacity(self.gpa, 1) catch return null; + + const spp: u32 = if (map_align >= page_size) @intCast(map_align / page_size) else 1; + var best: ?usize = null; + var best_first: u32 = 0; + var best_take: u32 = 0; + for (self.free_ready.items, 0..) |e, i| { + const from = std.mem.alignForward(u32, e.first, spp); + const to = std.mem.alignBackward(u32, e.first + e.pages, spp); + if (to <= from) continue; + const usable = to - from; + if (usable < min_pages) continue; + const take = @min(usable, max_pages); + // The longest run available, so the collection switches extents as + // rarely as possible -- every switch abandons what is left of the + // one before it. Ties go to the smallest source run, which leaves + // the big ones as whole as it can. + const better = if (best) |b| + take > best_take or + (take == best_take and e.pages < self.free_ready.items[b].pages) + else + true; + if (better) { + best = i; + best_first = from; + best_take = take; + } + } + const i = best orelse return null; + const e = self.free_ready.items[i]; + const head = best_first - e.first; + const tail_first = best_first + best_take; + const tail = (e.first + e.pages) - tail_first; + if (head > 0) { + self.free_ready.items[i] = .{ .first = e.first, .pages = head }; + if (tail > 0) self.free_ready.appendAssumeCapacity(.{ .first = tail_first, .pages = tail }); + } else if (tail > 0) { + self.free_ready.items[i] = .{ .first = tail_first, .pages = tail }; + } else { + _ = self.free_ready.swapRemove(i); + } + self.reserved_pages -= best_take; + hold.pages -= best_take; + self.unprotect(best_first, best_take); + self.mark_unpublished(best_first, best_take); + return .{ .first = best_first, .pages = best_take }; + } + /// Merge runs that touch, so the holes single-page frees leave behind can add /// up to an extent again. Without it the free list only ever fragments: every /// generation returns thousands of one-page copy-on-write victims, and an @@ -1971,6 +2055,65 @@ test "one-page requests do not carve up the runs the extents need" { try testing.expectEqual(tail_before, pg.alloc_tail); } +test "a slab run comes off the free list aligned, or not at all" { + // `alloc_slab_run` is a second allocation policy in the same allocator, so + // what it must not do is as important as what it must: + // + // - what it hands out starts and ends on a system-page boundary, because + // that is the granularity writeback tears at and the granularity + // reclamation gives back at; + // - a run too short to be worth an extent is left alone; + // - the pieces it trims off stay on the free list rather than leaking. + // + // Mutation checks: drop the `alignForward`/`alignBackward` and the first + // assertion goes red on the odd-page run; drop the `usable < min_pages` + // test and the short run is handed out. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tp = try TmpPager.init(io, 64 << 20); + defer tp.deinit(); + const pg = tp.pg(); + const spp: u32 = @intCast(map_align / page_size); + + // A long run deliberately starting one 4 KiB page past a boundary, and a + // short one, kept apart so coalescing cannot merge them. + _ = try pg.alloc_pages(std.mem.alignForward(u32, pg.alloc_tail, spp) - pg.alloc_tail + 1); + const long = try pg.alloc_pages(400); + _ = try pg.alloc_pages(1); // separator, never freed + const short = try pg.alloc_pages(8); + _ = try pg.alloc_pages(1); // separator, never freed + try testing.expect(long % spp != 0); + try pg.publish(.{ .seq = 1 }); + try pg.free_pages(long, 400); + try pg.free_pages(short, 8); + try pg.publish(.{ .seq = 2 }); + try pg.publish(.{ .seq = 3 }); + const ready_before = pg.free_ready_pages(); + + var hold: Reservation = .{}; + try pg.reserve_pages(&hold, 256); + const run = pg.alloc_slab_run(&hold, 64, 256) orelse return error.TestUnexpectedResult; + pg.release_reservation(&hold); + try testing.expectEqual(@as(u32, 0), run.first % spp); + try testing.expectEqual(@as(u32, 0), run.pages % spp); + try testing.expectEqual(@as(u32, 256), run.pages); + // It came out of the long run, not the short one, and past its unaligned + // first page. + try testing.expect(run.first > long); + try testing.expect(run.first < long + 400); + // Everything not handed out is still on the list. + try testing.expectEqual(ready_before - 256, pg.free_ready_pages()); + + // The short run is below the floor and stays where it is, whatever is asked + // of it; nothing else is left long enough either. + var hold2: Reservation = .{}; + try pg.reserve_pages(&hold2, 256); + try testing.expect(pg.alloc_slab_run(&hold2, 200, 256) == null); + pg.release_reservation(&hold2); + try testing.expectEqual(ready_before - 256, pg.free_ready_pages()); +} + test "a quiet checkpoint stops growing the file" { // Both streams a publish writes are allocated fresh every time, so that a // crash leaves the previous copy readable. Nothing gave them back, and a -- 2.39.5 From 0dd9a0c90881a804aa3cf450166c30dcb90a01b2 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 17:13:47 +0300 Subject: [PATCH 30/37] db: a rebuild copies only the collections that have garbage `compact` walked every collection unconditionally, so garbage in one paid for a full copy of the other thirty-nine. A copy is not free even where it reclaims nothing: it rewrites every document and every index, and it bumps the layout epoch, which kills every open cursor on a collection that had no reason to be touched. The gate is the share `note_compact` already applies to the engine's totals, and using the same one is what keeps them from disagreeing. If no collection passes it then `dead_i < live_i / 4` for every one, so `sum(dead) < sum(live) / 4` and the engine's trigger could not have fired either -- a compaction that runs always rebuilds at least one collection and cannot spin re-arming itself over garbage no rebuild will take. An absolute floor per collection would break exactly that: forty collections each under the floor can sum to well over it. Two existing tests needed real garbage, which is the change working. "a rebuild kills an offsets cursor and spares a streaming one" and "the epochs that invalidate a cursor move exactly when they must" both called `compact` on a clean collection and relied on it rewriting anyway. The first now rewrites all 60 documents (a replace, not a delete, so the drain still checks that the stream yields exactly 60 once each -- and with a changed field, since an identical replace is deliberately not a write). The new test asserts both halves: the dirty collection's epoch moves, the clean one's does not, and the clean one's `slab_used` is unchanged -- a repack starts the slab over, so that number could not survive one. Mutation: delete the guard, and the clean collection's epoch moves too. 185/185 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, e2e 49, e2e2 concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/commands.zig | 23 ++++++++++++- src/db.zig | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/src/commands.zig b/src/commands.zig index 75ee4d5..b3e6685 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -4291,7 +4291,9 @@ test "a rebuild kills an offsets cursor and spares a streaming one" { // // A rebuild is triggered directly rather than through churn, because whether // churn crosses the compaction threshold is not something a test should have - // to guess at. + // to guess at. The garbage below is still needed: `compact` now skips a + // collection with nothing to reclaim, so a clean one is not rewritten at + // all and there would be no rebuild to observe. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); @@ -4315,6 +4317,25 @@ test "a rebuild kills an offsets cursor and spares a streaming one" { try testing.expectEqual(@as(i32, 0), try dispatch_get_more(&ctx, "c", walk.id)); try testing.expectEqual(@as(i32, 0), try dispatch_get_more(&ctx, "c", narrowed.id)); + // Rewrite every document, so the collection has as many dead bytes as live + // ones and is worth rebuilding. A replace rather than a delete: the count + // stays at 60, which is what the drain below checks. + try ctx.engine.lock(); + var again: i32 = 1; + while (again <= 60) : (again += 1) { + const pairs = try testing.allocator.alloc(bson.Pair, 3); + defer testing.allocator.free(pairs); + pairs[0] = .{ .key = "_id", .value = .{ .int32 = again } }; + // A different value: an identical replace is deliberately not a write. + pairs[1] = .{ .key = "a", .value = .{ .int32 = @mod(again, 5) + 100 } }; + pairs[2] = .{ .key = "pad", .value = .{ .string = seed_pad } }; + var doc: bson.Document = .{ .arena = std.heap.ArenaAllocator.init(testing.allocator), .pairs = pairs }; + defer doc.arena.deinit(); + _ = try ctx.engine.replace("test", "c", &doc, ctx.oid_gen); + } + try ctx.engine.commit(); + ctx.engine.unlock(); + try ctx.engine.compact(); // The stream remembers key bytes, which a repack does not change. diff --git a/src/db.zig b/src/db.zig index d3a8717..9d75958 100644 --- a/src/db.zig +++ b/src/db.zig @@ -1930,6 +1930,32 @@ pub const Engine = struct { try self.checkpoint(); } + /// Whether rewriting this collection would pay for itself. The caller holds + /// its lock. + /// + /// A rebuild copies a collection's live bytes to reclaim its dead ones, so + /// the one thing it must not do is copy a collection that has none. It used + /// to: `compact` walked every collection unconditionally, so garbage in one + /// paid for a full copy of the other thirty-nine. + /// + /// The share is the same one `note_compact` applies to the engine's totals, + /// and that is what keeps the two from disagreeing. If no collection passes + /// this test then `dead_i < live_i / 4` for every one of them, so + /// `sum(dead) < sum(live) / 4` and the engine's trigger could not have fired + /// either. So a compaction that runs always rebuilds at least one + /// collection, and cannot spin re-arming itself over garbage no rebuild will + /// take. An absolute floor per collection would break exactly that: forty + /// collections each under the floor can sum to well over it. + fn wants_rebuild(coll: *const Collection) bool { + assert_msg( + coll.slab_used >= coll.live_bytes, + "a collection cannot hold more live bytes than it ever appended", + ); + const dead = coll.slab_used - coll.live_bytes; + if (dead == 0) return false; + return dead * 4 >= coll.live_bytes; + } + /// Copy one collection's live documents into fresh extents and rebuild every /// index against the new offsets. /// @@ -1940,6 +1966,7 @@ pub const Engine = struct { fn rebuild_collection(self: *Engine, coll: *Collection) !void { try coll.lock.lock(self.io); defer coll.lock.unlock(self.io); + if (!wants_rebuild(coll)) return; var old_extents = try self.gpa.alloc(pgr.Extent, coll.slab_runs.items.len); defer self.gpa.free(old_extents); @@ -3752,6 +3779,62 @@ test "a churning collection reuses its slab instead of growing the file" { try testing.expect(grew < 3 * per_round * doc_size / pgr.page_size / 4); } +test "a rebuild copies only the collections that have garbage" { + // `compact` walked every collection unconditionally, so garbage in one paid + // for a full copy of all the others -- and a copy is not free even when it + // reclaims nothing: it rewrites every document and every index, and it + // bumps the layout epoch, which kills every open cursor on a collection + // that had no reason to be touched. + // + // The epoch is the observable, and it is also the user-visible harm: the + // clean collection's cursors survive. + // + // Mutation check: delete the `wants_rebuild` guard. Red -- the clean + // collection's epoch moves too. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // rebuild only when told to + try engine.lock(); + + var i: i32 = 0; + while (i < 40) : (i += 1) { + var d = try make_padded(gpa, i, 3000); + defer d.deinit(); + try engine.insert("app", "dirty", &d, &env.gen); + var c = try make_padded(gpa, i, 3000); + defer c.deinit(); + try engine.insert("app", "clean", &c, &env.gen); + } + // Only one of them loses anything. + i = 0; + while (i < 40) : (i += 2) _ = try engine.remove_by_id("app", "dirty", .{ .int32 = i }); + try engine.commit(); + + const dirty = engine.get_collection("app", "dirty").?; + const clean = engine.get_collection("app", "clean").?; + const dirty_epoch = dirty.layout_epoch; + const clean_epoch = clean.layout_epoch; + const clean_used = clean.slab_used; + engine.unlock(); + + try engine.compact(); + + try testing.expect(dirty.layout_epoch != dirty_epoch); + try testing.expectEqual(clean_epoch, clean.layout_epoch); + // And it was not rewritten: a repack starts the slab over, so its byte + // count would not survive one unchanged. + try testing.expectEqual(clean_used, clean.slab_used); + try testing.expectEqual(@as(u64, 0), clean.slab_used - clean.live_bytes); +} + test "a replace that changes nothing is not a write" { // Mutation check: delete the byte comparison in `upsert`'s `.replace` arm. // Red on all three: the log grows, the document is superseded so the engine @@ -5513,6 +5596,11 @@ test "the epochs that invalidate a cursor move exactly when they must" { defer d.deinit(); try engine.insert("app", "c", &d, &env.gen); } + // Garbage, so the collection is worth rewriting: `compact` skips a + // collection with nothing to reclaim. + i = 0; + while (i < 20) : (i += 1) _ = try engine.remove_by_id("app", "c", .{ .int32 = i }); + try engine.commit(); const before = engine.get_collection("app", "c").?.layout_epoch; engine.unlock(); try testing.expect(before != 0); -- 2.39.5 From 8bf35e707c1f6332d4c394d319bcd77239648669 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 17:18:31 +0300 Subject: [PATCH 31/37] commands: serverStatus reports what the slab is doing A `multifora` subdocument -- named so nobody mistakes it for a MongoDB section -- carrying `liveBytes`, `deadBytes`, `slabBytes`, `reclaimedBytes`, `slabRuns`, `freeReadyPages`, `allocTail` and `compactions`. The milestone's gate cannot be read without them. A steady-state size ratio can look respectable while reclamation does nothing at all: the file grows, a rebuild periodically halves it, and the average comes out fine. What distinguishes the two is `reclaimedBytes` rising while `allocTail` stays put, and no ratio shows that. Same for rebuilds -- "the ratio improved" and "the ratio improved because reclamation worked rather than because a rebuild ran" are different results, so `compactions` counts collections rewritten. The byte figures are summed from the collections rather than read off the engine's running totals, so this reports the same side of the comparison `checkpoint` asserts. A counter that had drifted from the catalog would otherwise make the gate measure the drift. Every field is present at zero. A gate that cannot tell "no pages ready" from "field missing" cannot be read at all. 186/186 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, and the full e2e matrix. --- src/commands.zig | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ src/db.zig | 58 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/src/commands.zig b/src/commands.zig index b3e6685..90bb041 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -394,6 +394,24 @@ fn cmd_server_status(ctx: *Context, _: *wire.Message, reply: *wire.Reply) !void const connections = try reply.arena_alloc().alloc(bson.Pair, 1); connections[0] = .{ .key = "current", .value = .{ .int32 = @intCast(ctx.connection_id) } }; try reply.put("connections", .{ .doc = connections }); + + // Not a MongoDB section, and named so nobody mistakes it for one. It is + // what the churn gate reads: a steady-state size ratio can look healthy + // while reclamation does nothing at all -- the file grows, a rebuild + // periodically halves it, and the average comes out respectable. + // `reclaimedBytes` rising while `allocTail` stays put is the shape that + // says the free list is carrying the workload, and no ratio shows that. + const s = ctx.engine.slab_stats(); + const mf = try reply.arena_alloc().alloc(bson.Pair, 8); + mf[0] = .{ .key = "liveBytes", .value = .{ .int64 = @intCast(s.live_bytes) } }; + mf[1] = .{ .key = "deadBytes", .value = .{ .int64 = @intCast(s.dead_bytes) } }; + mf[2] = .{ .key = "slabBytes", .value = .{ .int64 = @intCast(s.slab_bytes) } }; + mf[3] = .{ .key = "reclaimedBytes", .value = .{ .int64 = @intCast(s.reclaimed_bytes) } }; + mf[4] = .{ .key = "slabRuns", .value = .{ .int64 = @intCast(s.slab_runs) } }; + mf[5] = .{ .key = "freeReadyPages", .value = .{ .int64 = @intCast(s.free_ready_pages) } }; + mf[6] = .{ .key = "allocTail", .value = .{ .int64 = @intCast(s.alloc_tail) } }; + mf[7] = .{ .key = "compactions", .value = .{ .int64 = @intCast(s.compactions) } }; + try reply.put("multifora", .{ .doc = mf }); try reply.put_ok(); } @@ -3008,6 +3026,47 @@ test "ping and hello replies parse" { try testing.expectEqual(@as(i32, 9), bson.get_pair(reply2.pairs.items, "maxWireVersion").?.int32); } +test "serverStatus reports what the slab is doing" { + // The churn gate reads these, and it needs them to be the collections' own + // figures rather than a second opinion about them -- a counter that drifts + // from what the catalog says would make the gate measure the drift. + // + // Mutation check: report the engine's running `live_bytes`/`dead_bytes` + // instead of summing the collections. Not red here, and that is the point: + // it is red in `checkpoint`, where the two are compared, which is why this + // reads the same side of that comparison. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + const io = threaded.io(); + var tdb = try TestDb.init(io); + defer tdb.deinit(); + var ctx = tdb.ctx(io); + + try seed_docs(&tdb, io, "c", 60); + + var reply = wire.Reply.init(testing.allocator); + defer reply.deinit(); + var msg = try parse_fake_msg("serverStatus", .{ .int32 = 1 }, &.{}); + defer msg.deinit(); + try dispatch(&ctx, &msg, &reply); + + const mf = bson.get_pair(reply.pairs.items, "multifora").?.doc; + const live = bson.get_pair(mf, "liveBytes").?.int64; + const dead = bson.get_pair(mf, "deadBytes").?.int64; + const slab = bson.get_pair(mf, "slabBytes").?.int64; + try testing.expect(live > 0); + try testing.expectEqual(slab, live + dead); + // Nothing has died and nothing has been reclaimed yet. + try testing.expectEqual(@as(i64, 0), dead); + try testing.expectEqual(@as(i64, 0), bson.get_pair(mf, "reclaimedBytes").?.int64); + try testing.expectEqual(@as(i64, 0), bson.get_pair(mf, "compactions").?.int64); + try testing.expectEqual(@as(i64, 1), bson.get_pair(mf, "slabRuns").?.int64); + try testing.expect(bson.get_pair(mf, "allocTail").?.int64 > 0); + // Present even at zero: a gate that cannot tell "no pages ready" from + // "field missing" cannot be read at all. + try testing.expect(bson.get_pair(mf, "freeReadyPages") != null); +} + test "the wire version agrees with the version the server calls itself" { // These two are read by different parts of a driver -- the handshake picks // features off the wire version, `runOnRequirements` in the spec suites diff --git a/src/db.zig b/src/db.zig index 9d75958..c6f96c7 100644 --- a/src/db.zig +++ b/src/db.zig @@ -706,6 +706,11 @@ pub const Engine = struct { /// share one tmp path and each ends in a rename onto the log, so two at /// once would publish one compaction's half-written file as the database. compacting: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), + /// Collections rewritten by a rebuild since the process started. Reported by + /// `serverStatus`, because "the ratio improved" and "the ratio improved + /// because reclamation worked rather than because a rebuild ran" are + /// different results and no ratio distinguishes them. Under `counter_lock`. + compactions: u64 = 0, log: storage.Log, /// The data file: documents live here, and the B+tree arenas follow. /// @@ -2017,6 +2022,59 @@ pub const Engine = struct { // would have been invalidated for nothing. self.layout_epoch_seq += 1; coll.layout_epoch = self.layout_epoch_seq; + self.counter_lock.lockUncancelable(self.io); + self.compactions += 1; + self.counter_lock.unlock(self.io); + } + + /// What the slab is doing, for `serverStatus`. Collections under the same + /// catalog-then-collection order everything else uses. + /// + /// It exists because the milestone's own gate cannot be read without it. A + /// steady-state size ratio can look healthy while reclamation does nothing + /// -- the file grows, a rebuild periodically halves it, and the average + /// comes out fine. `reclaimed_bytes` climbing while `alloc_tail` stays put + /// is the shape that says the free list is load-bearing; either one alone + /// says very little. + pub const SlabStats = struct { + live_bytes: u64 = 0, + dead_bytes: u64 = 0, + slab_bytes: u64 = 0, + reclaimed_bytes: u64 = 0, + slab_runs: u64 = 0, + free_ready_pages: u32 = 0, + alloc_tail: u32 = 0, + compactions: u64 = 0, + }; + + pub fn slab_stats(self: *Engine) SlabStats { + var out: SlabStats = .{}; + self.catalog_lock.lockSharedUncancelable(self.io); + var db_it = self.dbs.iterator(); + while (db_it.next()) |db_entry| { + var coll_it = db_entry.value_ptr.collections.iterator(); + while (coll_it.next()) |ce| { + const coll = ce.value_ptr.*; + coll.lock.lockSharedUncancelable(self.io); + defer coll.lock.unlockShared(self.io); + out.live_bytes += coll.live_bytes; + out.slab_bytes += coll.slab_used; + out.reclaimed_bytes += coll.reclaimed_bytes; + out.slab_runs += coll.slab_runs.items.len; + } + } + self.catalog_lock.unlockShared(self.io); + // From the collections rather than the engine's running total, so this + // is the same figure `write_catalog` asserts against rather than a + // second opinion about it. + assert_msg(out.slab_bytes >= out.live_bytes, "the slab cannot hold more live bytes than it has"); + out.dead_bytes = out.slab_bytes - out.live_bytes; + out.free_ready_pages = self.pager.free_ready_pages(); + out.alloc_tail = self.pager.alloc_tail; + self.counter_lock.lockUncancelable(self.io); + out.compactions = self.compactions; + self.counter_lock.unlock(self.io); + return out; } /// The engine's dead-byte total, recomputed from the collections that -- 2.39.5 From 343b25adc8db2cf7e54a0ca049dce7af01c0b809 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 17:43:28 +0300 Subject: [PATCH 32/37] db: a rebuild reclaims before it copies Found by the churn harness, and it is the difference between reclamation working and reclamation being unreachable. A checkpoint is what hands back empty slab windows, and a checkpoint is armed by log volume. A delete logs only an `_id`. So deleting half of a 190 MB collection moved the log by a couple of megabytes, no checkpoint ran, and the garbage sailed straight past the rebuild threshold -- the rebuild got there first every time and reset the window map it would have used. Measured before this: six rounds of delete-and-refill, six rebuilds, 1 MB reclaimed. After: the same six rounds, 256 MB reclaimed. The fix is one line of ordering. `compact` now checkpoints before it walks the collections, so the cheap half of the job runs first: a checkpoint hands back whole windows for the cost of one publish, where a rebuild copies every live byte in the database. The per-collection gate then judges what reclamation left rather than what it was about to take, so a collection whose garbage was all in empty windows is not rewritten at all. No new threshold and no new state -- the gate that decides is the one added in "a rebuild copies only the collections that have garbage", now reading a post-reclamation number. 186/186 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, the full e2e matrix, crash-fuzz 60 cycles. --- src/db.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/db.zig b/src/db.zig index c6f96c7..6cd38a3 100644 --- a/src/db.zig +++ b/src/db.zig @@ -1890,6 +1890,20 @@ pub const Engine = struct { if (self.compacting.swap(true, .acq_rel)) return; defer self.compacting.store(false, .release); + // Reclamation first, because it is the cheap half of the same job: a + // checkpoint hands back whole windows for the cost of one publish, + // where a rebuild copies every live byte in the database. Whatever it + // takes, the per-collection gate below no longer sees, so a collection + // whose garbage was all in empty windows is not rewritten at all. + // + // This is not a refinement, it is what makes reclamation reachable + // under a delete-heavy workload. A checkpoint is otherwise armed by log + // volume, and a delete logs only an `_id` -- so deleting half a 190 MB + // collection moves the log by a couple of megabytes and no checkpoint + // runs, while the garbage sails past the rebuild threshold. Measured + // with the churn harness: six rounds, six rebuilds, 1 MB reclaimed. + try self.checkpoint(); + try self.catalog_lock.lockShared(self.io); var rebuild_err: ?anyerror = null; var db_it = self.dbs.iterator(); -- 2.39.5 From 8f63c6df704cad0ad1583eb8501938fb6f8ee017 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 17:44:05 +0300 Subject: [PATCH 33/37] tests/e2e: the churn gate, as a committed harness D7.4 was the only block in `tests/e2e/results/m0-gates.txt` without a `reproduce:` line. The numbers were real and the harness was not committed, so the one measurement the whole free-list decision rested on could not be re-run against a change. This is that harness. Self-contained like `e2e6.js`: it spawns its own server on a fresh database. Two modes, delete-and-refill and repeated update, over `--docs` documents of `--doc-size`, with `--index` to put index maintenance inside the churn rather than beside it. Three things it does that the ad-hoc version did not: Live bytes are computed here, from the serialized size of one document, rather than read off the server. That is what makes a run against an older binary comparable -- and the first thing this harness was used for was measuring the pre-Stage-3 binary, which has no `multifora` section at all. Deleted ids are sampled from the ids actually live. Sampling blind from the id space re-picks dead ones, so a round deletes fewer documents than it inserts and a supposedly flat-live measurement quietly grows. The first run of this harness ended with 3211 documents where it should have had 2000. And it prints `inUse` beside `ratio`. The data file never shrinks, so `file / live` is a high-water mark and cannot come down however well reclamation works; `inUse` is `(allocTail - freeReady) / live`, which is what the database is actually occupying. On the update line those two read 2.46x and 1.06-1.26x for the same run, and the difference between them is the whole finding. A fixed seed, so two runs churn the same documents in the same order and a difference between them is the code rather than the dice. `--target x` fails the run above a ratio, for use as a gate; without it the harness measures and reports. --- tests/e2e/churn.js | 360 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 tests/e2e/churn.js diff --git a/tests/e2e/churn.js b/tests/e2e/churn.js new file mode 100644 index 0000000..c03a08c --- /dev/null +++ b/tests/e2e/churn.js @@ -0,0 +1,360 @@ +// The churn gate: how large the data file settles at, relative to the live +// data, under sustained rewriting. +// +// This is the measurement PLAN D7.4 was decided on, and until now it was the +// only gate in `tests/e2e/results/m0-gates.txt` without a `reproduce:` line -- +// the numbers were real but the harness was not committed, so nobody could +// re-run them against a change. That is what this file fixes. +// +// It spawns its own server on a fresh database, so it needs nothing running: +// +// node tests/e2e/churn.js --docs 40000 --doc-size 16k --index \ +// --mode delete-refill --rounds 6 +// node tests/e2e/churn.js --docs 40000 --doc-size 16k --index \ +// --mode update --multiple 5 +// +// What to read. The ratio alone does not say whether reclamation is working: +// a file that grows and is periodically halved by a rebuild averages out to a +// respectable number. So every round prints `reclaimed`, `allocTail` and +// `compactions` beside it. Reclamation is carrying the workload when +// `reclaimed` climbs while `allocTail` stays put. If instead `allocTail` grows +// by the full write volume of each round, the free list is decorative however +// good the ratio looks -- and that is a failure even at 1.2x. +// +// Small documents are expected not to improve on the delete-refill line, and +// that is a pass rather than a fault. Reclamation gives back whole system +// pages, so a 16 KiB page holds ~82 documents of 200 bytes and the chance all +// 82 are dead at once is nil. The mechanism to check there is that the +// counters decay correctly, not that the ratio moves. +// +// Options: +// --docs documents in the collection (default 40000) +// --doc-size payload bytes per document (default 16k) +// --index create one secondary index over a churned field +// --mode delete-refill (default) | update +// --rounds delete-refill rounds (default 6) +// --multiple update mode: total writes as a multiple of --docs +// --target fail unless the steady-state ratio is at or under x +// --port listen port (default 27320) +// --keep leave the database file behind +const { MongoClient } = require('mongodb'); +const { spawn } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +function parseSize(s) { + const m = String(s).match(/^(\d+)([kKmMgG]?)$/); + if (!m) throw new Error(`bad size: ${s}`); + const mult = { '': 1, k: 1 << 10, m: 1 << 20, g: 1 << 30 }[m[2].toLowerCase()]; + return Number(m[1]) * mult; +} + +function parseArgs(argv) { + const o = { + docs: 40000, docSize: 16 << 10, index: false, mode: 'delete-refill', + rounds: 6, multiple: 5, target: null, port: 27320, keep: false, + }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + const next = () => { + if (i + 1 >= argv.length) throw new Error(`${a} needs a value`); + return argv[++i]; + }; + switch (a) { + case '--docs': o.docs = Number(next()); break; + case '--doc-size': o.docSize = parseSize(next()); break; + case '--index': o.index = true; break; + case '--mode': o.mode = next(); break; + case '--rounds': o.rounds = Number(next()); break; + case '--multiple': o.multiple = Number(next()); break; + case '--target': o.target = Number(next()); break; + case '--port': o.port = Number(next()); break; + case '--keep': o.keep = true; break; + default: throw new Error(`unknown option ${a}`); + } + } + if (o.mode !== 'delete-refill' && o.mode !== 'update') { + throw new Error(`--mode must be delete-refill or update, got ${o.mode}`); + } + return o; +} + +const opt = parseArgs(process.argv.slice(2)); +const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb'); +const DBFILE = process.env.CHURN_DB || + path.resolve(__dirname, `../../.zig-cache/churn-${opt.port}.log`); +const URL = `mongodb://127.0.0.1:${opt.port}`; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// A fixed seed, so two runs churn the same documents in the same order and a +// difference between them is the code rather than the dice. +let seed = 0x9e3779b9; +function rnd() { + seed ^= seed << 13; seed >>>= 0; + seed ^= seed >> 17; + seed ^= seed << 5; seed >>>= 0; + return seed / 0x100000000; +} +const pick = (n) => Math.floor(rnd() * n); + +let server = null; +let serverDead = false; +let serverLog = ''; + +function cleanup() { + if (server && !serverDead) { + try { server.kill('SIGKILL'); } catch {} + } +} +process.on('exit', cleanup); +process.on('SIGINT', () => { cleanup(); process.exit(130); }); +process.on('SIGTERM', () => { cleanup(); process.exit(143); }); + +function startServer() { + return new Promise((resolve, reject) => { + fs.rmSync(DBFILE, { force: true }); + fs.rmSync(DBFILE + '.data', { force: true }); + serverDead = false; + server = spawn(BIN, ['--port', String(opt.port), '--db', DBFILE], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + server.stdout.on('data', (d) => (serverLog += d)); + server.stderr.on('data', (d) => (serverLog += d)); + server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`))); + server.on('exit', (code, sig) => { + // A child that dies must fail the start, or the poll below would find a + // *stale* server on the same port and measure the wrong database. + serverDead = true; + if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`; + }); + const deadline = Date.now() + 15000; + (async () => { + while (Date.now() < deadline) { + if (serverDead) { + reject(new Error(`server child exited during start (port ${opt.port} busy?)\n${serverLog}`)); + return; + } + const c = new MongoClient(URL, { serverSelectionTimeoutMS: 1000 }); + try { + await c.connect(); + await c.db('admin').command({ ping: 1 }); + await c.close(); + return resolve(); + } catch { + try { await c.close(); } catch {} + await sleep(100); + } + } + reject(new Error(`server did not come up on :${opt.port}\n${serverLog}`)); + })(); + }); +} + +async function stopServer() { + if (!server) return; + const exited = new Promise((r) => server.once('exit', r)); + server.kill('SIGTERM'); + await Promise.race([exited, sleep(5000)]); + serverDead = true; + server = null; +} + +function dataFileBytes() { + try { + return fs.statSync(DBFILE + '.data').size; + } catch { + return 0; + } +} + +const MB = (n) => (n / (1 << 20)).toFixed(1); + +// Live bytes are computed here rather than read off the server, so the ratio +// means the same thing whatever binary is under test. A `multifora` section is +// a recent addition; without it the counters print as n/a and the ratio -- the +// number the gate is actually about -- is still measured, which is what makes +// a run against an older build comparable. +let docBytes = 0; + +async function stats(client, coll) { + const s = await client.db('admin').command({ serverStatus: 1 }); + const m = s.multifora || null; + const num = (v) => Number(v); + const count = await coll.countDocuments({}); + return { + live: count * docBytes, + count, + file: dataFileBytes(), + m: m && { + live: num(m.liveBytes), + dead: num(m.deadBytes), + reclaimed: num(m.reclaimedBytes), + runs: num(m.slabRuns), + freeReady: num(m.freeReadyPages), + allocTail: num(m.allocTail), + compactions: num(m.compactions), + }, + }; +} + +function report(label, s) { + const ratio = s.live > 0 ? s.file / s.live : 0; + let line = ` ${label.padEnd(12)} ratio ${ratio.toFixed(2)}x file ${MB(s.file)}MB live ${MB(s.live)}MB`; + if (s.m) { + // What the database is actually occupying, as opposed to what it has ever + // had to occupy. The file never shrinks, so `ratio` is a high-water mark + // and cannot come down however well reclamation works; `inUse` is the + // number that moves when it does. + const inUse = (s.m.allocTail - s.m.freeReady) * 4096; + line += ` inUse ${(inUse / s.live).toFixed(2)}x` + + ` dead ${MB(s.m.dead)}MB reclaimed ${MB(s.m.reclaimed)}MB` + + ` allocTail ${MB(s.m.allocTail * 4096)}MB freeReady ${MB(s.m.freeReady * 4096)}MB` + + ` runs ${s.m.runs} compactions ${s.m.compactions}`; + } else { + line += ' (no multifora section: counters n/a)'; + } + console.log(line); + return ratio; +} + +// One document of about `opt.docSize` payload bytes. `k` is the field a +// secondary index covers and an update rewrites, so index maintenance is part +// of the churn rather than a constant. +const PAD = 'x'.repeat(Math.max(1, opt.docSize)); +function makeDoc(id) { + return { _id: id, k: id % 1000, pad: PAD }; +} + +// Batches sized so one insertMany stays well under the 48 MB wire limit +// whatever --doc-size is. +function batchSize() { + return Math.max(1, Math.min(1000, Math.floor((8 << 20) / (opt.docSize + 64)))); +} + +async function insertRange(coll, from, to) { + const bs = batchSize(); + for (let i = from; i < to; i += bs) { + const docs = []; + for (let j = i; j < Math.min(i + bs, to); j++) docs.push(makeDoc(j)); + await coll.insertMany(docs, { ordered: false }); + } +} + +async function main() { + console.log( + `churn: ${opt.docs} x ${opt.docSize} B, mode ${opt.mode}` + + `${opt.index ? ', one secondary index' : ''}` + + `${opt.mode === 'delete-refill' ? `, ${opt.rounds} rounds` : `, ${opt.multiple}x writes`}`, + ); + console.log(`churn: platform ${process.platform}/${process.arch}, binary ${BIN}`); + await startServer(); + const client = new MongoClient(URL, { serverSelectionTimeoutMS: 5000 }); + await client.connect(); + const db = client.db('churn'); + const coll = db.collection('c'); + + const t0 = Date.now(); + // The exact serialized size of one document, so `live` is a real byte count + // rather than the payload size the caller asked for. + docBytes = require('mongodb').BSON.serialize(makeDoc(0)).length; + await insertRange(coll, 0, opt.docs); + if (opt.index) await coll.createIndex({ k: 1 }); + const base = await stats(client, coll); + report('loaded', base); + + const ratios = []; + let nextId = opt.docs; + if (opt.mode === 'delete-refill') { + // Half the collection dies and is replaced by fresh documents, so the live + // size is flat and everything the file gains is garbage that was not + // reclaimed. + const half = Math.floor(opt.docs / 2); + // The ids actually live, so a round deletes exactly `half` documents and + // the collection stays the same size. Sampling blind from the id space + // re-picks already-dead ids, which deletes fewer than it inserts and turns + // a flat-live measurement into a growing one. + const live = Array.from({ length: opt.docs }, (_, i) => i); + for (let r = 0; r < opt.rounds; r++) { + const ids = []; + for (let i = 0; i < half; i++) { + const at = pick(live.length); + ids.push(live[at]); + live[at] = live[live.length - 1]; + live.pop(); + } + const bs = 5000; + for (let i = 0; i < ids.length; i += bs) { + await coll.deleteMany({ _id: { $in: ids.slice(i, i + bs) } }); + } + await insertRange(coll, nextId, nextId + ids.length); + for (let i = 0; i < ids.length; i++) live.push(nextId + i); + nextId += ids.length; + ratios.push(report(`round ${r + 1}`, await stats(client, coll))); + } + } else { + // The same documents rewritten over and over: every rewrite leaves the old + // copy behind, and old copies die in insertion order, which is the best + // case for reclaiming whole windows. + const total = opt.docs * opt.multiple; + const per = Math.floor(total / opt.rounds); + for (let r = 0; r < opt.rounds; r++) { + let done = 0; + while (done < per) { + const ops = []; + for (let i = 0; i < Math.min(2000, per - done); i++) { + const id = pick(opt.docs); + ops.push({ updateOne: { filter: { _id: id }, update: { $set: { k: pick(1000) } } } }); + } + await coll.bulkWrite(ops, { ordered: false }); + done += ops.length; + } + ratios.push(report(`round ${r + 1}`, await stats(client, coll))); + } + } + + const final = await stats(client, coll); + const count = final.count; + const elapsed = ((Date.now() - t0) / 1000).toFixed(0); + console.log(`churn: ${count} documents live at the end, ${elapsed}s`); + + // Steady state is the second half of the rounds: the first ones are still + // filling the file out and say nothing about where it settles. + const tail = ratios.slice(Math.floor(ratios.length / 2)); + const steady = tail.reduce((a, b) => a + b, 0) / tail.length; + const drift = tail.length > 1 ? tail[tail.length - 1] - tail[0] : 0; + console.log( + `churn: steady state ${steady.toFixed(2)}x over the last ${tail.length} rounds, ` + + `drift ${drift >= 0 ? '+' : ''}${drift.toFixed(2)}x`, + ); + if (final.m) { + console.log( + `churn: reclaimed ${MB(final.m.reclaimed)}MB total, ` + + `${final.m.compactions} collection rebuilds`, + ); + } + + await client.close(); + if (!opt.keep) { + fs.rmSync(DBFILE, { force: true }); + fs.rmSync(DBFILE + '.data', { force: true }); + } + await stopServer(); + + if (count !== opt.docs) { + console.log(`CHURN_FAIL: ${count} documents live, expected ${opt.docs}`); + process.exit(1); + } + if (opt.target !== null && steady > opt.target) { + console.log(`CHURN_FAIL: steady state ${steady.toFixed(2)}x is above the ${opt.target}x target`); + process.exit(1); + } + console.log('CHURN_OK'); +} + +main().catch((e) => { + console.error('CHURN_FAIL', e); + console.log('--- server log tail ---'); + console.log(serverLog.split('\n').slice(-40).join('\n')); + process.exit(1); +}); -- 2.39.5 From d4927268810cf202f9bf3d0173dc13a72cd3a3b8 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 18:08:00 +0300 Subject: [PATCH 34/37] db/pager: the catalog may not claim a page that is on the free list Reclamation runs as a checkpoint phase, so it frees pages concurrently with everything else -- and the one failure that arrangement can produce is silent. A catalog that claims a page already handed to the pager gets that page back two generations later, written over by somebody else; the crash that falls back to that generation then reads a document which is no longer there. Nothing fails at the time, and the `seq` retry cannot see it because neither a reclamation nor a rebuild appends a log record. So `write_catalog` now asserts it, per run, in test and Debug builds -- three list scans where every run is being walked anyway. It is proven to fire: have `reclaim_windows` free the pages and keep the old run list, and the suite panics on it. This is the double-ownership detector the plan said an enlarged free list deserves. And `checkpoint` takes a lock of its own. Two can be in flight -- a writer's epilogue claims the pending flag while another is inside `compact`, which checkpoints of its own. The publish was always safe, since it runs under `log_lock`; the phase in front of it is new. One checkpoint's `reclaim_slabs` frees pages under a collection's lock that the other's `write_catalog` may already have serialized, and that is exactly the shape above. Stated plainly: the argument for the lock is by construction, and no test reproduces the interleaving -- removing it leaves the new concurrency test green. What that test does do is run reclamation under two checkpointers and a writer with the ownership assertion armed, which is the harness that would catch the argument being wrong. This is the second instance of the shape PLAN records as still open (a rebuild frees pages under only the collection's lock while a checkpoint may have snapshotted a catalog claiming them). Reclamation is now excluded from it; `compact`'s rebuild walk still is not, and that remains recorded rather than fixed here. 187/187 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, the full e2e matrix, crash-fuzz 60 cycles. --- src/db.zig | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/pager.zig | 23 +++++++++++ 2 files changed, 134 insertions(+) diff --git a/src/db.zig b/src/db.zig index 6cd38a3..238b4b1 100644 --- a/src/db.zig +++ b/src/db.zig @@ -765,6 +765,27 @@ pub const Engine = struct { /// A leaf: nothing else is taken while it is held, and it is never held /// across an append, an fsync, or an allocation. counter_lock: std.Io.Mutex = .init, + /// One checkpoint at a time. + /// + /// Two can be in flight without it -- a writer's epilogue claims the + /// pending flag while another writer's epilogue is inside `compact`, which + /// checkpoints of its own. The publish itself was always safe, because it + /// runs under `log_lock`; what is not is the phase in front of it, which is + /// new. `reclaim_slabs` frees pages under a collection's lock, and the + /// other checkpoint's `write_catalog` may already have serialized that + /// collection's runs. It then publishes a catalog claiming pages that are + /// on the free list, and two generations later they are handed out and + /// written over -- so the crash that falls back to that generation reads a + /// document that is no longer there. The `seq` retry cannot see it, because + /// neither a reclamation nor a rebuild appends a log record. + /// + /// The argument is by construction; no test reproduces the interleaving. + /// What would catch it is the ownership assertion in `write_catalog`, which + /// is armed in test and Debug builds and is proven to fire. + /// + /// Taken before `catalog_lock`, so the order is checkpoint -> catalog -> + /// collection, the same descent everything else makes. + checkpoint_lock: std.Io.Mutex = .init, /// The checkpoint's own page promise, for the catalog and free-list pages it /// writes. Separate from any collection's for the same reason those are /// separate from each other. @@ -2335,6 +2356,17 @@ pub const Engine = struct { // no second read path to keep working. try put_u32(gpa, out, @intCast(coll.slab_runs.items.len)); for (coll.slab_runs.items) |r| { + // A catalog that claims a page already on the free list is + // the one failure this whole design can produce silently: + // the page is handed out two generations later, written + // over, and the crash that falls back to this generation + // reads a document that is no longer there. Nothing else + // notices. Checked where every run is walked anyway, in the + // builds that can afford three list scans. + if (builtin.is_test or builtin.mode == .Debug) assert_msg( + !self.pager.owns_freed(r.first, r.pages), + "the catalog claims a slab run that is already on the free list", + ); try put_u32(gpa, out, r.first); try put_u32(gpa, out, r.pages); } @@ -2629,6 +2661,8 @@ pub const Engine = struct { /// compaction has always used. That is the crash-recovery invariant (PLAN /// D6) reduced to an ordering. pub fn checkpoint(self: *Engine) !void { + try self.checkpoint_lock.lock(self.io); + defer self.checkpoint_lock.unlock(self.io); try self.commit(); self.reclaim_slabs(); @@ -4584,6 +4618,83 @@ test "a checkpoint runs alongside writers on several collections" { try testing.expectEqual(docs_sum, engine.live_docs); } +test "checkpoints reclaim under concurrent writers without losing a page" { + // Reclamation is now a checkpoint phase, so it runs concurrently with + // writers and with a second checkpoint -- two are reachable without + // contrivance, since a writer's epilogue can claim the pending flag while + // another is inside `compact`, which checkpoints of its own. + // + // The ownership assertion in `write_catalog` is armed here: no collection + // may claim a run the pager has already been given. That is the one failure + // this design can produce silently, and it is proven to fire -- have + // `reclaim_windows` free the pages and keep the old run list, and this test + // panics on it. + // + // What it does *not* do is reproduce the interleaving `checkpoint_lock` + // exists for; removing that lock leaves this green. The lock is there by + // argument, and this is the harness that would catch the argument being + // wrong. Said plainly rather than left to be assumed. + const gpa = testing.allocator; + var threaded: std.Io.Threaded = .init(gpa, .{}); + defer threaded.deinit(); + const io = threaded.io(); + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + engine.compact_threshold = std.math.maxInt(u64); // no rebuild may intervene + + var done = std.atomic.Value(usize).init(1); + const Worker = struct { + fn writer(e: *Engine, left: *std.atomic.Value(usize), alloc: std.mem.Allocator) error{Canceled}!void { + defer _ = left.fetchSub(1, .release); + for (0..300) |i| { + var doc = make_padded(alloc, @intCast(i), 4000) catch return error.Canceled; + defer doc.deinit(); + { + e.lock_catalog(false) catch return error.Canceled; + defer e.unlock_catalog(false); + const coll = (e.lock_collection("app", "c", true, true) catch + return error.Canceled) orelse return error.Canceled; + defer e.unlock_collection(coll, true); + e.insert("app", "c", &doc, undefined) catch return error.Canceled; + if (i % 3 == 2) _ = e.remove_by_id("app", "c", .{ .int32 = @intCast(i - 1) }) catch + return error.Canceled; + } + e.commit() catch return error.Canceled; + } + } + + fn checkpointer(e: *Engine, left: *std.atomic.Value(usize)) error{Canceled}!void { + while (left.load(.acquire) > 0) e.checkpoint() catch {}; + } + }; + + var group: std.Io.Group = .init; + defer group.cancel(io); + group.async(io, Worker.writer, .{ &engine, &done, gpa }); + group.async(io, Worker.checkpointer, .{ &engine, &done }); + group.async(io, Worker.checkpointer, .{ &engine, &done }); + try group.await(io); + + // Checkpoints actually happened, and the last one published cleanly: the + // watermark the pager loaded is the generation it just wrote. + const generation = engine.pager.generation; + try testing.expect(generation > 1); + try engine.checkpoint(); + try testing.expectEqual(generation + 1, engine.pager.generation); + try testing.expectEqual(generation + 1, engine.pager.loaded.generation); + + // And the accounting the two of them were racing on still adds up. + const coll = engine.get_collection("app", "c").?; + try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); + try testing.expectEqual( + coll.slab_used - coll.live_bytes, + coll.dead_located() + coll.dead_unlocated, + ); +} + test "concurrent readers and writers on a threaded Io" { // Real worker threads: writers hold the exclusive lock, readers the // shared lock. Proves the RwLock split keeps committed writes visible diff --git a/src/pager.zig b/src/pager.zig index aa960c6..d9e96fb 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -1216,6 +1216,29 @@ pub const Pager = struct { try self.free_pending.append(self.gpa, .{ .first = first, .pages = pages }); } + /// Whether any page of `[first, first+pages)` has been handed to the free + /// list. A consumer that still claims one is claiming a page the pager is + /// about to give to somebody else, and the symptom is a document quietly + /// overwritten rather than anything failing -- so this is the detector the + /// enlarged free list deserves, and it is why the free lists are readable + /// from outside at all. + /// + /// Walks three lists, so it is for assertions in test and Debug builds. + pub fn owns_freed(self: *Pager, first: u32, pages: u32) bool { + self.alloc_lock.lockUncancelable(self.io); + defer self.alloc_lock.unlock(self.io); + for ([_][]const Extent{ + self.free_pending.items, + self.free_hold.items, + self.free_ready.items, + }) |list| { + for (list) |e| { + if (first < e.first + e.pages and e.first < first + pages) return true; + } + } + return false; + } + /// Pages available for immediate reuse. pub fn free_ready_pages(self: *const Pager) u32 { var n: u32 = 0; -- 2.39.5 From 1491a474795d21a5707cbcd57a54c06ffea997f3 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 18:08:46 +0300 Subject: [PATCH 35/37] plan/results: the M1 churn numbers Amendment A5 and the `[M1.1]`/`[M1.2]` blocks. What they record is a result with two halves, and the value is in keeping both: The mechanism works. 934 MB reclaimed over the update run, occupancy at 1.06-1.26x its live data, and the counters that say so are in `serverStatus` rather than inferred. The ratio did not move. 1.94x delete-heavy and 2.46x update-heavy, identical to the end-of-Stage-2 binary measured with the same harness. `file / live` is a high-water mark because the data file never shrinks, and the mark is set in the first round by the one thing reclamation cannot avoid: a rebuild needs a whole second copy of the live data before the first can be freed. So ~2x is the floor of a rebuild-based design and no threshold reaches it -- rebuilding earlier lowers the garbage term and nothing else, rebuilding later raises it. The plan said in advance what to do if this happened, which was to write it down rather than tune, and to name incremental compaction through a doc-id-to-offset indirection layer as the successor. Recorded, with its cost: a second copy-on-write B+tree per collection, a second random read on point lookup, and it undoes A3. A second lever is named that the plan had not: 52% of the steady-state file is space the database owns and is not using, so returning it to the filesystem is worth more here than reclaiming harder. It needs the file never to shrink below what the fallback generation references, which is its own crash-safety pass. D7.4's 1.65x for the delete line is corrected to 1.94x, and the correction is the harness rather than a regression -- the same 1.94x comes out of the binary that predates any of this work. The old ad-hoc version sampled ids to delete blindly, which re-picks dead ones, so it deleted fewer documents than it inserted and measured a collection that was quietly growing. The update line reproduces D7.4 exactly, 2.46 against 2.47. 200-byte documents reclaim nothing, exactly as forecast, and the forecast being written down beforehand is what makes that a result instead of a disappointment. 3.93x on both binaries. Also noted, because the number invites misreading: at that document size the two index trees are comparable to the documents themselves and the file is already 2.18x before any churn -- index structure, not slab garbage. --- PLAN.md | 99 ++++++++++++++++++++++++++++++---- tests/e2e/results/m0-gates.txt | 97 +++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 9 deletions(-) diff --git a/PLAN.md b/PLAN.md index 7186101..8c73ab4 100644 --- a/PLAN.md +++ b/PLAN.md @@ -320,6 +320,71 @@ takes the first one's place. That commit is where this needs handling — a pre-flight scan for compare-equal `_id`s, refusing to drop the map silently while any exist — not here. +### Amendment A5 — the doc-level free list, and what it did not fix (amends A2, closes D7.4) + +D7.4 left M0 with a bound rather than a target: 1.65× delete-heavy, 2.47× +update-heavy against a hoped-for ~1.3×, and the stated conclusion that +doc-level free lists were an M1 item. They are built. The mechanism is +measured, it works, and **the steady-state ratio did not move**. Both halves +of that are the amendment. + +**What was built.** A collection's slab carries a dense map of dead bytes per +`map_align` window — two bytes per window, so 2.7 MB for a 21 GB slab — and a +checkpoint hands back every window with nothing live left in it, splitting the +runs around what is kept. The window is the unit because it is the smallest +thing that can be given back at all: `mark_appendable` refuses an unaligned +start and `protect_stable` rounds outwards. Counting is the whole liveness +test, because `evict_doc` removes a document's index entries before marking its +bytes dead, so "no live bytes in this window" and "nothing references these +bytes" are the same statement. Reclamation lives inside `checkpoint` rather +than beside it so that the run split and the `free_pages` become durable under +one `publish`; there is no new record type, no new catalog version and no +replay path. `alloc_slab_run` is a second policy in the same allocator, because +`take_free`'s best fit — which exists to stop one-page copy-on-write requests +dismantling the extents — can never match a request for 2048 pages against +runs that come back a few windows at a time. + +**What it does not fix, and why no threshold reaches it.** The data file never +shrinks, so `file / live` is a high-water mark, and the mark is set once by the +one thing reclamation cannot avoid: a rebuild needs a whole second copy of the +live data before the first can be freed. Live + garbage-at-trigger + copy is +the peak, and it is reached in the first round, before any free pool exists to +build the copy out of. Rebuilding earlier lowers the garbage term and nothing +else; rebuilding later raises it. So ~2× is the floor of a rebuild-based +design, and tuning is the wrong instrument. Measured occupancy tells the other +half of the story: 1.06–1.26× in use against a 2.46× file, with 934 MB +reclaimed over the run. + +**The successor, named here so the next session does not re-derive it.** +Incremental compaction through a doc-id → offset indirection layer, which is +rejected option (b) of the M1 design, promoted. It is the only thing that +removes the second copy: a rebuild becomes a move of one document at a time +with the map updated behind it. The cost is the one that got it rejected — the +map has to be persistent and crash-safe, i.e. a second copy-on-write B+tree per +collection and a second random read on the point-lookup path — and it undoes +A3. That is a milestone, not a knob. Second and cheaper: return free space to +the filesystem, since 52% of the steady-state file is space the database owns +and is not using; it needs the file never to shrink below what the fallback +generation references, which is its own crash-safety design pass. + +**Small documents behave exactly as forecast**, and the forecast being written +down in advance is what makes it a result. 200-byte documents reclaim nothing +at all — a 16 KiB system page holds ~70 of them and they never all die at once +— and the counters show a mechanism correctly doing nothing rather than one +misfiring. The payoff scales as `doc_size / map_align`, so 4 KiB pages read +four times better on the same code. + +**One thing the gate found that the design had not.** A checkpoint is what +reclaims and a checkpoint is armed by log volume, but a delete logs only an +`_id`. Deleting half a 190 MB collection moved the log by a couple of megabytes +so no checkpoint ran, the garbage sailed past the rebuild threshold, and the +rebuild reset the window map it would have used — six rounds, six rebuilds, +1 MB reclaimed. `compact` now checkpoints before it copies, which is also the +right order on its own terms: the cheap half of the job first, and the +per-collection gate judges what reclamation left. Same six rounds, 256 MB. +Numbers and reproduction in `tests/e2e/results/m0-gates.txt` under `[M1.1]` and +`[M1.2]`. + --- ## 3. Milestones and gates @@ -721,6 +786,16 @@ between `compact` and `checkpoint`; it is out of this scope because it wants its own design pass, and because the free list must not add a second instance of the same shape. +*Where this stands after the free list.* It did add a second instance — +reclamation frees pages as a checkpoint phase, and two checkpoints can be in +flight — so that half is closed: `checkpoint` takes a lock of its own. Two +things came out of doing it. The publish was never the exposure, because it +already runs under `log_lock`; and the whole class is now *detectable* rather +than only arguable, because `write_catalog` asserts per run that the pager has +not already been given it, in test and Debug builds. That assertion is proven +to fire. The original instance — `compact`'s rebuild walk against a concurrent +checkpoint — is unchanged and still wants the design pass. + ### The spec runner starts reading `expectEvents` 354 of the 487 cases declare `expectEvents` and the runner read none of them, @@ -823,15 +898,21 @@ has to be its own commit with its own re-recorded scorecard. the anchor rewritten — resuming at it returned updated documents twice, caught by draining a collection being updated underneath. - Still open in M1: the doc-level free list. The eight reclamation bugs above - were cleared first, as preconditions for the free list rather than as work of - their own; command-monitoring (`expectEvents`) landed next, so that what - followed is measured by an instrument no longer known to overstate. - **A prerequisite the free list must honour**, recorded here while it is - still being designed: *an offset that was ever a record start must remain a - record start.* `doc_bytes` reads a `u32` length prefix in place, so an - offset landing mid-record after a re-split is a garbage-length read rather - than a wrong answer — and an offsets cursor holds exactly such offsets. + The doc-level free list is built; see amendment A5 for what it did and did + not achieve, and `[M1.1]`/`[M1.2]` in the results file for the numbers. The + eight reclamation bugs above were cleared first, as preconditions for it + rather than as work of their own; command-monitoring (`expectEvents`) landed + next, so that what followed is measured by an instrument no longer known to + overstate. + **The prerequisite it had to honour** — *an offset that was ever a record + start must remain a record start*, because `doc_bytes` reads a `u32` length + prefix in place and an offsets cursor holds exactly such offsets — is met + structurally rather than by checking: a window is handed back only when every + byte in it is dead, which means every document touching it has already been + through `evict_doc` and out of every index. What remains is the cursor + holding a *saved* offset list, and that is answered the way a rebuild answers + it, by bumping `layout_epoch` when and only when a collection actually gave + something back. - **M1 sessions** — *settled and implemented.* `lsid` is parsed, validated and deliberately acted on in no way; `txnNumber`, `startTransaction` and `autocommit` are refused; `endSessions` validates the array it discards. diff --git a/tests/e2e/results/m0-gates.txt b/tests/e2e/results/m0-gates.txt index 1718d37..8574f0d 100644 --- a/tests/e2e/results/m0-gates.txt +++ b/tests/e2e/results/m0-gates.txt @@ -157,3 +157,100 @@ # not a write`. The scorecard above is the M0 figure and is left as measured; # those two commits took it to 163 pass / 129 fail, and `tests/spec/scorecard.txt` # always holds the current one. + + +# =========================================================================== +# M1 — doc-level free list (PLAN amendment A5) +# =========================================================================== +# +# Same machine, same driver. Server at the M1 commit named per block, +# ReleaseFast. These are the numbers D7.4 said an M1 item owed. + +[M1.1] churn gate — the doc-level free list + reproduce: node tests/e2e/churn.js --docs 40000 --doc-size 16k --index \ + --mode delete-refill --rounds 6 + node tests/e2e/churn.js --docs 40000 --doc-size 16k --index \ + --mode update --multiple 5 + baseline: the same harness against the end-of-Stage-2 binary (e416ad1), + which has no reclamation and no `multifora` section. + + The harness is committed this time (`tests/e2e/churn.js`), which is half the + point of the block: D7.4's numbers were real and unrepeatable. + + Stage 2 M1 target + delete half and refill, 6x 1.94x 1.94x <= 1.45x NOT MET + random $set over 5x the coll. 2.46x 2.46x <= 1.60x NOT MET + Both flat, drift +0.00x over the last three rounds. + + D7.4 recorded 1.65x for the delete line. This harness reads 1.94x for the + *same binary* D7.4 was measured against the descendants of, so that gap is + the harness, not a regression: the ad-hoc version sampled ids to delete + blindly, which re-picks already-dead ids, deletes fewer than it inserts and + measures a collection that is quietly growing. The update line reproduces + D7.4 exactly (2.46 vs 2.47). + + THE RATIO DID NOT MOVE AND THE MECHANISM WORKS. Both are true, and the + counters are what separate them: + + round 6, update line: reclaimed 934.0MB dead 83.9MB + allocTail 1523.5MB freeReady 797.0MB + inUse 1.16x file/live 2.46x + + Reclamation returned 934 MB over the run and the collection is occupying + 1.16x its live data. What 2.46x measures is the data file's high-water mark, + and the file never shrinks. The mark is set once, in round 1, by the one + thing reclamation cannot avoid: a rebuild needs a whole second copy of the + live data before the first copy can be freed. 626 MB live + the garbage + standing at the moment it fires + 626 MB of copy is the number, and it is + reached before any free pool exists to build the copy out of. + + So the floor for a rebuild-based design is ~2x, and no threshold reaches it. + Rebuilding earlier lowers the garbage term and raises nothing; rebuilding + later raises it. The plan anticipated this exact outcome and said what to do + about it, which is to write it down rather than tune: the remaining lever is + incremental compaction -- a doc-id-to-offset indirection layer, so a rebuild + moves documents without a second copy of everything. That is amendment A5's + successor and it is a milestone of its own, not a knob. + + A second lever, cheaper and not attempted: give free space back to the + filesystem. `freeReady` stands at 797 MB with `allocTail` flat, so 52% of the + file is space the database owns and is not using. Returning the tail-adjacent + part of it needs the file never to shrink below what the fallback generation + references, which is a crash-safety argument and its own design pass. + + What did change, and is the reason the mechanism is worth keeping: + + delete-refill, 12k x 16 KiB reclaimed 1 MB -> 256 MB (six rounds) + + The first figure is what reclamation achieved before `compact` was made to + checkpoint before it copies. A checkpoint is what reclaims and a checkpoint + is armed by log volume; a delete logs only an `_id`, so deleting half a + collection moved the log by a couple of megabytes, no checkpoint ran, and the + rebuild got there first every time and reset the window map it would have + used. The harness found that on its first serious run, which is the argument + for committing it. + +[M1.2] churn gate — 200-byte documents + reproduce: node tests/e2e/churn.js --docs 150000 --doc-size 200 --index \ + --mode delete-refill --rounds 4 + Predicted in advance, in the plan, as a pass rather than a fault: + + reclaimed 0.0 MB over four rounds, exactly as forecast. + ratio 3.93x on both the Stage 2 binary and M1 -- identical, flat. + + Reclamation hands back whole system pages. A 16 KiB page on this machine + holds ~70 documents of 200 bytes and the chance that all 70 are dead at once + under uniform deletion is nil, so nothing is ever handed back. The forecast + said the ratio would not improve and the counters would show a mechanism + that correctly does nothing, rather than one that silently misfires; that is + what they show. + + Read the ratio on this line with care. `live` counts document bytes, and at + 200 bytes the two index trees are comparable in size to the documents + themselves -- the file is already 2.18x at load, before any churn. That + overhead is index structure, not slab garbage, and it is not what this gate + is about. + + The payoff of window reclamation scales as doc_size / map_align, so a 4 KiB + system page (x86-64 Linux) reads four times better on the same code. Every + number in this file is Apple Silicon with 16 KiB pages. -- 2.39.5 From 36311d0faa4cee34be21536148e409988570f67a Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 18:09:36 +0300 Subject: [PATCH 36/37] plan: the in-place replace commit, declined on the measurement The plan made it conditional on the update line still being the worst. It is -- but the gate says the peak is set by the first rebuild's second copy, not by write amplification, so generating less garbage moves the number by nothing. That would be a change to the write path, on the stretch that runs after the log record is durable, for no measured gain. Recorded as declined with the reason rather than done because it was on the list. --- PLAN.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/PLAN.md b/PLAN.md index 8c73ab4..f0bcc77 100644 --- a/PLAN.md +++ b/PLAN.md @@ -367,6 +367,15 @@ the filesystem, since 52% of the steady-state file is space the database owns and is not using; it needs the file never to shrink below what the fallback generation references, which is its own crash-safety design pass. +**One planned commit was declined on the measurement.** A same-length replace +could be written in place when the document's bytes are on an unpublished page, +and the plan made it conditional on the update line still being the worst. It +is the worst — but the measurement says why, and it is not write amplification: +the peak is set by the first rebuild's second copy, so generating less garbage +moves the number by nothing. It would buy a write-path change, on the path that +runs after the log record is durable, for no gate movement. Left undone with +the reason, rather than done because it was listed. + **Small documents behave exactly as forecast**, and the forecast being written down in advance is what makes it a result. 200-byte documents reclaim nothing at all — a 16 KiB system page holds ~70 of them and they never all die at once -- 2.39.5 From a748a3d08c936d16c86b7843019c834bee998857 Mon Sep 17 00:00:00 2001 From: "A.Shakhmatov" Date: Sun, 9 Aug 2026 19:08:15 +0300 Subject: [PATCH 37/37] db/pager/tests: cleanup pass over the free list No behaviour is meant to change and the gate confirms it: 1.94x / 679.2 MB reclaimed / 9 rebuilds and 2.46x / 934.0 MB / 13 on the two 16 KiB lines, 0.0 MB on the 200-byte line, all identical to the numbers recorded for them. Deduplication. `pages_for` was written in db.zig and again in pager.zig and twice more inline; there is now one, public, and the two pre-existing copies call it. `pages_per_map_align` replaces three hand-rolled `map_align / page_size`. `SlabRun.window_first` was a stored field that could never legally disagree with `first` and was maintained by hand at two sites -- now a method. `keep_piece` re-derived `SlabRun.window_count` character for character; it calls it. `insert_run` scanned linearly for a position `run_of` binary-searches for, which made loading a fragmented catalog quadratic; both now go through one `run_lower_bound`. Freeing a run's window map was written four times; one helper. The 20% rebuild share was stated in `note_compact` and again in `wants_rebuild`, with a comment arguing at length that they must be the same number -- `worth_rewriting` makes that structural. Efficiency. The identity assert in `reclaim_windows` called `dead_located()`, an O(every window) walk, and `assert_msg` is live in ReleaseFast -- so it doubled the scan the reclamation was about to make (2.75 MB streamed twice per reclaiming checkpoint at the 21 GB the gate targets). `dead_located` is now a maintained counter, the check is O(1) in every build, and the scan cross-checks it while it is there. `SlabRun.full` lets a run with nothing to give be copied without its counters being read at all, so the common case is O(runs) rather than O(windows). The pager's two allocation policies were hand-copying the claim step, and the copy had already lost two of the three preconditions -- `alloc_slab_run` never checked `pages <= reserved_pages`. Both now go through `claim_locked`. `reclaimed_bytes` moves from Collection to Engine, beside `compactions`, which is how it is read and the only place it can be honest: a life-of-the-process total must not lose a dropped collection's share. Both join `Counters`, so `slab_stats` stops opening `counter_lock` by hand. The ownership assertion in `write_catalog` was gated on `is_test or Debug`, a predicate nothing else in the codebase uses, which left the one silent failure this design can produce unchecked in ReleaseSafe. It is now `!= ReleaseFast`, the line `protect_stable` already draws. Measured: no change to the suite's runtime. Altitude. `note_checkpoint` was called from exactly one place, the tail of `upsert` -- so a delete armed no checkpoint by any route, which is why reclamation only ever ran when the *rebuild* trigger fired and the rebuild then reset the window map it would have used. `remove` and the TTL sweep arm one now, next to the `note_compact` calls that were added for the same omission a milestone ago. `compact`'s leading checkpoint stays, demoted in its comment from the mechanism to the local ordering it actually guarantees. serverStatus reports `allocTailBytes`/`freeReadyBytes` instead of page counts, so the harness stops hard-coding 4096 -- the kind of constant this milestone was blindsided by once already. tests/e2e/churn.js: `deleteMany({_id: {$in: [5000 ids]}})` exceeded `index.max_combos`, so the planner refused the index and every delete became a full collection scan re-filtering each document against 5000 members. That was the entire runtime of the harness. One delete spec per id instead: the 40k x 16 KiB gate goes 57 s -> 6 s, and the 150k x 200 B line 483 s -> 3 s, with identical output. Also: the per-round `countDocuments` is gone (the harness knows the count), and the server log is a bounded ring rather than a rope that grows with everything the server ever said. Reverted from the review: reusing one MongoClient across the startup poll. A client whose first connect fails tears its topology down and every later command on it fails identically, so it turns "not up yet" into "never comes up" -- it broke the first run. The reason is now a comment. 187/187 unit tests in ReleaseFast and ReleaseSafe, 83/83 fuzz, e2e 49, e2e2 concurrent 2 and the crash pair, e2e3 16, e2e4 17, e2e6 72, e2e7 86, crash-fuzz 60 cycles. --- src/commands.zig | 10 +- src/db.zig | 395 ++++++++++++++++++++------------- src/index.zig | 2 +- src/pager.zig | 70 +++--- tests/e2e/churn.js | 66 ++++-- tests/e2e/results/m0-gates.txt | 5 +- 6 files changed, 333 insertions(+), 215 deletions(-) diff --git a/src/commands.zig b/src/commands.zig index 90bb041..a19d7fe 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -408,8 +408,8 @@ fn cmd_server_status(ctx: *Context, _: *wire.Message, reply: *wire.Reply) !void mf[2] = .{ .key = "slabBytes", .value = .{ .int64 = @intCast(s.slab_bytes) } }; mf[3] = .{ .key = "reclaimedBytes", .value = .{ .int64 = @intCast(s.reclaimed_bytes) } }; mf[4] = .{ .key = "slabRuns", .value = .{ .int64 = @intCast(s.slab_runs) } }; - mf[5] = .{ .key = "freeReadyPages", .value = .{ .int64 = @intCast(s.free_ready_pages) } }; - mf[6] = .{ .key = "allocTail", .value = .{ .int64 = @intCast(s.alloc_tail) } }; + mf[5] = .{ .key = "freeReadyBytes", .value = .{ .int64 = @intCast(s.free_ready_bytes) } }; + mf[6] = .{ .key = "allocTailBytes", .value = .{ .int64 = @intCast(s.alloc_tail_bytes) } }; mf[7] = .{ .key = "compactions", .value = .{ .int64 = @intCast(s.compactions) } }; try reply.put("multifora", .{ .doc = mf }); try reply.put_ok(); @@ -3061,10 +3061,10 @@ test "serverStatus reports what the slab is doing" { try testing.expectEqual(@as(i64, 0), bson.get_pair(mf, "reclaimedBytes").?.int64); try testing.expectEqual(@as(i64, 0), bson.get_pair(mf, "compactions").?.int64); try testing.expectEqual(@as(i64, 1), bson.get_pair(mf, "slabRuns").?.int64); - try testing.expect(bson.get_pair(mf, "allocTail").?.int64 > 0); - // Present even at zero: a gate that cannot tell "no pages ready" from + try testing.expect(bson.get_pair(mf, "allocTailBytes").?.int64 > 0); + // Present even at zero: a gate that cannot tell "nothing ready" from // "field missing" cannot be read at all. - try testing.expect(bson.get_pair(mf, "freeReadyPages") != null); + try testing.expect(bson.get_pair(mf, "freeReadyBytes") != null); } test "the wire version agrees with the version the server calls itself" { diff --git a/src/db.zig b/src/db.zig index 238b4b1..d80e1aa 100644 --- a/src/db.zig +++ b/src/db.zig @@ -39,11 +39,6 @@ const slab_extent_pages: u32 = (8 * 1024 * 1024) / pgr.page_size; /// off whatever is left of the one before it. const slab_run_min_pages: u32 = slab_extent_pages / 8; -/// Pages a slab allocation of `len` bytes needs at minimum. -fn pages_for(len: usize) u32 { - return @intCast((len + pgr.page_size - 1) / pgr.page_size); -} - const LogKind = enum { upsert, delete, index_create, index_drop }; /// Dead bytes in one `map_align` window. The window is the unit of reclamation @@ -71,14 +66,15 @@ const WindowDead = if (pgr.map_align <= std.math.maxInt(u16)) u16 else u32; const SlabRun = struct { first: u32, pages: u32, - /// The run's start, rounded up to `map_align`: the first offset that begins - /// a whole window. `alloc_pages` works in 4 KiB pages, so a run need not - /// start on a system page. - window_first: u64, /// Dead bytes per window, `dead[i]` covering - /// `[window_first + i*map_align, +map_align)`. `map_align` means the window - /// holds nothing live and can be handed back. + /// `[window_first() + i*map_align, +map_align)`. `map_align` means the + /// window holds nothing live and can be handed back. dead: []WindowDead, + /// How many of those have reached `map_align`, so a reclamation scan can + /// skip a run with nothing to give without touching its counters. Always + /// zero in a run that survived a reclamation, because every full window is + /// taken. + full: u32, /// Windows wholly inside the pages `[first, first+pages)`. The bytes /// outside them -- below `window_first`, and the tail after the last whole @@ -97,13 +93,21 @@ const SlabRun = struct { return @as(u64, self.first) << pgr.page_shift; } + /// The run's start, rounded up to `map_align`: the first offset that begins + /// a whole window. `alloc_pages` works in 4 KiB pages, so a run need not + /// start on a system page. Derived rather than stored, so it cannot + /// disagree with `first`. + fn window_first(self: SlabRun) u64 { + return std.mem.alignForward(u64, self.start(), pgr.map_align); + } + fn end(self: SlabRun) u64 { return (@as(u64, self.first) + self.pages) << pgr.page_shift; } /// One past the last byte covered by a window counter. fn window_end(self: SlabRun) u64 { - return self.window_first + self.dead.len * pgr.map_align; + return self.window_first() + self.dead.len * pgr.map_align; } }; @@ -165,6 +169,13 @@ pub const Collection = struct { /// update. What it costs is only that garbage from before a restart is not /// reclaimed window-wise; it still arms compaction like any other. dead_unlocated: u64, + /// The other half of it: garbage that *is* in a window, i.e. the sum of + /// every run's counters. Maintained rather than summed, so the identity + /// above is an O(1) check on a path that runs in every build -- summing it + /// walked every window of the slab, doubling the scan the reclamation was + /// about to make anyway (1.38M counters at the 21 GB the gate targets). + /// The scan cross-checks it while it is there. + dead_located: u64, /// Windows whose counter has reached `map_align`, i.e. how much there is /// for the next checkpoint to give back. /// @@ -174,11 +185,6 @@ pub const Collection = struct { /// help, small documents on large system pages, is exactly the one that /// would pay that for no return. full_windows: u32, - /// Slab handed back to the pager by window reclamation, cumulative for the - /// life of the process. Purely an observation: it is what distinguishes - /// "the ratio improved because reclamation worked" from "the ratio improved - /// for some other reason", which is the only way to read the churn gate. - reclaimed_bytes: u64, /// Secondary indexes (persisted through the log). Heap-allocated, so an /// `*Index` handed out by `find_index` or `create_index` stays valid when /// a sibling index is dropped. Held by value, `orderedRemove` memmoved the @@ -229,8 +235,8 @@ pub const Collection = struct { .slab_used = 0, .live_bytes = 0, .dead_unlocated = 0, + .dead_located = 0, .full_windows = 0, - .reclaimed_bytes = 0, .hold = .{}, .indexes = .empty, .id_index = undefined, @@ -263,8 +269,7 @@ pub const Collection = struct { const dead = try gpa.alloc(WindowDead, SlabRun.window_count(first, pages)); errdefer gpa.free(dead); @memset(dead, 0); - var at: usize = 0; - while (at < self.slab_runs.items.len and self.slab_runs.items[at].first < first) at += 1; + const at = self.run_lower_bound(first); // A recycled run must not overlap one this collection already owns: // that would be the pager handing out pages twice, and the symptom // would be a document quietly overwritten rather than anything failing. @@ -278,30 +283,39 @@ pub const Collection = struct { try self.slab_runs.insert(gpa, at, .{ .first = first, .pages = pages, - .window_first = std.mem.alignForward(u64, @as(u64, first) << pgr.page_shift, pgr.map_align), .dead = dead, + .full = 0, }); } - /// The run holding `off`, or null if no run does. Binary search, which the - /// sorted list is for: `mark_dead` runs once per evicted document, and a - /// collection with a fragmented slab can own thousands of runs. - fn run_of(self: *const Collection, off: u64) ?usize { - const page: u32 = @intCast(off >> pgr.page_shift); + /// The first run at or past `page`. The one search over the sorted list: + /// `run_of` adds a range test to it and `insert_run` uses it as the + /// insertion point, so the ordering is interpreted in one place. + /// + /// Binary rather than linear, which matters at both ends: `mark_dead` runs + /// once per evicted document, and `read_catalog` inserts every run of a + /// fragmented collection at open. + fn run_lower_bound(self: *const Collection, page: u32) usize { var lo: usize = 0; var hi: usize = self.slab_runs.items.len; while (lo < hi) { const mid = lo + (hi - lo) / 2; - const r = self.slab_runs.items[mid]; - if (page < r.first) { - hi = mid; - } else if (page >= r.first + r.pages) { - lo = mid + 1; - } else { - return mid; - } + if (self.slab_runs.items[mid].first < page) lo = mid + 1 else hi = mid; } - return null; + return lo; + } + + /// The run holding `off`, or null if no run does. + fn run_of(self: *const Collection, off: u64) ?usize { + const page: u32 = @intCast(off >> pgr.page_shift); + // `lower_bound` lands on the run starting at `page` if there is one, + // otherwise on the one after it -- so the candidate is that one or its + // predecessor, and only the predecessor can contain an interior page. + const at = self.run_lower_bound(page); + if (at < self.slab_runs.items.len and self.slab_runs.items[at].first == page) return at; + if (at == 0) return null; + const prev = &self.slab_runs.items[at - 1]; + return if (page < prev.first + prev.pages) at - 1 else null; } /// Record that `[off, off+len)` of slab is garbage. @@ -328,46 +342,50 @@ pub const Collection = struct { // layout -- a stale index entry, which is the failure the layout epoch // exists to prevent. assert_msg(stop <= r.end(), "a dead slab range crosses the end of the run holding it"); + const window_first = r.window_first(); var pos = off; - if (pos < r.window_first) { - const n = @min(stop, r.window_first) - pos; + if (pos < window_first) { + const n = @min(stop, window_first) - pos; self.dead_unlocated += n; pos += n; } const win_end = r.window_end(); while (pos < stop and pos < win_end) { - const w: usize = @intCast((pos - r.window_first) / pgr.map_align); - const w_end = r.window_first + (w + 1) * pgr.map_align; + const w: usize = @intCast((pos - window_first) / pgr.map_align); + const w_end = window_first + (w + 1) * pgr.map_align; const n = @min(stop, w_end) - pos; // A window cannot hold more dead bytes than it has bytes. Tripping // this means the same range was marked twice -- a double eviction, // or a recycled offset marked against the previous owner's map. assert_msg(r.dead[w] + n <= pgr.map_align, "a slab window holds more dead bytes than it has"); - const was_full = r.dead[w] == pgr.map_align; r.dead[w] += @intCast(n); - if (!was_full and r.dead[w] == pgr.map_align) self.full_windows += 1; + if (r.dead[w] == pgr.map_align and n > 0) { + // Just filled: `n > 0` and the bound above mean this cannot be + // a window that was already full. + r.full += 1; + self.full_windows += 1; + } + self.dead_located += n; pos += n; } if (pos < stop) self.dead_unlocated += stop - pos; } - /// Garbage this collection has placed in windows. Walks every window, so it - /// belongs to the reclamation scan and to tests, not to a hot path. - fn dead_located(self: *const Collection) u64 { - var sum: u64 = 0; - for (self.slab_runs.items) |r| { - for (r.dead) |d| sum += d; - } - return sum; - } - - /// Drop the window maps and the run list. The pages themselves are the - /// caller's business -- a drop hands them to the pager, a rebuild has - /// already done so. + /// Drop the window maps and the run list, and forget where the garbage was. + /// The pages themselves are the caller's business -- a drop hands them to + /// the pager, a rebuild has already done so. fn free_runs(self: *Collection, gpa: std.mem.Allocator) void { - for (self.slab_runs.items) |r| gpa.free(r.dead); + free_window_maps(gpa, self.slab_runs.items); self.slab_runs.clearRetainingCapacity(); self.full_windows = 0; + self.dead_located = 0; + } + + /// Release the window map of every run in `runs`. The one place a map is + /// freed, because there are four callers and missing one is a leak nothing + /// would notice. + fn free_window_maps(gpa: std.mem.Allocator, runs: []const SlabRun) void { + for (runs) |r| gpa.free(r.dead); } /// One piece of a run that survives reclamation, with a window map of its @@ -385,17 +403,18 @@ pub const Collection = struct { p0: u32, p1: u32, ) !void { - const wf = std.mem.alignForward(u64, @as(u64, p0) << pgr.page_shift, pgr.map_align); - const we = std.mem.alignBackward(u64, @as(u64, p1) << pgr.page_shift, pgr.map_align); - const count: usize = if (we > wf) @intCast((we - wf) / pgr.map_align) else 0; + const piece: SlabRun = .{ .first = p0, .pages = p1 - p0, .dead = &.{}, .full = 0 }; + const count = SlabRun.window_count(p0, p1 - p0); const dead = try gpa.alloc(WindowDead, count); errdefer gpa.free(dead); // A piece boundary is either the run's own start/end or a window // boundary, so the piece's windows line up with a contiguous stretch of // the original's and the counters can be copied rather than rebuilt. - const base: usize = @intCast((wf - r.window_first) / pgr.map_align); + const base: usize = @intCast((piece.window_first() - r.window_first()) / pgr.map_align); @memcpy(dead, r.dead[base..][0..count]); - try out.append(gpa, .{ .first = p0, .pages = p1 - p0, .window_first = wf, .dead = dead }); + // `full` stays zero: every full window is taken, so what survives has + // none. Asserted by the caller, which knows the whole run's count. + try out.append(gpa, .{ .first = p0, .pages = p1 - p0, .dead = dead, .full = 0 }); } /// Give back every window with nothing live left in it, splitting the runs @@ -415,58 +434,41 @@ pub const Collection = struct { self.slab_used >= self.live_bytes, "a collection cannot hold more live bytes than it ever appended", ); - // The identity, checked where every window is being walked anyway. + // The identity the window map rests on. O(1), because both halves are + // maintained by `mark_dead`; the scan below cross-checks the located + // half against the counters themselves. assert_msg( - self.dead_located() + self.dead_unlocated == self.slab_used - self.live_bytes, + self.dead_located + self.dead_unlocated == self.slab_used - self.live_bytes, "the collection's placed and unplaced garbage must add up to its garbage", ); var out: std.ArrayListUnmanaged(SlabRun) = .empty; errdefer { - for (out.items) |p| gpa.free(p.dead); + free_window_maps(gpa, out.items); out.deinit(gpa); } var give: std.ArrayListUnmanaged(pgr.Extent) = .empty; defer give.deinit(gpa); var freed: u64 = 0; + var full_seen: u32 = 0; for (self.slab_runs.items) |r| { - var keep_from = r.first; - var i: usize = 0; - while (i < r.dead.len) { - if (r.dead[i] != pgr.map_align) { - i += 1; - continue; - } - var j = i + 1; - while (j < r.dead.len and r.dead[j] == pgr.map_align) j += 1; - const from = r.window_first + i * pgr.map_align; - const to = r.window_first + j * pgr.map_align; - // The appender's own extent is off limits, and not by - // filtering: bytes above the cursor have never been written, so - // no window covering them can have reached `map_align` dead. - // Tripping this means a range was marked dead twice. - assert_msg( - to <= self.slab_tail or from >= self.slab_end, - "reclaiming a slab window the append cursor is still walking", - ); - const p_from: u32 = @intCast(from >> pgr.page_shift); - const p_to: u32 = @intCast(to >> pgr.page_shift); - if (p_from > keep_from) try keep_piece(&out, gpa, r, keep_from, p_from); - try give.append(gpa, .{ .first = p_from, .pages = p_to - p_from }); - freed += to - from; - keep_from = p_to; - i = j; - } - if (keep_from < r.first + r.pages) { - try keep_piece(&out, gpa, r, keep_from, r.first + r.pages); + full_seen += r.full; + // The common case, and what `full` is for: a run with nothing to + // give is copied without its counters being looked at. + if (r.full == 0) { + try keep_piece(&out, gpa, r, r.first, r.first + r.pages); + continue; } + freed += try self.reclaim_run(gpa, r, &out, &give); } + // `full_windows` is the gate that decides whether a collection is + // scanned at all, so a drift low silently stops reclaiming it -- and the + // symptom would be "the ratio did not move", which is the one conclusion + // this milestone had to work hardest to tell apart from a real result. + assert_msg(full_seen == self.full_windows, "the collection's full-window count disagrees with its runs"); if (freed == 0) { - for (out.items) |p| gpa.free(p.dead); + free_window_maps(gpa, out.items); out.deinit(gpa); - // Every full window was given back or there were none, so nothing - // is left for the next checkpoint to find. - self.full_windows = 0; return 0; } @@ -474,14 +476,58 @@ pub const Collection = struct { // over. A `free_pages` that fails here leaks the run -- it is no longer // the collection's and not yet the pager's -- which costs space and // nothing else. The other order would leave the same pages owned twice. - for (self.slab_runs.items) |r| gpa.free(r.dead); + free_window_maps(gpa, self.slab_runs.items); self.slab_runs.deinit(gpa); self.slab_runs = out; + // Every full window was given back, so nothing is left for the next + // checkpoint to find. self.full_windows = 0; for (give.items) |e| self.pager.free_pages(e.first, e.pages) catch {}; assert_msg(self.slab_used >= self.live_bytes + freed, "reclaiming more slab than the collection has"); self.slab_used -= freed; - self.reclaimed_bytes += freed; + self.dead_located -= freed; + return freed; + } + + /// The full-window stretches of one run: `give` collects the extents handed + /// back, `out` the pieces that survive around them. Returns the bytes freed. + fn reclaim_run( + self: *const Collection, + gpa: std.mem.Allocator, + r: SlabRun, + out: *std.ArrayListUnmanaged(SlabRun), + give: *std.ArrayListUnmanaged(pgr.Extent), + ) !u64 { + const window_first = r.window_first(); + var freed: u64 = 0; + var keep_from = r.first; + var i: usize = 0; + while (i < r.dead.len) { + if (r.dead[i] != pgr.map_align) { + i += 1; + continue; + } + var j = i + 1; + while (j < r.dead.len and r.dead[j] == pgr.map_align) j += 1; + const from = window_first + i * pgr.map_align; + const to = window_first + j * pgr.map_align; + // The appender's own extent is off limits, and not by filtering: + // bytes above the cursor have never been written, so no window + // covering them can have reached `map_align` dead. Tripping this + // means a range was marked dead twice. + assert_msg( + to <= self.slab_tail or from >= self.slab_end, + "reclaiming a slab window the append cursor is still walking", + ); + const p_from: u32 = @intCast(from >> pgr.page_shift); + const p_to: u32 = @intCast(to >> pgr.page_shift); + if (p_from > keep_from) try keep_piece(out, gpa, r, keep_from, p_from); + try give.append(gpa, .{ .first = p_from, .pages = p_to - p_from }); + freed += to - from; + keep_from = p_to; + i = j; + } + if (keep_from < r.first + r.pages) try keep_piece(out, gpa, r, keep_from, r.first + r.pages); return freed; } @@ -543,7 +589,7 @@ pub const Collection = struct { const skipped = self.note_skip(self.slab_end - self.slab_tail); // A document larger than the standard extent gets one of its own; BSON // reaches 16 MB and the extent is 8 MiB. - const want_pages: u32 = @max(slab_extent_pages, pages_for(len)); + const want_pages: u32 = @max(slab_extent_pages, pgr.pages_for(len)); try self.pager.reserve_pages(&self.hold, want_pages); // Off the free list first, or window reclamation is decorative: the // pages come back, nothing asks for them in a shape they arrive in, and @@ -551,7 +597,7 @@ pub const Collection = struct { // because a shorter extent is exhausted after a handful of documents // and every exhaustion abandons what is left of it -- and because the // floor is what makes trimming a larger run harmless. - const min_pages: u32 = @min(want_pages, @max(pages_for(len), slab_run_min_pages)); + const min_pages: u32 = @min(want_pages, @max(pgr.pages_for(len), slab_run_min_pages)); const run = self.pager.alloc_slab_run(&self.hold, min_pages, want_pages) orelse pgr.Extent{ .first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages), .pages = want_pages, @@ -705,12 +751,25 @@ pub const Engine = struct { /// Set while a compaction runs, so only one runs at a time. Compactions /// share one tmp path and each ends in a rename onto the log, so two at /// once would publish one compaction's half-written file as the database. + /// + /// A loser here *skips*, where a loser on `checkpoint_lock` waits, and the + /// difference is what the caller is owed. A compaction is idempotent and + /// its garbage keeps, so the winner's pass covers the loser's reason for + /// asking. A checkpoint is a postcondition its caller depends on -- most of + /// all `compact`, which needs reclamation to have run before it copies -- + /// so skipping one would silently break that. compacting: std.atomic.Value(bool) = std.atomic.Value(bool).init(false), - /// Collections rewritten by a rebuild since the process started. Reported by - /// `serverStatus`, because "the ratio improved" and "the ratio improved - /// because reclamation worked rather than because a rebuild ran" are - /// different results and no ratio distinguishes them. Under `counter_lock`. + /// Collections rewritten by a rebuild since the process started, and slab + /// handed back by window reclamation. Reported by `serverStatus`, because + /// "the ratio improved" and "the ratio improved because reclamation worked + /// rather than because a rebuild ran" are different results and no ratio + /// distinguishes them. Both under `counter_lock`. + /// + /// On the engine rather than per collection, which is how they are read and + /// also the only place they can be honest: a dropped collection's share of + /// a life-of-the-process total must not vanish with it. compactions: u64 = 0, + reclaimed_bytes: u64 = 0, log: storage.Log, /// The data file: documents live here, and the B+tree arenas follow. /// @@ -939,7 +998,14 @@ pub const Engine = struct { /// about a *relation* -- the compaction trigger about the ratio of two of /// them, the checkpoint about how they compare to the sum over collections /// -- so reading them one at a time would be comparing two moments. - const Counters = struct { live_docs: u64, dead_docs: u64, live_bytes: u64, dead_bytes: u64 }; + const Counters = struct { + live_docs: u64, + dead_docs: u64, + live_bytes: u64, + dead_bytes: u64, + compactions: u64, + reclaimed_bytes: u64, + }; fn counters(self: *Engine) Counters { self.counter_lock.lockUncancelable(self.io); @@ -949,6 +1015,8 @@ pub const Engine = struct { .dead_docs = self.dead_docs, .live_bytes = self.live_bytes, .dead_bytes = self.dead_bytes, + .compactions = self.compactions, + .reclaimed_bytes = self.reclaimed_bytes, }; } @@ -1501,6 +1569,13 @@ pub const Engine = struct { // Deletes grow the log too. Without this a delete-heavy workload // never compacts, because only upsert and ttl_sweep used to check. self.note_compact(); + // And the same omission a second time, found by the churn harness: a + // checkpoint is what reclaims empty slab windows, and only `upsert` + // armed one. So deleting half a collection reclaimed nothing until the + // *rebuild* trigger fired, and the rebuild then reset the window map it + // would have used -- six rounds of delete-and-refill, six rebuilds, 1 MB + // reclaimed. Armed where the garbage is made, it is 256 MB. + self.note_checkpoint(); return true; } @@ -1649,9 +1724,13 @@ pub const Engine = struct { deleted += try self.ttl_sweep_coll(coll, now_ms, db_entry.key_ptr.*, coll_entry.key_ptr.*); } } - // A TTL-only workload never reaches the threshold check in `upsert`, - // so the log would otherwise grow without bound. - if (deleted > 0) self.note_compact(); + // A TTL-only workload never reaches the threshold checks in `upsert`, + // so the log would otherwise grow without bound and nothing would ever + // reclaim the slab the expired documents left behind. + if (deleted > 0) { + self.note_compact(); + self.note_checkpoint(); + } return deleted; } @@ -1852,12 +1931,9 @@ pub const Engine = struct { // Absolute volume first: a rewrite costs a full copy of the live data, // so it is not worth doing for a few kilobytes however bad the ratio. if (c.dead_bytes < self.compact_threshold) return; - // Then the share, dead / (live + dead), firing at ~20%: the file stays - // near 1.25x the live data and each rebuild is paid for by the space it - // reclaims. Bytes rather than document counts, because a rewrite copies - // bytes -- 100k evicted 40 B documents are not worth the same rebuild as - // 100k evicted 16 KiB ones. - if (c.dead_bytes * 4 < c.live_bytes) return; + // Then the share: the file stays near 1.25x the live data and each + // rebuild is paid for by the space it reclaims. + if (!worth_rewriting(c.dead_bytes, c.live_bytes)) return; self.compact_pending.store(true, .release); } @@ -1917,12 +1993,12 @@ pub const Engine = struct { // takes, the per-collection gate below no longer sees, so a collection // whose garbage was all in empty windows is not rewritten at all. // - // This is not a refinement, it is what makes reclamation reachable - // under a delete-heavy workload. A checkpoint is otherwise armed by log - // volume, and a delete logs only an `_id` -- so deleting half a 190 MB - // collection moves the log by a couple of megabytes and no checkpoint - // runs, while the garbage sails past the rebuild threshold. Measured - // with the churn harness: six rounds, six rebuilds, 1 MB reclaimed. + // Belt and braces rather than the mechanism: `remove` and the TTL sweep + // arm a checkpoint of their own, so reclamation runs on the cadence of + // garbage production rather than waiting for the rebuild trigger. What + // this guarantees is only the local ordering -- that whatever the + // collections are about to be judged on has already had the cheap half + // applied to it. try self.checkpoint(); try self.catalog_lock.lockShared(self.io); @@ -1991,9 +2067,19 @@ pub const Engine = struct { coll.slab_used >= coll.live_bytes, "a collection cannot hold more live bytes than it ever appended", ); - const dead = coll.slab_used - coll.live_bytes; - if (dead == 0) return false; - return dead * 4 >= coll.live_bytes; + return worth_rewriting(coll.slab_used - coll.live_bytes, coll.live_bytes); + } + + /// dead / (live + dead) at or above ~20%. The one share both the engine's + /// trigger and the per-collection gate apply, written once so the argument + /// above -- that a compaction which runs always rebuilds something -- is + /// enforced by construction rather than by two literals agreeing. + /// + /// Bytes rather than document counts, because a rewrite copies bytes: 100k + /// evicted 40 B documents are not worth the same rebuild as 100k evicted + /// 16 KiB ones. + fn worth_rewriting(dead: u64, live: u64) bool { + return dead > 0 and dead * 4 >= live; } /// Copy one collection's live documents into fresh extents and rebuild every @@ -2077,8 +2163,11 @@ pub const Engine = struct { slab_bytes: u64 = 0, reclaimed_bytes: u64 = 0, slab_runs: u64 = 0, - free_ready_pages: u32 = 0, - alloc_tail: u32 = 0, + /// Bytes, like every other figure here. Pages would make a reader + /// outside the process supply `page_size` from somewhere, and the + /// harness that reads this is a Node script. + free_ready_bytes: u64 = 0, + alloc_tail_bytes: u64 = 0, compactions: u64 = 0, }; @@ -2094,7 +2183,6 @@ pub const Engine = struct { defer coll.lock.unlockShared(self.io); out.live_bytes += coll.live_bytes; out.slab_bytes += coll.slab_used; - out.reclaimed_bytes += coll.reclaimed_bytes; out.slab_runs += coll.slab_runs.items.len; } } @@ -2104,11 +2192,11 @@ pub const Engine = struct { // second opinion about it. assert_msg(out.slab_bytes >= out.live_bytes, "the slab cannot hold more live bytes than it has"); out.dead_bytes = out.slab_bytes - out.live_bytes; - out.free_ready_pages = self.pager.free_ready_pages(); - out.alloc_tail = self.pager.alloc_tail; - self.counter_lock.lockUncancelable(self.io); - out.compactions = self.compactions; - self.counter_lock.unlock(self.io); + out.free_ready_bytes = @as(u64, self.pager.free_ready_pages()) << pgr.page_shift; + out.alloc_tail_bytes = self.pager.allocated_bytes(); + const c = self.counters(); + out.compactions = c.compactions; + out.reclaimed_bytes = c.reclaimed_bytes; return out; } @@ -2363,7 +2451,10 @@ pub const Engine = struct { // reads a document that is no longer there. Nothing else // notices. Checked where every run is walked anyway, in the // builds that can afford three list scans. - if (builtin.is_test or builtin.mode == .Debug) assert_msg( + // Off in ReleaseFast only, the same line `protect_stable` + // draws: the check is O(runs x free list) and the builds + // that do the checking are the ones that pay for it. + if (builtin.mode != .ReleaseFast) assert_msg( !self.pager.owns_freed(r.first, r.pages), "the catalog claims a slab run that is already on the free list", ); @@ -2638,6 +2729,7 @@ pub const Engine = struct { self.counter_lock.lockUncancelable(self.io); assert_msg(self.dead_bytes >= freed, "reclaiming more slab than the engine counts as dead"); self.dead_bytes -= freed; + self.reclaimed_bytes += freed; self.counter_lock.unlock(self.io); // A cursor holding slab offsets is now holding some that name pages // this collection no longer owns -- and reading them would succeed, @@ -3414,7 +3506,7 @@ fn make_padded(gpa: std.mem.Allocator, id: i32, size: usize) !bson.Document { /// Every dead byte the collection knows about, placed or not. fn dead_total(coll: *const Collection) u64 { - return coll.dead_located() + coll.dead_unlocated; + return coll.dead_located + coll.dead_unlocated; } test "every dead slab byte is counted in exactly one place" { @@ -3480,7 +3572,7 @@ test "every dead slab byte is counted in exactly one place" { try testing.expectEqual(coll.slab_used - coll.live_bytes, dead_total(coll)); // And the evicted bytes are mostly placeable: 6000-byte documents are far // smaller than a window, so they fall inside one rather than off its edge. - try testing.expect(coll.dead_located() > coll.dead_unlocated); + try testing.expect(coll.dead_located > coll.dead_unlocated); } test "dead bytes outside a whole window are counted but not placed" { @@ -3527,18 +3619,18 @@ test "dead bytes outside a whole window are counted but not placed" { const ri = coll.run_of(@as(u64, aligned + 1) << pgr.page_shift).?; const r = coll.slab_runs.items[ri]; try testing.expectEqual(@as(usize, 2), r.dead.len); - try testing.expect(r.window_first > r.start()); + try testing.expect(r.window_first() > r.start()); try testing.expect(r.window_end() < r.end()); const before = coll.dead_unlocated; // The head, one whole window, and the tail. - coll.mark_dead(r.start(), r.window_first - r.start()); - coll.mark_dead(r.window_first, pgr.map_align); + coll.mark_dead(r.start(), r.window_first() - r.start()); + coll.mark_dead(r.window_first(), pgr.map_align); coll.mark_dead(r.window_end(), r.end() - r.window_end()); - try testing.expectEqual(@as(u64, pgr.map_align), coll.dead_located()); + try testing.expectEqual(@as(u64, pgr.map_align), coll.dead_located); try testing.expectEqual( - before + (r.window_first - r.start()) + (r.end() - r.window_end()), + before + (r.window_first() - r.start()) + (r.end() - r.window_end()), coll.dead_unlocated, ); // The whole window is full and the one beside it untouched: the head and @@ -3675,7 +3767,7 @@ test "a restart forgets where the garbage is, not that there is any" { } try engine.commit(); const coll = engine.get_collection("app", "c").?; - try testing.expect(coll.dead_located() > 0); + try testing.expect(coll.dead_located > 0); dead_before = coll.slab_used - coll.live_bytes; try engine.checkpoint(); } @@ -3684,7 +3776,7 @@ test "a restart forgets where the garbage is, not that there is any" { defer engine2.deinit(); const coll = engine2.get_collection("app", "c").?; try testing.expectEqual(dead_before, coll.slab_used - coll.live_bytes); - try testing.expectEqual(@as(u64, 0), coll.dead_located()); + try testing.expectEqual(@as(u64, 0), coll.dead_located); try testing.expectEqual(dead_before, coll.dead_unlocated); try testing.expectEqual(dead_before, dead_total(coll)); // The runs came back too, and in a shape `run_of` can use. @@ -3748,7 +3840,7 @@ test "a slab window with one live document in it is never given back" { try engine.checkpoint(); // Most of the slab went back... - try testing.expect(coll.reclaimed_bytes > 100 * 2000); + try testing.expect(engine.reclaimed_bytes > 100 * 2000); // ...but not the window the survivor is in, and it still reads. try testing.expect(coll.run_of(survivor) != null); try testing.expect(std.mem.indexOf(u8, coll.doc_bytes(survivor), "xxxx") != null); @@ -3756,7 +3848,7 @@ test "a slab window with one live document in it is never given back" { try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); try testing.expectEqual( coll.slab_used - coll.live_bytes, - coll.dead_located() + coll.dead_unlocated, + coll.dead_located + coll.dead_unlocated, ); // And once the survivor is gone, its window goes too. @@ -3803,11 +3895,11 @@ test "a reclaimed slab window is not reusable until two publishes later" { while (i < 200) : (i += 1) _ = try engine.remove_by_id("app", "c", .{ .int32 = i }); try engine.commit(); try engine.checkpoint(); - try testing.expect(coll.reclaimed_bytes > 0); + try testing.expect(engine.reclaimed_bytes > 0); // A page from the first window given back: the run's first window is all // dead now, so its first page is no longer the collection's. - const gone = @as(u32, @intCast(owned_before.window_first >> pgr.page_shift)); + const gone = @as(u32, @intCast(owned_before.window_first() >> pgr.page_shift)); try testing.expect(coll.run_of(@as(u64, gone) << pgr.page_shift) == null); try testing.expect(gone >= owned_before.first); @@ -3877,8 +3969,7 @@ test "a churning collection reuses its slab instead of growing the file" { if (round == 0) tail_after_first = engine.pager.alloc_tail; } - const coll = engine.get_collection("app", "c").?; - try testing.expect(coll.reclaimed_bytes > 0); + try testing.expect(engine.reclaimed_bytes > 0); // Three more rounds of the same volume after the first. Anything left is // fragmentation the windows could not cover, not the write volume. const grew = engine.pager.alloc_tail - tail_after_first; @@ -4256,8 +4347,8 @@ test "the slab counts what the appender skips" { // the live documents below it, and the bytes of the run outside any whole // window. try engine.checkpoint(); - try testing.expect(coll.reclaimed_bytes > 4 * 1024 * 1024); - try testing.expectEqual(gap + abandoned - coll.reclaimed_bytes, engine.dead_bytes); + try testing.expect(engine.reclaimed_bytes > 4 * 1024 * 1024); + try testing.expectEqual(gap + abandoned - engine.reclaimed_bytes, engine.dead_bytes); try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); try testing.expect(engine.dead_bytes < gap + 2 * pgr.map_align); } @@ -4314,7 +4405,7 @@ test "a rebuild leaves behind what its own copying skipped" { // assert but the same statement: the counter is *not* zeroed, and it equals // what the collection actually has. try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); - try testing.expect(coll.reclaimed_bytes > 2 * 1024 * 1024); + try testing.expect(engine.reclaimed_bytes > 2 * 1024 * 1024); try testing.expect(engine.dead_bytes > 0); try testing.expect(engine.dead_bytes < 4 * pgr.map_align); } @@ -4691,7 +4782,7 @@ test "checkpoints reclaim under concurrent writers without losing a page" { try testing.expectEqual(coll.slab_used - coll.live_bytes, engine.dead_bytes); try testing.expectEqual( coll.slab_used - coll.live_bytes, - coll.dead_located() + coll.dead_unlocated, + coll.dead_located + coll.dead_unlocated, ); } @@ -5832,7 +5923,7 @@ test "the epochs that invalidate a cursor move exactly when they must" { try engine.commit(); try engine.checkpoint(); const after_reclaim = engine.get_collection("app", "c").?.layout_epoch; - try testing.expect(engine.get_collection("app", "c").?.reclaimed_bytes > 0); + try testing.expect(engine.reclaimed_bytes > 0); try testing.expect(after_reclaim != quiet_before); // And the index-level token, which guards the position hint. diff --git a/src/index.zig b/src/index.zig index a06897e..0c3f93f 100644 --- a/src/index.zig +++ b/src/index.zig @@ -491,7 +491,7 @@ pub const Index = struct { // 16 MB). const want_pages: u32 = @intCast(@max( ovf_extent_pages, - (overflow_bytes + pgr.page_size - 1) / pgr.page_size, + pgr.pages_for(overflow_bytes), )); try self.pager.reserve_pages(&self.hold, want_pages); const first = self.pager.alloc_pages_assume_reserved(&self.hold, want_pages); diff --git a/src/pager.zig b/src/pager.zig index d9e96fb..f95ef7e 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -108,6 +108,10 @@ const header_hashed_len: usize = 24; /// cursors up to this rather than to page_size. pub const map_align = std.heap.page_size_min; +/// Logical pages per system page. At least one: the comptime block below only +/// requires one of the two sizes to divide the other. +pub const pages_per_map_align: u32 = @max(1, map_align / page_size); + /// Growth granularity. Large enough that growth is rare and each `setLength` /// covers many allocations, and a multiple of every supported system page size. const grow_chunk_pages: u32 = 2048; // 8 MiB @@ -135,8 +139,9 @@ pub const Extent = struct { /// What a checkpoint publishes. Everything here is authoritative except the /// two cached counters, which are hints the engine recomputes if they look /// wrong. -/// Pages a stream of `len` bytes occupies. -fn pages_for(len: u64) u32 { +/// Pages a run of `len` bytes occupies, rounded up. The one place the +/// partial-page rule is written. +pub fn pages_for(len: u64) u32 { return @intCast((len + page_size - 1) / page_size); } @@ -700,7 +705,6 @@ pub const Pager = struct { } else { self.free_ready.items[i] = .{ .first = e.first + n, .pages = e.pages - n }; } - self.unprotect(first, n); return first; } @@ -736,19 +740,19 @@ pub const Pager = struct { assert(min_pages <= max_pages); self.alloc_lock.lockUncancelable(self.io); defer self.alloc_lock.unlock(self.io); - assert_msg(max_pages <= hold.pages, "a slab run request overran reserve_pages' promise"); + // A split can leave a piece at each end, so one entry may become two. // Out of memory before anything is disturbed: the caller falls back to // bumping the tail, which is what it would have done anyway. self.free_ready.ensureUnusedCapacity(self.gpa, 1) catch return null; - const spp: u32 = if (map_align >= page_size) @intCast(map_align / page_size) else 1; var best: ?usize = null; var best_first: u32 = 0; var best_take: u32 = 0; + var best_src: u32 = 0; for (self.free_ready.items, 0..) |e, i| { - const from = std.mem.alignForward(u32, e.first, spp); - const to = std.mem.alignBackward(u32, e.first + e.pages, spp); + const from = std.mem.alignForward(u32, e.first, pages_per_map_align); + const to = std.mem.alignBackward(u32, e.first + e.pages, pages_per_map_align); if (to <= from) continue; const usable = to - from; if (usable < min_pages) continue; @@ -756,16 +760,14 @@ pub const Pager = struct { // The longest run available, so the collection switches extents as // rarely as possible -- every switch abandons what is left of the // one before it. Ties go to the smallest source run, which leaves - // the big ones as whole as it can. - const better = if (best) |b| - take > best_take or - (take == best_take and e.pages < self.free_ready.items[b].pages) - else - true; - if (better) { + // the big ones as whole as it can. `best_take` starts at zero and + // every candidate takes at least `min_pages`, so "nothing yet" is + // already encoded. + if (take > best_take or (take == best_take and e.pages < best_src)) { best = i; best_first = from; best_take = take; + best_src = e.pages; } } const i = best orelse return null; @@ -781,10 +783,7 @@ pub const Pager = struct { } else { _ = self.free_ready.swapRemove(i); } - self.reserved_pages -= best_take; - hold.pages -= best_take; - self.unprotect(best_first, best_take); - self.mark_unpublished(best_first, best_take); + self.claim_locked(hold, best_first, best_take); return .{ .first = best_first, .pages = best_take }; } @@ -823,33 +822,38 @@ pub const Pager = struct { /// For callers already holding `alloc_lock`; see `reserve_pages_locked`. fn alloc_assume_reserved_locked(self: *Pager, hold: *Reservation, n: u32) u32 { assert(n > 0); - assert_msg( - n <= hold.pages, - "page allocation overran reserve_pages' promise", - ); - assert_msg( - n <= self.reserved_pages, - "page allocation overran the pager's total promise", - ); assert_msg( self.alloc_tail + n <= self.mapped_pages, "page allocation past the mapped end of the data file", ); - self.reserved_pages -= n; - hold.pages -= n; // Reuse before growing. Without this the free list is decorative and the // file grows without bound under churn, because copy-on-write abandons // every page it touches in every generation (PLAN amendment A2). if (self.take_free(n)) |recycled| { - self.mark_unpublished(recycled, n); + self.claim_locked(hold, recycled, n); return recycled; } const first = self.alloc_tail; self.alloc_tail += n; - self.mark_unpublished(first, n); + self.claim_locked(hold, first, n); return first; } + /// Charge `[first, first+pages)` against the reservation and make it + /// writable. The one place a claim is booked, because there are two + /// allocation policies above it and hand-copying this is how they drift -- + /// the copy in `alloc_slab_run` had already lost one of the preconditions. + fn claim_locked(self: *Pager, hold: *Reservation, first: u32, pages: u32) void { + assert_msg(pages <= hold.pages, "page allocation overran reserve_pages' promise"); + assert_msg(pages <= self.reserved_pages, "page allocation overran the pager's total promise"); + self.reserved_pages -= pages; + hold.pages -= pages; + // A recycled page was inside a published image once, so its protection + // has to be lifted before it is handed out again. + self.unprotect(first, pages); + self.mark_unpublished(first, pages); + } + fn mark_unpublished(self: *Pager, first: u32, n: u32) void { // `grow_to` sizes the set to the mapping, and `reserve_pages` has already // grown the mapping past this run, so the range is in bounds. @@ -963,7 +967,7 @@ pub const Pager = struct { // where the chunk stops being `grow_chunk_pages`. const chunk = @max(grow_chunk_pages, self.mapped_pages / 8); var new_pages = std.mem.alignForwardAnyAlign(u32, want_pages, chunk); - const sys_pages: u32 = @intCast(map_align / page_size); + const sys_pages: u32 = pages_per_map_align; if (sys_pages > 1) new_pages = std.mem.alignForward(u32, new_pages, sys_pages); if (@as(u64, new_pages) << page_shift > self.reserve.len) { @@ -1271,7 +1275,7 @@ pub const Pager = struct { self.alloc_lock.lockUncancelable(self.io); defer self.alloc_lock.unlock(self.io); const bound = self.free_ready.items.len + self.free_hold.items.len + self.free_pending.items.len; - const pages: u32 = @intCast((8 + bound * 8 + 8 + page_size - 1) / page_size); + const pages: u32 = pages_for(8 + bound * 8 + 8); var hold: Reservation = .{}; try self.reserve_pages_locked(&hold, pages); const first = self.alloc_assume_reserved_locked(&hold, pages); @@ -2097,7 +2101,7 @@ test "a slab run comes off the free list aligned, or not at all" { var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const pg = tp.pg(); - const spp: u32 = @intCast(map_align / page_size); + const spp: u32 = pages_per_map_align; // A long run deliberately starting one 4 KiB page past a boundary, and a // short one, kept apart so coalescing cannot merge them. diff --git a/tests/e2e/churn.js b/tests/e2e/churn.js index c03a08c..7142453 100644 --- a/tests/e2e/churn.js +++ b/tests/e2e/churn.js @@ -32,7 +32,8 @@ // --doc-size payload bytes per document (default 16k) // --index create one secondary index over a churned field // --mode delete-refill (default) | update -// --rounds delete-refill rounds (default 6) +// --rounds rounds in each mode; update mode splits its writes +// across them (default 6) // --multiple update mode: total writes as a multiple of --docs // --target fail unless the steady-state ratio is at or under x // --port listen port (default 27320) @@ -100,7 +101,13 @@ const pick = (n) => Math.floor(rnd() * n); let server = null; let serverDead = false; +// Bounded: the listeners below run for the life of the process and only the +// last few lines are ever read, so an unbounded string would hold a rope +// proportional to everything the server ever said. let serverLog = ''; +const noteServerLog = (d) => { + serverLog = (serverLog + d).slice(-65536); +}; function cleanup() { if (server && !serverDead) { @@ -119,14 +126,14 @@ function startServer() { server = spawn(BIN, ['--port', String(opt.port), '--db', DBFILE], { stdio: ['ignore', 'pipe', 'pipe'], }); - server.stdout.on('data', (d) => (serverLog += d)); - server.stderr.on('data', (d) => (serverLog += d)); + server.stdout.on('data', noteServerLog); + server.stderr.on('data', noteServerLog); server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`))); server.on('exit', (code, sig) => { // A child that dies must fail the start, or the poll below would find a // *stale* server on the same port and measure the wrong database. serverDead = true; - if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`; + if (code !== null && sig === null) noteServerLog(`\n[child exited rc=${code}]`); }); const deadline = Date.now() + 15000; (async () => { @@ -135,6 +142,10 @@ function startServer() { reject(new Error(`server child exited during start (port ${opt.port} busy?)\n${serverLog}`)); return; } + // A fresh client per attempt, deliberately: a MongoClient whose first + // connect fails tears its topology down and every later command on it + // fails the same way, so reusing one turns "not up yet" into "never + // comes up". const c = new MongoClient(URL, { serverSelectionTimeoutMS: 1000 }); try { await c.connect(); @@ -177,23 +188,21 @@ const MB = (n) => (n / (1 << 20)).toFixed(1); // a run against an older build comparable. let docBytes = 0; -async function stats(client, coll) { +async function stats(client, count) { const s = await client.db('admin').command({ serverStatus: 1 }); const m = s.multifora || null; - const num = (v) => Number(v); - const count = await coll.countDocuments({}); return { live: count * docBytes, count, file: dataFileBytes(), m: m && { - live: num(m.liveBytes), - dead: num(m.deadBytes), - reclaimed: num(m.reclaimedBytes), - runs: num(m.slabRuns), - freeReady: num(m.freeReadyPages), - allocTail: num(m.allocTail), - compactions: num(m.compactions), + live: Number(m.liveBytes), + dead: Number(m.deadBytes), + reclaimed: Number(m.reclaimedBytes), + runs: Number(m.slabRuns), + freeReady: Number(m.freeReadyBytes), + allocTail: Number(m.allocTailBytes), + compactions: Number(m.compactions), }, }; } @@ -206,10 +215,10 @@ function report(label, s) { // had to occupy. The file never shrinks, so `ratio` is a high-water mark // and cannot come down however well reclamation works; `inUse` is the // number that moves when it does. - const inUse = (s.m.allocTail - s.m.freeReady) * 4096; + const inUse = s.m.allocTail - s.m.freeReady; line += ` inUse ${(inUse / s.live).toFixed(2)}x` + ` dead ${MB(s.m.dead)}MB reclaimed ${MB(s.m.reclaimed)}MB` + - ` allocTail ${MB(s.m.allocTail * 4096)}MB freeReady ${MB(s.m.freeReady * 4096)}MB` + + ` allocTail ${MB(s.m.allocTail)}MB freeReady ${MB(s.m.freeReady)}MB` + ` runs ${s.m.runs} compactions ${s.m.compactions}`; } else { line += ' (no multifora section: counters n/a)'; @@ -260,7 +269,7 @@ async function main() { docBytes = require('mongodb').BSON.serialize(makeDoc(0)).length; await insertRange(coll, 0, opt.docs); if (opt.index) await coll.createIndex({ k: 1 }); - const base = await stats(client, coll); + const base = await stats(client, opt.docs); report('loaded', base); const ratios = []; @@ -283,14 +292,26 @@ async function main() { live[at] = live[live.length - 1]; live.pop(); } + // One delete spec per id, not one `$in` over thousands of them. The + // server's planner refuses to use an index for an `$in` wider than + // `index.max_combos` (100), so a 5000-element one falls back to a full + // collection scan that re-filters every document against every member -- + // quadratic in `--docs`, and it was the whole of this harness's runtime: + // the documented 40k x 16 KiB gate took 62 s and takes 11 s now, and the + // 150k x 200 B line went from ~480 s to 3 s. Reported ratios are + // unchanged, which is the point: this was the instrument's cost, not the + // database's. const bs = 5000; for (let i = 0; i < ids.length; i += bs) { - await coll.deleteMany({ _id: { $in: ids.slice(i, i + bs) } }); + await coll.bulkWrite( + ids.slice(i, i + bs).map((id) => ({ deleteOne: { filter: { _id: id } } })), + { ordered: false }, + ); } await insertRange(coll, nextId, nextId + ids.length); for (let i = 0; i < ids.length; i++) live.push(nextId + i); nextId += ids.length; - ratios.push(report(`round ${r + 1}`, await stats(client, coll))); + ratios.push(report(`round ${r + 1}`, await stats(client, opt.docs))); } } else { // The same documents rewritten over and over: every rewrite leaves the old @@ -309,12 +330,13 @@ async function main() { await coll.bulkWrite(ops, { ordered: false }); done += ops.length; } - ratios.push(report(`round ${r + 1}`, await stats(client, coll))); + ratios.push(report(`round ${r + 1}`, await stats(client, opt.docs))); } } - const final = await stats(client, coll); - const count = final.count; + // The one real count in the run, and the only one the check below needs. + const count = await coll.countDocuments({}); + const final = await stats(client, count); const elapsed = ((Date.now() - t0) / 1000).toFixed(0); console.log(`churn: ${count} documents live at the end, ${elapsed}s`); diff --git a/tests/e2e/results/m0-gates.txt b/tests/e2e/results/m0-gates.txt index 8574f0d..48def7e 100644 --- a/tests/e2e/results/m0-gates.txt +++ b/tests/e2e/results/m0-gates.txt @@ -175,7 +175,8 @@ which has no reclamation and no `multifora` section. The harness is committed this time (`tests/e2e/churn.js`), which is half the - point of the block: D7.4's numbers were real and unrepeatable. + point of the block: D7.4's numbers were real and unrepeatable. Each line + above runs in 6-10 s. Stage 2 M1 target delete half and refill, 6x 1.94x 1.94x <= 1.45x NOT MET @@ -232,7 +233,7 @@ [M1.2] churn gate — 200-byte documents reproduce: node tests/e2e/churn.js --docs 150000 --doc-size 200 --index \ - --mode delete-refill --rounds 4 + --mode delete-refill --rounds 4 (3 s) Predicted in advance, in the plan, as a pass rather than a fault: reclaimed 0.0 MB over four rounds, exactly as forecast. -- 2.39.5