dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index/pager: place a split's new sibling positionally, and fix mmap growth alignment

Browse Source 
Two bugs, both of which the crash fuzzer surfaced and neither of which any
existing test could see.

**A split put the new sibling in the wrong slot when separators repeat.**
`split_leaf` located the new right sibling with `separator_pos(node, key)`, a
search for the promoted key. That agrees with "immediately after `left`" only
while separators are distinct. When several children share one -- ten distinct
values across thousands of documents, so each value spans dozens of leaves --
`separator_pos` returns the slot after the *whole* equal-key run, which puts
the sibling at the end of that run while the leaf chain has it right after
`left`.

Parent child order then stops matching leaf chain order, and that is the one
thing a lookup cannot survive: `descend_lower` picks the last child of the equal
run, and `lookup_eq` walks forward from there over keys *smaller* than the one
it wants, stops at the first mismatch, and reports nothing. Every entry is
present, the chain is correctly ordered, `count()` is right -- and the query
returns empty. `crash-fuzz.js` found it after ~700 heavy cycles as
`find({k: 3})` returning 0 of 401 documents while every other key was exact.

Fixed by `child_slot_after`, which is positional by construction.

**mmap growth rounded with a non-power-of-two alignment.** Past 64 MiB the
growth chunk becomes a proportion of the current size (`mapped_pages / 8`),
which is not a power of two -- and `std.mem.alignForward` asserts that it is.
In safe builds that panicked; in ReleaseFast, where the assert is compiled out,
it computed `(addr + align - 1) & ~(align - 1)` with a non-power-of-two mask,
which can round *down*. A mapping shorter than intended is survivable, but a
mapping longer than the file is exactly what this function exists to prevent: a
store into a mapped page past end-of-file raises SIGBUS, which no error path
catches. `alignForwardAnyAlign` instead. Never noticed because no unit test grew
a pager past 64 MiB.

Also here, because both bugs were invisible rather than merely unfixed:

- `assert_indexes_cover_every_document` (db.zig) checks the index invariant
  directly -- an index generates candidates and the full filter is re-applied to
  those, so a missing entry is a missing query result nothing else detects.
- `Index.unreachable_key_count` counts keys present in the leaf chain but not
  reachable by descending from the root, which is precisely the state above:
  healthy by every other measure.
- `Index.dbg_root` dumps parent/chain agreement. Marked TEMPORARY; drop it once
  the invariant checks have earned their keep.
- `crash-fuzz.js` now asks the same question without the index, so a failure
  says whether the documents are wrong or only the index's answer about them,
  and reports per-key totals so one lost leaf is distinguishable from an empty
  index.

Verified: `zig build test` in ReleaseFast and ReleaseSafe, and seeded fuzzer
runs that previously reproduced the split bug.
This commit is contained in:
Alexey Shakhmatov
2026-08-04 14:51:56 +03:00
parent 62caf9fefc
commit cd88e1a4d1
4 changed files with 624 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

303
src/index.zig
View File

@@ -798,6 +798,70 @@ pub const Index = struct {
return .{ .ix = self, .leaf = self.first_leaf, .slot = 0 };
}
/// TEMPORARY: does the parent's child order match the leaf chain, and is each
/// separator really its child's first key?
pub fn dbg_root(self: *const Index) void {
const rt = self.page(self.root);
std.debug.print(" root={d} count={d} first_child={d} depth={d}\n", .{ self.root, rt.count, rt.first_child, self.depth });
// chain position of every leaf
var pos_of = std.mem.zeroes([512]i32);
for (&pos_of) |*v| v.* = -1;
var lf = self.first_leaf;
var pos: i32 = 0;
while (lf != 0) : (pos += 1) {
if (lf < pos_of.len) pos_of[lf] = pos;
lf = self.page(lf).next;
}
var prev_pos: i32 = pos_of[rt.first_child];
var i: u32 = 0;
while (i < rt.count) : (i += 1) {
const child = get_slot(rt, i).extra;
const cp = if (child < pos_of.len) pos_of[child] else -2;
const sep = self.key_of(self.root, i);
const cfirst = if (self.page(child).count > 0) self.key_of(child, 0) else "";
const sep_wrong = cfirst.len > 0 and !std.mem.eql(u8, sep, cfirst);
const order_wrong = cp != prev_pos + 1;
if (sep_wrong or order_wrong) {
std.debug.print(" slot[{d}] child={d} chainpos={d} (prev {d}){s}{s}\n sep ={x}\n first={x}\n", .{
i, child, cp, prev_pos,
if (order_wrong) " ORDER" else "", if (sep_wrong) " SEP!=FIRST" else "", sep, cfirst,
});
}
prev_pos = cp;
}
}
/// Debug aid: how many distinct keys are present in the leaf chain but not
/// findable by descending from the root.
///
/// Iteration and descent are two independent ways to reach an entry, and a
/// query only ever uses descent. `count()` cannot tell them apart -- it
/// returns a stored counter -- so an index whose leaves are intact but whose
/// interior nodes no longer route to them looks perfectly healthy by every
/// other measure, and silently answers a query with fewer documents than it
/// holds. That is what the crash fuzzer caught: a handful of key values
/// returning nothing while every other value was exact.
///
/// O(distinct keys x depth). For assertions and tests, not for the hot path.
pub fn unreachable_key_count(self: *const Index) u32 {
var bad: u32 = 0;
var it = self.iter();
var prev: ?[]const u8 = null;
while (it.next()) |e| {
if (prev) |p| {
if (std.mem.eql(u8, p, e.key)) continue;
}
prev = e.key;
var probe = self.seek(e.key);
const first = probe.next() orelse {
bad += 1;
continue;
};
if (cmp_prefix(e.key, first.key) != .eq) bad += 1;
}
return bad;
}
/// Reverse ordered iteration. Leaves are doubly linked and `prev` has
/// always been maintained -- nothing walked it until now, so a descending
/// scan had to materialize every candidate and reverse the list. This turns
@@ -1242,6 +1306,36 @@ pub const Index = struct {
/// Separator position in an internal node: after any equal keys, so the
/// "last separator <= key" descent lands on the newest right child.
/// The slot at which a new right sibling of `left` belongs: immediately after
/// `left`'s own position among this node's children.
///
/// Deliberately positional, not a search for the promoted key. The two agree
/// only while separators are distinct. When several children share a
/// separator -- ten distinct values across thousands of documents, so each
/// value spans dozens of leaves -- `separator_pos` returns the slot after the
/// *whole* equal-key run, which puts the new sibling at the end of that run
/// while the leaf chain has it immediately after `left`.
///
/// Parent child order then no longer matches leaf chain order, and that is
/// the one thing a lookup cannot survive: `descend_lower` picks the last
/// child of the equal run, and `lookup_eq` walks forward from there over keys
/// that are *smaller* than the one it wants, so it stops at the first
/// mismatch and reports nothing. The entries are all present, the chain is
/// correctly ordered, `count()` is right -- and a query returns an empty
/// result. Found by tests/fuzz/crash-fuzz.js after ~700 heavy cycles as
/// `find({k: 3})` returning 0 of 401 documents while every other key was
/// exact.
fn child_slot_after(self: *const Index, node_id: u32, left: u32) u32 {
const node = self.page(node_id);
if (node.first_child == left) return 0;
var i: u32 = 0;
while (i < node.count) : (i += 1) {
if (get_slot(node, i).extra == left) return i + 1;
}
assert_msg(false, "a split's left sibling must be a child of the node taking its separator");
return node.count;
}
fn separator_pos(self: *const Index, node_id: u32, key: []const u8) u32 {
const node = self.page(node_id);
var lo: u32 = 0;
@@ -1352,7 +1446,7 @@ pub const Index = struct {
}
const child = self.descend_insert(node_id, key);
const res = self.insert_rec(child, key, off) orelse return null;
return self.insert_separator(node_id, res);
return self.insert_separator(node_id, child, res);
}
/// Split a full leaf around the record being inserted. The new record
@@ -1426,7 +1520,7 @@ pub const Index = struct {
/// Insert a promoted separator into an internal node, splitting it when
/// full. Returns the next promotion, or null.
fn insert_separator(self: *Index, node_id: u32, split: Split) ?Split {
fn insert_separator(self: *Index, node_id: u32, left: u32, split: Split) ?Split {
// The incoming key may live in the promo buffer, which a nested
// split_internal (below) would overwrite with its own promoted key;
// spilled keys already live in the immutable slab. Copy inline keys
@@ -1443,7 +1537,7 @@ pub const Index = struct {
self.repack_keep_prefix(node_id, self.page(node_id).count);
}
if (self.fits(node_id, key.len)) {
self.store_record(node_id, self.separator_pos(node_id, key), .{
self.store_record(node_id, self.child_slot_after(node_id, left), .{
.key = key,
.child = split.right,
.spill_off = split.spill_off,
@@ -1451,7 +1545,7 @@ pub const Index = struct {
self.page_mut(split.right).parent = node_id;
return null;
}
return self.split_internal(node_id, key, split.spill_off, split.right);
return self.split_internal(node_id, left, key, split.spill_off, split.right);
}
/// Split a full internal node around the separator being inserted: the
@@ -1462,13 +1556,15 @@ pub const Index = struct {
fn split_internal(
self: *Index,
node_id: u32,
left: u32,
key: []const u8,
spill_off: ?u64,
child: u32,
) Split {
const old_count = self.page(node_id).count;
std.debug.assert(old_count >= 2);
const pos = self.separator_pos(node_id, key);
// Positional, for the reason `child_slot_after` documents.
const pos = self.child_slot_after(node_id, left);
const n = old_count + 1;
var costs: [max_slots + 1]u32 = undefined;
@@ -2345,6 +2441,50 @@ fn simple_index(
return Index.init(gpa, pager, "test", keys[0..paths.len], unique, sparse, null);
}
test "a bulk build with many duplicate keys stays reachable for every key" {
// The shape the crash fuzzer failed on: ~4000 entries over 10 distinct key
// values, so each value spans several leaves and the interior separators
// repeat. `find({k:v})` came back empty for a few values and exactly right
// for the rest, with `count()` still reporting every entry -- entries that
// exist and cannot be reached.
const gpa = testing.allocator;
var ix = try simple_index(gpa, test_pager(), &.{"k"}, false, false);
defer ix.deinit(gpa);
const n_docs: usize = 4000;
const n_keys: i32 = 10;
var docs: std.ArrayListUnmanaged([]u8) = .empty;
defer {
for (docs.items) |d| gpa.free(d);
docs.deinit(gpa);
}
for (0..n_docs) |i| {
const k: i32 = @intCast(@mod(@as(i32, @intCast(i)), n_keys));
const pairs = [_]bson.Pair{.{ .key = "k", .value = .{ .int32 = k } }};
const bytes = try bytes_of(gpa, &pairs);
try docs.append(gpa, bytes);
try ix.append_doc_entries(gpa, bytes, @intCast(i + 1));
}
_ = try ix.finish_bulk(gpa, false);
try testing.expectEqual(n_docs, ix.count());
// Iteration must see every entry: that separates "never inserted" from
// "inserted and unreachable from the root".
var walked: usize = 0;
var wit = ix.iter();
while (wit.next()) |_| walked += 1;
try testing.expectEqual(n_docs, walked);
// And every key must be reachable by descent, which is what a query does.
var k: i32 = 0;
while (k < n_keys) : (k += 1) {
var out: std.ArrayListUnmanaged(u64) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .int32 = k }}, &out);
try testing.expectEqual(n_docs / @as(usize, @intCast(n_keys)), out.items.len);
}
}
/// Look up the documents under `key` and compare with the expected offsets.
/// A leaf record's payload is a slab offset now, so tests identify documents by
/// small distinct numbers rather than by byte-string ids.
@@ -2884,6 +3024,159 @@ test "incremental inserts and removals stay identical to a brute-force model" {
}
}
test "splitting inside a run of equal separators keeps every key reachable" {
// The crash fuzzer's exact shape, and the reason the two differentials above
// miss it: bulk-pack first, *then* keep inserting.
//
// A packed tree has full leaves, so the next inserts split leaves in the
// middle of a run of equal separators -- and a new right sibling placed by
// key rather than by position lands at the end of that run, so the parent's
// child order stops matching the leaf chain. A purely incremental build
// leaves leaves half full and rarely produces the geometry.
const gpa = testing.allocator;
var ix = try simple_index(gpa, test_pager(), &.{"k"}, false, false);
defer ix.deinit(gpa);
const n_keys: i32 = 10;
const packed_docs: usize = 4000;
const grown_docs: usize = 2000;
// Phase 1: bulk pack, which fills every leaf.
for (0..packed_docs) |i| {
const k: i32 = @intCast(@mod(@as(i32, @intCast(i)), n_keys));
const d = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = k } }});
defer gpa.free(d);
try ix.append_doc_entries(gpa, d, @intCast(i + 1));
}
_ = try ix.finish_bulk(gpa, false);
ix.pager.release_reservation(&ix.hold);
// Phase 2: grow it in random key order, which is what puts a split on the
// leaf *before* an equal-key run -- the case where the new right sibling's
// promoted key equals the run's key while its chain position is at the run's
// start. Round-robin never produces it: every insert routes to the last
// child of its run, and a split there belongs at the end of the run anyway.
var prng = std.Random.DefaultPrng.init(0xbad_5eed);
const rand = prng.random();
for (0..grown_docs) |j| {
const k = rand.intRangeLessThan(i32, 0, n_keys);
const d = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = k } }});
defer gpa.free(d);
_ = try ix.add_doc(gpa, d, @intCast(packed_docs + j + 1), false);
ix.pager.release_reservation(&ix.hold);
if (j % 25 == 0) {
testing.expectEqual(@as(u32, 0), ix.unreachable_key_count()) catch |err| {
std.debug.print(" went unreachable after {d} grown inserts\n", .{j});
return err;
};
}
}
try testing.expectEqual(packed_docs + grown_docs, ix.count());
// Every entry the chain holds must be findable by descent too.
try testing.expectEqual(@as(u32, 0), ix.unreachable_key_count());
// And a lookup must find as many entries as iteration holds for that key.
var k: i32 = 0;
while (k < n_keys) : (k += 1) {
var enc: std.ArrayListUnmanaged(u8) = .empty;
defer enc.deinit(gpa);
try bson.encode_key(bson.Value{ .int32 = k }, gpa, &enc);
var want: usize = 0;
var wit = ix.iter();
while (wit.next()) |e| {
if (cmp_prefix(enc.items, e.key) == .eq) want += 1;
}
var out: std.ArrayListUnmanaged(u64) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .int32 = k }}, &out);
testing.expectEqual(want, out.items.len) catch |err| {
std.debug.print(" key {d}: reachable {d}, iteration holds {d} (entry_count {d})\n", .{
k, out.items.len, want, ix.count(),
});
return err;
};
}
}
test "an incrementally built index with few distinct keys stays reachable" {
// The crash fuzzer's shape, which the existing differentials miss: a single
// key with only ten distinct values over thousands of documents, so each
// value spans dozens of leaves and most interior separators are duplicates.
// The compound-key differential above uses ~961 combinations over 600
// inserts, which is almost no duplication at all.
//
// Symptom being hunted: `lookup_eq` returning nothing for a few values while
// every other value is exactly right, and `count()` still reporting every
// entry -- entries that exist and cannot be reached by descent.
const gpa = testing.allocator;
var prng = std.Random.DefaultPrng.init(0xd0_0d_1e);
const rand = prng.random();
var ix = try simple_index(gpa, test_pager(), &.{"k"}, false, false);
defer ix.deinit(gpa);
const n_keys: i32 = 10;
const n: usize = 3000;
var keys: std.ArrayListUnmanaged(i32) = .empty;
defer keys.deinit(gpa);
var live: std.ArrayListUnmanaged(bool) = .empty;
defer live.deinit(gpa);
// Interleave inserts and removals, which is what a real workload does and
// what leaves half-empty leaves and one-child internal nodes behind.
for (0..n) |i| {
const k = rand.intRangeAtMost(i32, 0, n_keys - 1);
const off: u64 = @intCast(i + 1);
const d = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = k } }});
defer gpa.free(d);
_ = try ix.add_doc(gpa, d, off, false);
// A bare index test has no engine to do this at a write boundary, and
// without it the promise accumulates and grows the shared test file
// without bound.
ix.pager.release_reservation(&ix.hold);
try keys.append(gpa, k);
try live.append(gpa, true);
// Remove an earlier document every few inserts.
if (i > 20 and i % 3 == 0) {
const victim = rand.intRangeLessThan(usize, 0, keys.items.len);
if (live.items[victim]) {
const vd = try bytes_of(gpa, &.{.{ .key = "k", .value = .{ .int32 = keys.items[victim] } }});
defer gpa.free(vd);
ix.remove_doc(gpa, vd, @intCast(victim + 1));
live.items[victim] = false;
}
}
// Every value must be reachable by descent, and the counts must match a
// brute-force pass over the model. Checked periodically rather than every
// step: this is 10 descents over a tree of thousands of entries.
if (i % 250 != 0 and i != n - 1) continue;
var k_check: i32 = 0;
while (k_check < n_keys) : (k_check += 1) {
var want: usize = 0;
for (keys.items, live.items) |kk, is_live| {
if (is_live and kk == k_check) want += 1;
}
var out: std.ArrayListUnmanaged(u64) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .int32 = k_check }}, &out);
testing.expectEqual(want, out.items.len) catch |err| {
std.debug.print(
" at insert {d}: key {d} reachable {d}, expected {d} (entry_count {d})\n",
.{ i, k_check, out.items.len, want, ix.count() },
);
return err;
};
}
// And iteration must see exactly as many entries as the tree claims.
var walked: usize = 0;
var wit = ix.iter();
while (wit.next()) |_| walked += 1;
try testing.expectEqual(ix.count(), walked);
}
}
/// One document's facts in the incremental-mutation differential.
const ModelFact = struct { a: i32, b: i32, off: u64 };