index: B+tree over the encoded keys (roadmap item 1)

Replace Index.entries (one sorted array) with a B+tree so writes into an
already-built index stop being quadratic. Nodes are fixed 4 KiB slotted
pages in a flat u32-addressed ArrayListUnmanaged(Node); records longer
than a quarter page spill to an append-only overflow slab (BSON strings
reach 16 MB). Leaves are doubly linked for ordered iteration; the flat
node array stays one contiguous byte range for a later checkpoint.

Insertion descends by separator and splits leaves/internals upward,
promoting keys via a stable copy (a nested split can otherwise clobber
the promoted-key scratch). Deletion does not rebalance: emptied leaves
are unlinked and dropped from their parent, internal nodes may carry one
child, and dead pages are abandoned in place (node memory peaks at the
tree's peak size, exactly what the old array's capacity did). Lookups
are lower-bound seeks plus leaf-chain band scans, so equal keys may
span leaves freely. Bulk build (append_doc_entries + finish_bulk) sorts
a staging array and packs leaves bottom-up. reserve_for now takes the
built entries and reserves exact overflow bytes plus a worst-case node
count, keeping insert_entries infallible after the log append.

db.zig: TTL sweep now seeks the minimum-datetime encoded key and walks
the contiguous datetime band, stopping at the cutoff or type change.

Measured (tests/e2e/results/phase2.txt): updateMany 17.3 -> 1.8 ms
(2.8x slower than MongoDB -> 3.7x faster), createIndex 62 -> 51 ms.

Verified: unit suite ReleaseFast/ReleaseSafe/Debug (incl. the existing
lookup_range and remove_doc differentials, plus a new incremental
insert/remove differential against a brute-force model), the crash pair,
e2e3/e2e4/e2e6, and dev stress tests for depth-2 splits, full drains,
and spilled records through internal levels.
This commit is contained in:
2026-08-02 21:10:25 +03:00
parent 71112b0ff7
commit 61fe952125
8 changed files with 1450 additions and 218 deletions

74
src/spill.zig Normal file
View File

@@ -0,0 +1,74 @@
// Dev stress test for the overflow slab (records > 1024 bytes):
// zig run -O ReleaseFast src/spill.zig
// Keys straddling the spill threshold (10 B .. 100 KB) through insert,
// lookup, delete and iteration.
const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
}
pub fn main() !void {
const gpa = std.heap.page_allocator;
var prng = std.Random.DefaultPrng.init(0x1234_5678);
const rand = prng.random();
var keys = [_]index.IndexKey{.{ .path = "tag", .descending = false }};
var ix = try index.Index.init(gpa, "tag", &keys, false, false, null);
// Keys straddling the spill threshold: inline, exactly at the limit,
// just over, and one very long. Each id is a short static string.
const lens = [_]usize{ 10, 1023, 1024, 1025, 2000, 100_000 };
var strings: [lens.len][]u8 = undefined;
var docs: [lens.len]bson.Document = undefined;
var pairs: [2]bson.Pair = undefined;
for (lens, 0..) |len, i| {
strings[i] = try gpa.alloc(u8, len);
for (strings[i]) |*c| c.* = 'a' + @as(u8, @intCast(rand.intRangeAtMost(u8, 0, 25)));
// add a distinguishing suffix so keys are unique
std.mem.copyForwards(u8, strings[i][len - 4 ..], &[_]u8{ @intCast(i), 0xff, 0x00, 0x00 });
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = strings[i] } };
docs[i] = doc_of(&pairs);
_ = try ix.add_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) }, true);
}
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
if (ix.overflow.items.len < 100_000) return error.NoSpill;
// Every entry is found by exact key.
for (lens, 0..) |_, i| {
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = strings[i] }}, &out);
if (out.items.len != 1) {
std.debug.print("lookup {d} got {d}\n", .{ i, out.items.len });
return error.Bad;
}
if (!std.mem.eql(u8, out.items[0], &[_]u8{ 'i', 'd', @intCast(i + 1) })) return error.Bad;
}
// Delete the spilled ones and the inline ones alternately.
for (lens, 0..) |_, i| {
if (i % 2 == 0) continue;
ix.remove_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) });
}
if (ix.count() != 3) return error.Bad;
for (lens, 0..) |_, i| {
if (i % 2 == 0) continue;
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = strings[i] }}, &out);
if (out.items.len != 0) return error.Bad;
}
// Iteration still sees the survivors in order.
var it = ix.iter();
var seen: usize = 0;
while (it.next()) |e| {
seen += 1;
_ = e;
}
if (seen != 3) return error.Bad;
std.debug.print("SPILL OK (seen={d})\n", .{seen});
}