index: B+tree over the encoded keys (roadmap item 1)

Replace Index.entries (one sorted array) with a B+tree so writes into an
already-built index stop being quadratic. Nodes are fixed 4 KiB slotted
pages in a flat u32-addressed ArrayListUnmanaged(Node); records longer
than a quarter page spill to an append-only overflow slab (BSON strings
reach 16 MB). Leaves are doubly linked for ordered iteration; the flat
node array stays one contiguous byte range for a later checkpoint.

Insertion descends by separator and splits leaves/internals upward,
promoting keys via a stable copy (a nested split can otherwise clobber
the promoted-key scratch). Deletion does not rebalance: emptied leaves
are unlinked and dropped from their parent, internal nodes may carry one
child, and dead pages are abandoned in place (node memory peaks at the
tree's peak size, exactly what the old array's capacity did). Lookups
are lower-bound seeks plus leaf-chain band scans, so equal keys may
span leaves freely. Bulk build (append_doc_entries + finish_bulk) sorts
a staging array and packs leaves bottom-up. reserve_for now takes the
built entries and reserves exact overflow bytes plus a worst-case node
count, keeping insert_entries infallible after the log append.

db.zig: TTL sweep now seeks the minimum-datetime encoded key and walks
the contiguous datetime band, stopping at the cutoff or type change.

Measured (tests/e2e/results/phase2.txt): updateMany 17.3 -> 1.8 ms
(2.8x slower than MongoDB -> 3.7x faster), createIndex 62 -> 51 ms.

Verified: unit suite ReleaseFast/ReleaseSafe/Debug (incl. the existing
lookup_range and remove_doc differentials, plus a new incremental
insert/remove differential against a brute-force model), the crash pair,
e2e3/e2e4/e2e6, and dev stress tests for depth-2 splits, full drains,
and spilled records through internal levels.
This commit is contained in:
2026-08-02 21:10:25 +03:00
parent 71112b0ff7
commit 61fe952125
8 changed files with 1450 additions and 218 deletions

88
src/spill2.zig Normal file
View File

@@ -0,0 +1,88 @@
// Dev stress test: spilled records through leaf splits and internal levels.
// zig run -O ReleaseFast src/spill2.zig
// 5000 docs with 2 KiB keys: every record spills; the tree still answers
// exact lookups and deletes half of them.
const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
}
pub fn main() !void {
const gpa = std.heap.page_allocator;
var keys = [_]index.IndexKey{.{ .path = "tag", .descending = false }};
var ix = try index.Index.init(gpa, "tag", &keys, false, false, null);
// 5000 docs, each with a 2 KiB key: spills on every record, forcing
// leaves and internal nodes to hold overflow references.
const N = 5000;
var ids: std.ArrayListUnmanaged([]u8) = .empty;
defer {
for (ids.items) |x| gpa.free(x);
ids.deinit(gpa);
}
var pairs: [2]bson.Pair = undefined;
var buf = try gpa.alloc(u8, 2000);
defer gpa.free(buf);
var facts: std.ArrayListUnmanaged(struct { key: []u8 }) = .empty;
defer {
for (facts.items) |f| gpa.free(f.key);
facts.deinit(gpa);
}
for (0..N) |i| {
for (buf) |*c| c.* = 'x';
// unique suffix
std.mem.writeInt(u32, buf[0..4], @intCast(i), .little);
const key = try gpa.dupe(u8, buf);
try facts.append(gpa, .{ .key = key });
const id = try std.fmt.allocPrint(gpa, "id{d}", .{i});
try ids.append(gpa, id);
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = key } };
const d = doc_of(&pairs);
_ = try ix.add_doc(gpa, &d, id, false);
}
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
if (ix.count() != N) return error.Bad;
// Spot-check exact lookups.
var prng2 = std.Random.DefaultPrng.init(0xabc);
const rand2 = prng2.random();
for (0..300) |_| {
const i = rand2.intRangeAtMost(usize, 0, N - 1);
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = facts.items[i].key }}, &out);
if (out.items.len != 1 or !std.mem.eql(u8, out.items[0], ids.items[i])) {
std.debug.print("lookup mismatch at {d}\n", .{i});
return error.Bad;
}
}
// Delete half (random), verify count and no leftover.
var order: std.ArrayListUnmanaged(usize) = .empty;
defer order.deinit(gpa);
for (0..N) |i| if (i % 2 == 0) try order.append(gpa, i);
rand2.shuffle(usize, order.items);
var removed: usize = 0;
for (order.items) |i| {
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = facts.items[i].key } };
const d = doc_of(&pairs);
ix.remove_doc(gpa, &d, ids.items[i]);
removed += 1;
if (ix.count() != N - removed) {
std.debug.print("count mismatch at {d}: {d} != {d}\n", .{ i, ix.count(), N - removed });
return error.Bad;
}
}
for (order.items) |i| {
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = facts.items[i].key }}, &out);
if (out.items.len != 0) return error.Bad;
}
std.debug.print("SPILL2 OK (leaves={d} depth={d})\n", .{ ix.leaf_count, ix.depth });
}