dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index: B+tree over the encoded keys (roadmap item 1)

Browse Source 
Replace Index.entries (one sorted array) with a B+tree so writes into an
already-built index stop being quadratic. Nodes are fixed 4 KiB slotted
pages in a flat u32-addressed ArrayListUnmanaged(Node); records longer
than a quarter page spill to an append-only overflow slab (BSON strings
reach 16 MB). Leaves are doubly linked for ordered iteration; the flat
node array stays one contiguous byte range for a later checkpoint.

Insertion descends by separator and splits leaves/internals upward,
promoting keys via a stable copy (a nested split can otherwise clobber
the promoted-key scratch). Deletion does not rebalance: emptied leaves
are unlinked and dropped from their parent, internal nodes may carry one
child, and dead pages are abandoned in place (node memory peaks at the
tree's peak size, exactly what the old array's capacity did). Lookups
are lower-bound seeks plus leaf-chain band scans, so equal keys may
span leaves freely. Bulk build (append_doc_entries + finish_bulk) sorts
a staging array and packs leaves bottom-up. reserve_for now takes the
built entries and reserves exact overflow bytes plus a worst-case node
count, keeping insert_entries infallible after the log append.

db.zig: TTL sweep now seeks the minimum-datetime encoded key and walks
the contiguous datetime band, stopping at the cutoff or type change.

Measured (tests/e2e/results/phase2.txt): updateMany 17.3 -> 1.8 ms
(2.8x slower than MongoDB -> 3.7x faster), createIndex 62 -> 51 ms.

Verified: unit suite ReleaseFast/ReleaseSafe/Debug (incl. the existing
lookup_range and remove_doc differentials, plus a new incremental
insert/remove differential against a brute-force model), the crash pair,
e2e3/e2e4/e2e6, and dev stress tests for depth-2 splits, full drains,
and spilled records through internal levels.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 21:10:25 +03:00
parent 71112b0ff7
commit 61fe952125
8 changed files with 1450 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/db.zig
View File

@@ -255,10 +255,10 @@ pub const Engine = struct {
};
}
// 4. Reserve entry capacity — the last fallible step, so the entry
// 4. Reserve tree capacity — the last fallible step, so the entry
// insertion after the log append is infallible.
for (built_list.items) |*b| {
try b.ix.reserve_for(self.gpa, b.built.entries.items.len);
try b.ix.reserve_for(self.gpa, b.built.entries.items);
}
// 5. Log (and sync) before anything becomes visible.
@@ -365,7 +365,7 @@ pub const Engine = struct {
while (doc_it.next()) |entry| {
try ix.append_doc_entries(self.gpa, entry.value_ptr.*, entry.key_ptr.*);
}
_ = try ix.finish_bulk(true);
_ = try ix.finish_bulk(self.gpa, true);
// Reserve the collection slot, then persist and publish.
try coll.indexes.ensureUnusedCapacity(self.gpa, 1);
@@ -428,16 +428,16 @@ pub const Engine = struct {
for (coll_entry.value_ptr.indexes.items) |*ix| {
const ttl = ix.ttl orelse continue;
const cutoff: i128 = @as(i128, now_ms) - @as(i128, ttl) * 1000;
for (ix.entries.items) |e| {
// Still a linear walk: the type test cannot be a
// one-sided range lookup, because bson compare order
// ranks datetime above null, numbers and strings, so
// a datetime upper bound would also select every
// value of a lesser type. (Datetimes are contiguous
// in that order, so a two-sided band lookup would
// work — that comes with the tree.)
const ms = bson.encoded_leading_datetime(e.key) orelse continue;
if (@as(i128, ms) > cutoff) continue;
// bson compare order ranks datetime above null, numbers
// and strings and below only timestamp and maxKey, so
// datetimes form a contiguous band in the encoded key
// order: seek the minimum datetime and stop when the
// leading type changes or the cutoff is passed.
const min_dt = [_]u8{ bson.encoded_datetime_tag, 0, 0, 0, 0, 0, 0, 0, 0 };
var it = ix.seek(&min_dt);
while (it.next()) |e| {
const ms = bson.encoded_leading_datetime(e.key) orelse break;
if (@as(i128, ms) > cutoff) break;
try ids.append(self.gpa, try self.gpa.dupe(u8, e.id));
}
}
@@ -624,7 +624,7 @@ pub const Engine = struct {
var coll_it = db_entry.value_ptr.collections.iterator();
while (coll_it.next()) |coll_entry| {
for (coll_entry.value_ptr.indexes.items) |*ix| {
if (ix.entries.items.len > 0) continue; // defensive
if (ix.count() > 0) continue; // defensive
var doc_it = coll_entry.value_ptr.docs.iterator();
while (doc_it.next()) |doc_entry| {
ix.append_doc_entries(self.gpa, doc_entry.value_ptr.*, doc_entry.key_ptr.*) catch |err| switch (err) {
@@ -636,7 +636,7 @@ pub const Engine = struct {
};
}
// Tolerated, not enforced: the database must always open.
if (try ix.finish_bulk(false)) {
if (try ix.finish_bulk(self.gpa, false)) {
std.debug.print("mongo-lite: WARNING: unique index '{s}' has duplicate keys in existing data; duplicates not enforced for existing documents\n", .{ix.name});
}
}
@@ -1369,12 +1369,12 @@ test "ttl_sweep deletes expired documents and the deletion survives reopen" {
}
const coll = engine.get_collection("app", "sessions").?;
try testing.expectEqual(@as(usize, 6), coll.docs.count());
try testing.expectEqual(@as(usize, 6), coll.indexes.items[0].entries.items.len);
try testing.expectEqual(@as(usize, 6), coll.indexes.items[0].count());
// The cutoff is inclusive: doc 2 goes with doc 1.
try testing.expectEqual(@as(usize, 2), try engine.ttl_sweep(now_ms));
try testing.expectEqual(@as(usize, 4), coll.docs.count());
try testing.expectEqual(@as(usize, 4), coll.indexes.items[0].entries.items.len);
try testing.expectEqual(@as(usize, 4), coll.indexes.items[0].count());
try testing.expectEqual(@as(usize, 0), try index_count(gpa, &engine, "app", "sessions", "expireAt_1", .{ .datetime = now_ms - 120_000 }));
// The string and the missing field are untouched by any sweep.
try testing.expectEqual(@as(usize, 1), try index_count(gpa, &engine, "app", "sessions", "expireAt_1", .{ .string = "tomorrow" }));
@@ -1397,7 +1397,7 @@ test "ttl_sweep deletes expired documents and the deletion survives reopen" {
try testing.expectEqual(@as(usize, 3), coll.docs.count());
try testing.expectEqual(@as(usize, 1), coll.indexes.items.len);
try testing.expectEqual(@as(?i64, 60), coll.indexes.items[0].ttl);
try testing.expectEqual(@as(usize, 3), coll.indexes.items[0].entries.items.len);
try testing.expectEqual(@as(usize, 3), coll.indexes.items[0].count());
for ([_]i32{ 1, 2, 3 }) |id| {
const id_key = try bson.serialize_value(gpa, bson.Value{ .int32 = id });
defer gpa.free(id_key);

1173
src/index.zig
View File

File diff suppressed because it is too large Load Diff

74
src/spill.zig Normal file
View File

@@ -0,0 +1,74 @@
// Dev stress test for the overflow slab (records > 1024 bytes):
// zig run -O ReleaseFast src/spill.zig
// Keys straddling the spill threshold (10 B .. 100 KB) through insert,
// lookup, delete and iteration.
const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
}
pub fn main() !void {
const gpa = std.heap.page_allocator;
var prng = std.Random.DefaultPrng.init(0x1234_5678);
const rand = prng.random();
var keys = [_]index.IndexKey{.{ .path = "tag", .descending = false }};
var ix = try index.Index.init(gpa, "tag", &keys, false, false, null);
// Keys straddling the spill threshold: inline, exactly at the limit,
// just over, and one very long. Each id is a short static string.
const lens = [_]usize{ 10, 1023, 1024, 1025, 2000, 100_000 };
var strings: [lens.len][]u8 = undefined;
var docs: [lens.len]bson.Document = undefined;
var pairs: [2]bson.Pair = undefined;
for (lens, 0..) |len, i| {
strings[i] = try gpa.alloc(u8, len);
for (strings[i]) |*c| c.* = 'a' + @as(u8, @intCast(rand.intRangeAtMost(u8, 0, 25)));
// add a distinguishing suffix so keys are unique
std.mem.copyForwards(u8, strings[i][len - 4 ..], &[_]u8{ @intCast(i), 0xff, 0x00, 0x00 });
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = strings[i] } };
docs[i] = doc_of(&pairs);
_ = try ix.add_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) }, true);
}
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
if (ix.overflow.items.len < 100_000) return error.NoSpill;
// Every entry is found by exact key.
for (lens, 0..) |_, i| {
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = strings[i] }}, &out);
if (out.items.len != 1) {
std.debug.print("lookup {d} got {d}\n", .{ i, out.items.len });
return error.Bad;
}
if (!std.mem.eql(u8, out.items[0], &[_]u8{ 'i', 'd', @intCast(i + 1) })) return error.Bad;
}
// Delete the spilled ones and the inline ones alternately.
for (lens, 0..) |_, i| {
if (i % 2 == 0) continue;
ix.remove_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) });
}
if (ix.count() != 3) return error.Bad;
for (lens, 0..) |_, i| {
if (i % 2 == 0) continue;
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = strings[i] }}, &out);
if (out.items.len != 0) return error.Bad;
}
// Iteration still sees the survivors in order.
var it = ix.iter();
var seen: usize = 0;
while (it.next()) |e| {
seen += 1;
_ = e;
}
if (seen != 3) return error.Bad;
std.debug.print("SPILL OK (seen={d})\n", .{seen});
}

88
src/spill2.zig Normal file
View File

@@ -0,0 +1,88 @@
// Dev stress test: spilled records through leaf splits and internal levels.
// zig run -O ReleaseFast src/spill2.zig
// 5000 docs with 2 KiB keys: every record spills; the tree still answers
// exact lookups and deletes half of them.
const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
}
pub fn main() !void {
const gpa = std.heap.page_allocator;
var keys = [_]index.IndexKey{.{ .path = "tag", .descending = false }};
var ix = try index.Index.init(gpa, "tag", &keys, false, false, null);
// 5000 docs, each with a 2 KiB key: spills on every record, forcing
// leaves and internal nodes to hold overflow references.
const N = 5000;
var ids: std.ArrayListUnmanaged([]u8) = .empty;
defer {
for (ids.items) |x| gpa.free(x);
ids.deinit(gpa);
}
var pairs: [2]bson.Pair = undefined;
var buf = try gpa.alloc(u8, 2000);
defer gpa.free(buf);
var facts: std.ArrayListUnmanaged(struct { key: []u8 }) = .empty;
defer {
for (facts.items) |f| gpa.free(f.key);
facts.deinit(gpa);
}
for (0..N) |i| {
for (buf) |*c| c.* = 'x';
// unique suffix
std.mem.writeInt(u32, buf[0..4], @intCast(i), .little);
const key = try gpa.dupe(u8, buf);
try facts.append(gpa, .{ .key = key });
const id = try std.fmt.allocPrint(gpa, "id{d}", .{i});
try ids.append(gpa, id);
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = key } };
const d = doc_of(&pairs);
_ = try ix.add_doc(gpa, &d, id, false);
}
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
if (ix.count() != N) return error.Bad;
// Spot-check exact lookups.
var prng2 = std.Random.DefaultPrng.init(0xabc);
const rand2 = prng2.random();
for (0..300) |_| {
const i = rand2.intRangeAtMost(usize, 0, N - 1);
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = facts.items[i].key }}, &out);
if (out.items.len != 1 or !std.mem.eql(u8, out.items[0], ids.items[i])) {
std.debug.print("lookup mismatch at {d}\n", .{i});
return error.Bad;
}
}
// Delete half (random), verify count and no leftover.
var order: std.ArrayListUnmanaged(usize) = .empty;
defer order.deinit(gpa);
for (0..N) |i| if (i % 2 == 0) try order.append(gpa, i);
rand2.shuffle(usize, order.items);
var removed: usize = 0;
for (order.items) |i| {
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = facts.items[i].key } };
const d = doc_of(&pairs);
ix.remove_doc(gpa, &d, ids.items[i]);
removed += 1;
if (ix.count() != N - removed) {
std.debug.print("count mismatch at {d}: {d} != {d}\n", .{ i, ix.count(), N - removed });
return error.Bad;
}
}
for (order.items) |i| {
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .string = facts.items[i].key }}, &out);
if (out.items.len != 0) return error.Bad;
}
std.debug.print("SPILL2 OK (leaves={d} depth={d})\n", .{ ix.leaf_count, ix.depth });
}

174
src/stress.zig Normal file
View File

@@ -0,0 +1,174 @@
// Dev stress test for the index B+tree (not part of the build):
// zig run -O ReleaseFast src/stress.zig
// Bulk-builds 30k entries, inserts 20k more one at a time (splits at depth 2),
// deletes 16.6k randomly (empty-leaf cascades), drains everything, and checks
// lookups against a brute-force model throughout.
const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
}
const Fact = struct { a: i32, b: i32 };
fn check_range(gpa: std.mem.Allocator, ix: *const index.Index, prefix: bson.Value, lo: ?bson.Value, hi: ?bson.Value, facts: []const Fact, alive: []const bool) !void {
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_range(gpa, &.{prefix}, lo, true, hi, false, &out);
var expected: usize = 0;
for (facts, 0..) |f, fi| {
if (!alive[fi]) continue;
if (f.a != prefix.int32) continue;
if (lo) |l| if (f.b < l.int32) continue;
if (hi) |h| if (f.b >= h.int32) continue;
expected += 1;
}
if (out.items.len != expected) {
std.debug.print("MISMATCH: prefix={d} lo={?d} hi={?d}: got {d}, want {d}\n", .{ prefix.int32, if (lo) |l| l.int32 else null, if (hi) |h| h.int32 else null, out.items.len, expected });
std.process.exit(1);
}
}
pub fn main() !void {
const gpa = std.heap.page_allocator;
var prng = std.Random.DefaultPrng.init(0xBADCAFE);
const rand = prng.random();
const N = 30_000;
var ids: std.ArrayListUnmanaged([]u8) = .empty;
var facts: std.ArrayListUnmanaged(Fact) = .empty;
var alive: std.ArrayListUnmanaged(bool) = .empty;
var pairs: [2]bson.Pair = undefined;
// 1. Bulk build an index over N docs.
var keys = [_]index.IndexKey{ .{ .path = "a", .descending = false }, .{ .path = "b", .descending = false } };
var ix = try index.Index.init(gpa, "ab", &keys, false, false, null);
for (0..N) |i| {
const id = try std.fmt.allocPrint(gpa, "id{d}", .{i});
try ids.append(gpa, id);
const a = rand.intRangeAtMost(i32, 0, 99);
const b = rand.intRangeAtMost(i32, 0, 999);
try facts.append(gpa, .{ .a = a, .b = b });
try alive.append(gpa, true);
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
try ix.append_doc_entries(gpa, &d, id);
}
_ = try ix.finish_bulk(gpa, false);
std.debug.print("bulk: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
if (ix.count() != N) return error.BadCount;
// Random range checks against brute force.
for (0..500) |_| {
const a = rand.intRangeAtMost(i32, 0, 99);
const lo_v = rand.intRangeAtMost(i32, -10, 1009);
const hi_v = rand.intRangeAtMost(i32, -10, 1009);
const use_lo = rand.boolean();
const use_hi = rand.boolean();
try check_range(gpa, &ix, .{ .int32 = a }, if (use_lo) .{ .int32 = lo_v } else null, if (use_hi) .{ .int32 = hi_v } else null, facts.items, alive.items);
}
// 2. Incremental inserts, random order (splits + rebalancing-free path).
const M = 20_000;
for (0..M) |i| {
const id = try std.fmt.allocPrint(gpa, "new{d}", .{i});
try ids.append(gpa, id);
const a = rand.intRangeAtMost(i32, 0, 99);
const b = rand.intRangeAtMost(i32, 0, 999);
try facts.append(gpa, .{ .a = a, .b = b });
try alive.append(gpa, true);
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
_ = try ix.add_doc(gpa, &d, id, false);
}
std.debug.print("after inserts: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
if (ix.count() != N + M) return error.BadCount;
for (0..500) |_| {
const a = rand.intRangeAtMost(i32, 0, 99);
const lo_v = rand.intRangeAtMost(i32, -10, 1009);
const hi_v = rand.intRangeAtMost(i32, -10, 1009);
const use_lo = rand.boolean();
const use_hi = rand.boolean();
try check_range(gpa, &ix, .{ .int32 = a }, if (use_lo) .{ .int32 = lo_v } else null, if (use_hi) .{ .int32 = hi_v } else null, facts.items, alive.items);
}
// 3. Delete every 3rd doc in random order (empty-leaf cascades,
// one-child internals).
var order: std.ArrayListUnmanaged(usize) = .empty;
for (0..N + M) |i| if (i % 3 == 0) try order.append(gpa, i);
rand.shuffle(usize, order.items);
for (order.items) |i| {
const a = facts.items[i].a;
const b = facts.items[i].b;
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
ix.remove_doc(gpa, &d, ids.items[i]);
alive.items[i] = false;
}
std.debug.print("after deletes: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
if (ix.count() != (N + M) - (N + M) / 3 - 1) return error.BadCount;
for (0..500) |_| {
const a = rand.intRangeAtMost(i32, 0, 99);
const lo_v = rand.intRangeAtMost(i32, -10, 1009);
const hi_v = rand.intRangeAtMost(i32, -10, 1009);
const use_lo = rand.boolean();
const use_hi = rand.boolean();
try check_range(gpa, &ix, .{ .int32 = a }, if (use_lo) .{ .int32 = lo_v } else null, if (use_hi) .{ .int32 = hi_v } else null, facts.items, alive.items);
}
// 4. Equality lookups still exact.
for (0..300) |_| {
const a = rand.intRangeAtMost(i32, 0, 99);
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .int32 = a }}, &out);
var expected: usize = 0;
for (facts.items, 0..) |f, fi| { if (alive.items[fi] and f.a == a) expected += 1; }
if (out.items.len != expected) {
std.debug.print("EQ MISMATCH a={d}: got {d} want {d}\n", .{ a, out.items.len, expected });
return error.BadCount;
}
}
// 5. Delete everything (empty-leaf cascades, one-child internals),
// then verify the tree still works for fresh inserts.
var live: std.ArrayListUnmanaged(usize) = .empty;
defer live.deinit(gpa);
for (facts.items, 0..) |_, i| if (alive.items[i]) try live.append(gpa, i);
rand.shuffle(usize, live.items);
var remaining = live.items.len;
for (live.items) |i| {
const a = facts.items[i].a;
const b = facts.items[i].b;
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
ix.remove_doc(gpa, &d, ids.items[i]);
remaining -= 1;
if (ix.count() != remaining) {
std.debug.print("COUNT MISMATCH during drain: {d} != {d}\n", .{ ix.count(), remaining });
return error.BadCount;
}
}
std.debug.print("after full drain: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
if (ix.count() != 0) return error.BadCount;
// The drained tree still accepts and finds entries.
pairs[0] = .{ .key = "a", .value = .{ .int32 = 7 } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = 42 } };
const d2 = doc_of(&pairs);
_ = try ix.add_doc(gpa, &d2, "final", false);
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .int32 = 7 }}, &out);
if (out.items.len != 1) return error.BadCount;
var it = ix.iter();
if (it.next() == null) return error.BadCount;
if (it.next() != null) return error.BadCount;
std.debug.print("STRESS OK\n", .{});
}