dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index: B+tree over the encoded keys (roadmap item 1)

Browse Source 
Replace Index.entries (one sorted array) with a B+tree so writes into an
already-built index stop being quadratic. Nodes are fixed 4 KiB slotted
pages in a flat u32-addressed ArrayListUnmanaged(Node); records longer
than a quarter page spill to an append-only overflow slab (BSON strings
reach 16 MB). Leaves are doubly linked for ordered iteration; the flat
node array stays one contiguous byte range for a later checkpoint.

Insertion descends by separator and splits leaves/internals upward,
promoting keys via a stable copy (a nested split can otherwise clobber
the promoted-key scratch). Deletion does not rebalance: emptied leaves
are unlinked and dropped from their parent, internal nodes may carry one
child, and dead pages are abandoned in place (node memory peaks at the
tree's peak size, exactly what the old array's capacity did). Lookups
are lower-bound seeks plus leaf-chain band scans, so equal keys may
span leaves freely. Bulk build (append_doc_entries + finish_bulk) sorts
a staging array and packs leaves bottom-up. reserve_for now takes the
built entries and reserves exact overflow bytes plus a worst-case node
count, keeping insert_entries infallible after the log append.

db.zig: TTL sweep now seeks the minimum-datetime encoded key and walks
the contiguous datetime band, stopping at the cutoff or type change.

Measured (tests/e2e/results/phase2.txt): updateMany 17.3 -> 1.8 ms
(2.8x slower than MongoDB -> 3.7x faster), createIndex 62 -> 51 ms.

Verified: unit suite ReleaseFast/ReleaseSafe/Debug (incl. the existing
lookup_range and remove_doc differentials, plus a new incremental
insert/remove differential against a brute-force model), the crash pair,
e2e3/e2e4/e2e6, and dev stress tests for depth-2 splits, full drains,
and spilled records through internal levels.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 21:10:25 +03:00
parent 71112b0ff7
commit 61fe952125
8 changed files with 1450 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
src/db.zig
View File

@@ -255,10 +255,10 @@ pub const Engine = struct {
};
}
// 4. Reserve entry capacity — the last fallible step, so the entry
// 4. Reserve tree capacity — the last fallible step, so the entry
// insertion after the log append is infallible.
for (built_list.items) |*b| {
try b.ix.reserve_for(self.gpa, b.built.entries.items.len);
try b.ix.reserve_for(self.gpa, b.built.entries.items);
}
// 5. Log (and sync) before anything becomes visible.
@@ -365,7 +365,7 @@ pub const Engine = struct {
while (doc_it.next()) |entry| {
try ix.append_doc_entries(self.gpa, entry.value_ptr.*, entry.key_ptr.*);
}
_ = try ix.finish_bulk(true);
_ = try ix.finish_bulk(self.gpa, true);
// Reserve the collection slot, then persist and publish.
try coll.indexes.ensureUnusedCapacity(self.gpa, 1);
@@ -428,16 +428,16 @@ pub const Engine = struct {
for (coll_entry.value_ptr.indexes.items) |*ix| {
const ttl = ix.ttl orelse continue;
const cutoff: i128 = @as(i128, now_ms) - @as(i128, ttl) * 1000;
for (ix.entries.items) |e| {
// Still a linear walk: the type test cannot be a
// one-sided range lookup, because bson compare order
// ranks datetime above null, numbers and strings, so
// a datetime upper bound would also select every
// value of a lesser type. (Datetimes are contiguous
// in that order, so a two-sided band lookup would
// work — that comes with the tree.)
const ms = bson.encoded_leading_datetime(e.key) orelse continue;
if (@as(i128, ms) > cutoff) continue;
// bson compare order ranks datetime above null, numbers
// and strings and below only timestamp and maxKey, so
// datetimes form a contiguous band in the encoded key
// order: seek the minimum datetime and stop when the
// leading type changes or the cutoff is passed.
const min_dt = [_]u8{ bson.encoded_datetime_tag, 0, 0, 0, 0, 0, 0, 0, 0 };
var it = ix.seek(&min_dt);
while (it.next()) |e| {
const ms = bson.encoded_leading_datetime(e.key) orelse break;
if (@as(i128, ms) > cutoff) break;
try ids.append(self.gpa, try self.gpa.dupe(u8, e.id));
}
}
@@ -624,7 +624,7 @@ pub const Engine = struct {
var coll_it = db_entry.value_ptr.collections.iterator();
while (coll_it.next()) |coll_entry| {
for (coll_entry.value_ptr.indexes.items) |*ix| {
if (ix.entries.items.len > 0) continue; // defensive
if (ix.count() > 0) continue; // defensive
var doc_it = coll_entry.value_ptr.docs.iterator();
while (doc_it.next()) |doc_entry| {
ix.append_doc_entries(self.gpa, doc_entry.value_ptr.*, doc_entry.key_ptr.*) catch |err| switch (err) {
@@ -636,7 +636,7 @@ pub const Engine = struct {
};
}
// Tolerated, not enforced: the database must always open.
if (try ix.finish_bulk(false)) {
if (try ix.finish_bulk(self.gpa, false)) {
std.debug.print("mongo-lite: WARNING: unique index '{s}' has duplicate keys in existing data; duplicates not enforced for existing documents\n", .{ix.name});
}
}
@@ -1369,12 +1369,12 @@ test "ttl_sweep deletes expired documents and the deletion survives reopen" {
}
const coll = engine.get_collection("app", "sessions").?;
try testing.expectEqual(@as(usize, 6), coll.docs.count());
try testing.expectEqual(@as(usize, 6), coll.indexes.items[0].entries.items.len);
try testing.expectEqual(@as(usize, 6), coll.indexes.items[0].count());
// The cutoff is inclusive: doc 2 goes with doc 1.
try testing.expectEqual(@as(usize, 2), try engine.ttl_sweep(now_ms));
try testing.expectEqual(@as(usize, 4), coll.docs.count());
try testing.expectEqual(@as(usize, 4), coll.indexes.items[0].entries.items.len);
try testing.expectEqual(@as(usize, 4), coll.indexes.items[0].count());
try testing.expectEqual(@as(usize, 0), try index_count(gpa, &engine, "app", "sessions", "expireAt_1", .{ .datetime = now_ms - 120_000 }));
// The string and the missing field are untouched by any sweep.
try testing.expectEqual(@as(usize, 1), try index_count(gpa, &engine, "app", "sessions", "expireAt_1", .{ .string = "tomorrow" }));
@@ -1397,7 +1397,7 @@ test "ttl_sweep deletes expired documents and the deletion survives reopen" {
try testing.expectEqual(@as(usize, 3), coll.docs.count());
try testing.expectEqual(@as(usize, 1), coll.indexes.items.len);
try testing.expectEqual(@as(?i64, 60), coll.indexes.items[0].ttl);
try testing.expectEqual(@as(usize, 3), coll.indexes.items[0].entries.items.len);
try testing.expectEqual(@as(usize, 3), coll.indexes.items[0].count());
for ([_]i32{ 1, 2, 3 }) |id| {
const id_key = try bson.serialize_value(gpa, bson.Value{ .int32 = id });
defer gpa.free(id_key);