index: B+tree over the encoded keys (roadmap item 1)
Replace Index.entries (one sorted array) with a B+tree so writes into an already-built index stop being quadratic. Nodes are fixed 4 KiB slotted pages in a flat u32-addressed ArrayListUnmanaged(Node); records longer than a quarter page spill to an append-only overflow slab (BSON strings reach 16 MB). Leaves are doubly linked for ordered iteration; the flat node array stays one contiguous byte range for a later checkpoint. Insertion descends by separator and splits leaves/internals upward, promoting keys via a stable copy (a nested split can otherwise clobber the promoted-key scratch). Deletion does not rebalance: emptied leaves are unlinked and dropped from their parent, internal nodes may carry one child, and dead pages are abandoned in place (node memory peaks at the tree's peak size, exactly what the old array's capacity did). Lookups are lower-bound seeks plus leaf-chain band scans, so equal keys may span leaves freely. Bulk build (append_doc_entries + finish_bulk) sorts a staging array and packs leaves bottom-up. reserve_for now takes the built entries and reserves exact overflow bytes plus a worst-case node count, keeping insert_entries infallible after the log append. db.zig: TTL sweep now seeks the minimum-datetime encoded key and walks the contiguous datetime band, stopping at the cutoff or type change. Measured (tests/e2e/results/phase2.txt): updateMany 17.3 -> 1.8 ms (2.8x slower than MongoDB -> 3.7x faster), createIndex 62 -> 51 ms. Verified: unit suite ReleaseFast/ReleaseSafe/Debug (incl. the existing lookup_range and remove_doc differentials, plus a new incremental insert/remove differential against a brute-force model), the crash pair, e2e3/e2e4/e2e6, and dev stress tests for depth-2 splits, full drains, and spilled records through internal levels.
This commit is contained in:
74
src/spill.zig
Normal file
74
src/spill.zig
Normal file
@@ -0,0 +1,74 @@
|
||||
// Dev stress test for the overflow slab (records > 1024 bytes):
|
||||
// zig run -O ReleaseFast src/spill.zig
|
||||
// Keys straddling the spill threshold (10 B .. 100 KB) through insert,
|
||||
// lookup, delete and iteration.
|
||||
const std = @import("std");
|
||||
const index = @import("index.zig");
|
||||
const bson = @import("bson.zig");
|
||||
|
||||
fn doc_of(pairs: []const bson.Pair) bson.Document {
|
||||
return .{ .arena = undefined, .pairs = pairs };
|
||||
}
|
||||
|
||||
pub fn main() !void {
|
||||
const gpa = std.heap.page_allocator;
|
||||
var prng = std.Random.DefaultPrng.init(0x1234_5678);
|
||||
const rand = prng.random();
|
||||
|
||||
var keys = [_]index.IndexKey{.{ .path = "tag", .descending = false }};
|
||||
var ix = try index.Index.init(gpa, "tag", &keys, false, false, null);
|
||||
|
||||
// Keys straddling the spill threshold: inline, exactly at the limit,
|
||||
// just over, and one very long. Each id is a short static string.
|
||||
const lens = [_]usize{ 10, 1023, 1024, 1025, 2000, 100_000 };
|
||||
var strings: [lens.len][]u8 = undefined;
|
||||
var docs: [lens.len]bson.Document = undefined;
|
||||
var pairs: [2]bson.Pair = undefined;
|
||||
for (lens, 0..) |len, i| {
|
||||
strings[i] = try gpa.alloc(u8, len);
|
||||
for (strings[i]) |*c| c.* = 'a' + @as(u8, @intCast(rand.intRangeAtMost(u8, 0, 25)));
|
||||
// add a distinguishing suffix so keys are unique
|
||||
std.mem.copyForwards(u8, strings[i][len - 4 ..], &[_]u8{ @intCast(i), 0xff, 0x00, 0x00 });
|
||||
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
|
||||
pairs[1] = .{ .key = "tag", .value = .{ .string = strings[i] } };
|
||||
docs[i] = doc_of(&pairs);
|
||||
_ = try ix.add_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) }, true);
|
||||
}
|
||||
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
|
||||
if (ix.overflow.items.len < 100_000) return error.NoSpill;
|
||||
|
||||
// Every entry is found by exact key.
|
||||
for (lens, 0..) |_, i| {
|
||||
var out: std.ArrayListUnmanaged([]const u8) = .empty;
|
||||
defer out.deinit(gpa);
|
||||
try ix.lookup_eq(gpa, &.{.{ .string = strings[i] }}, &out);
|
||||
if (out.items.len != 1) {
|
||||
std.debug.print("lookup {d} got {d}\n", .{ i, out.items.len });
|
||||
return error.Bad;
|
||||
}
|
||||
if (!std.mem.eql(u8, out.items[0], &[_]u8{ 'i', 'd', @intCast(i + 1) })) return error.Bad;
|
||||
}
|
||||
|
||||
// Delete the spilled ones and the inline ones alternately.
|
||||
for (lens, 0..) |_, i| {
|
||||
if (i % 2 == 0) continue;
|
||||
ix.remove_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) });
|
||||
}
|
||||
if (ix.count() != 3) return error.Bad;
|
||||
for (lens, 0..) |_, i| {
|
||||
if (i % 2 == 0) continue;
|
||||
var out: std.ArrayListUnmanaged([]const u8) = .empty;
|
||||
defer out.deinit(gpa);
|
||||
try ix.lookup_eq(gpa, &.{.{ .string = strings[i] }}, &out);
|
||||
if (out.items.len != 0) return error.Bad;
|
||||
}
|
||||
// Iteration still sees the survivors in order.
|
||||
var it = ix.iter();
|
||||
var seen: usize = 0;
|
||||
while (it.next()) |e| {
|
||||
seen += 1;
|
||||
_ = e;
|
||||
}
|
||||
if (seen != 3) return error.Bad;
|
||||
std.debug.print("SPILL OK (seen={d})\n", .{seen});
|
||||
}
|
||||
Reference in New Issue
Block a user