index: B+tree over the encoded keys (roadmap item 1)
Replace Index.entries (one sorted array) with a B+tree so writes into an already-built index stop being quadratic. Nodes are fixed 4 KiB slotted pages in a flat u32-addressed ArrayListUnmanaged(Node); records longer than a quarter page spill to an append-only overflow slab (BSON strings reach 16 MB). Leaves are doubly linked for ordered iteration; the flat node array stays one contiguous byte range for a later checkpoint. Insertion descends by separator and splits leaves/internals upward, promoting keys via a stable copy (a nested split can otherwise clobber the promoted-key scratch). Deletion does not rebalance: emptied leaves are unlinked and dropped from their parent, internal nodes may carry one child, and dead pages are abandoned in place (node memory peaks at the tree's peak size, exactly what the old array's capacity did). Lookups are lower-bound seeks plus leaf-chain band scans, so equal keys may span leaves freely. Bulk build (append_doc_entries + finish_bulk) sorts a staging array and packs leaves bottom-up. reserve_for now takes the built entries and reserves exact overflow bytes plus a worst-case node count, keeping insert_entries infallible after the log append. db.zig: TTL sweep now seeks the minimum-datetime encoded key and walks the contiguous datetime band, stopping at the cutoff or type change. Measured (tests/e2e/results/phase2.txt): updateMany 17.3 -> 1.8 ms (2.8x slower than MongoDB -> 3.7x faster), createIndex 62 -> 51 ms. Verified: unit suite ReleaseFast/ReleaseSafe/Debug (incl. the existing lookup_range and remove_doc differentials, plus a new incremental insert/remove differential against a brute-force model), the crash pair, e2e3/e2e4/e2e6, and dev stress tests for depth-2 splits, full drains, and spilled records through internal levels.
This commit is contained in:
88
src/spill2.zig
Normal file
88
src/spill2.zig
Normal file
@@ -0,0 +1,88 @@
|
||||
// Dev stress test: spilled records through leaf splits and internal levels.
|
||||
// zig run -O ReleaseFast src/spill2.zig
|
||||
// 5000 docs with 2 KiB keys: every record spills; the tree still answers
|
||||
// exact lookups and deletes half of them.
|
||||
const std = @import("std");
|
||||
const index = @import("index.zig");
|
||||
const bson = @import("bson.zig");
|
||||
|
||||
fn doc_of(pairs: []const bson.Pair) bson.Document {
|
||||
return .{ .arena = undefined, .pairs = pairs };
|
||||
}
|
||||
|
||||
pub fn main() !void {
|
||||
const gpa = std.heap.page_allocator;
|
||||
|
||||
var keys = [_]index.IndexKey{.{ .path = "tag", .descending = false }};
|
||||
var ix = try index.Index.init(gpa, "tag", &keys, false, false, null);
|
||||
|
||||
// 5000 docs, each with a 2 KiB key: spills on every record, forcing
|
||||
// leaves and internal nodes to hold overflow references.
|
||||
const N = 5000;
|
||||
var ids: std.ArrayListUnmanaged([]u8) = .empty;
|
||||
defer {
|
||||
for (ids.items) |x| gpa.free(x);
|
||||
ids.deinit(gpa);
|
||||
}
|
||||
var pairs: [2]bson.Pair = undefined;
|
||||
var buf = try gpa.alloc(u8, 2000);
|
||||
defer gpa.free(buf);
|
||||
var facts: std.ArrayListUnmanaged(struct { key: []u8 }) = .empty;
|
||||
defer {
|
||||
for (facts.items) |f| gpa.free(f.key);
|
||||
facts.deinit(gpa);
|
||||
}
|
||||
for (0..N) |i| {
|
||||
for (buf) |*c| c.* = 'x';
|
||||
// unique suffix
|
||||
std.mem.writeInt(u32, buf[0..4], @intCast(i), .little);
|
||||
const key = try gpa.dupe(u8, buf);
|
||||
try facts.append(gpa, .{ .key = key });
|
||||
const id = try std.fmt.allocPrint(gpa, "id{d}", .{i});
|
||||
try ids.append(gpa, id);
|
||||
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
|
||||
pairs[1] = .{ .key = "tag", .value = .{ .string = key } };
|
||||
const d = doc_of(&pairs);
|
||||
_ = try ix.add_doc(gpa, &d, id, false);
|
||||
}
|
||||
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
|
||||
if (ix.count() != N) return error.Bad;
|
||||
|
||||
// Spot-check exact lookups.
|
||||
var prng2 = std.Random.DefaultPrng.init(0xabc);
|
||||
const rand2 = prng2.random();
|
||||
for (0..300) |_| {
|
||||
const i = rand2.intRangeAtMost(usize, 0, N - 1);
|
||||
var out: std.ArrayListUnmanaged([]const u8) = .empty;
|
||||
defer out.deinit(gpa);
|
||||
try ix.lookup_eq(gpa, &.{.{ .string = facts.items[i].key }}, &out);
|
||||
if (out.items.len != 1 or !std.mem.eql(u8, out.items[0], ids.items[i])) {
|
||||
std.debug.print("lookup mismatch at {d}\n", .{i});
|
||||
return error.Bad;
|
||||
}
|
||||
}
|
||||
// Delete half (random), verify count and no leftover.
|
||||
var order: std.ArrayListUnmanaged(usize) = .empty;
|
||||
defer order.deinit(gpa);
|
||||
for (0..N) |i| if (i % 2 == 0) try order.append(gpa, i);
|
||||
rand2.shuffle(usize, order.items);
|
||||
var removed: usize = 0;
|
||||
for (order.items) |i| {
|
||||
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
|
||||
pairs[1] = .{ .key = "tag", .value = .{ .string = facts.items[i].key } };
|
||||
const d = doc_of(&pairs);
|
||||
ix.remove_doc(gpa, &d, ids.items[i]);
|
||||
removed += 1;
|
||||
if (ix.count() != N - removed) {
|
||||
std.debug.print("count mismatch at {d}: {d} != {d}\n", .{ i, ix.count(), N - removed });
|
||||
return error.Bad;
|
||||
}
|
||||
}
|
||||
for (order.items) |i| {
|
||||
var out: std.ArrayListUnmanaged([]const u8) = .empty;
|
||||
defer out.deinit(gpa);
|
||||
try ix.lookup_eq(gpa, &.{.{ .string = facts.items[i].key }}, &out);
|
||||
if (out.items.len != 0) return error.Bad;
|
||||
}
|
||||
std.debug.print("SPILL2 OK (leaves={d} depth={d})\n", .{ ix.leaf_count, ix.depth });
|
||||
}
|
||||
Reference in New Issue
Block a user