dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

storage: byte documents in a per-collection slab (roadmap item 4)

Browse Source 
Documents live as canonical BSON bytes in a segmented per-collection slab
(fixed 8 MiB segments keep capacity slack under one segment); the docs map
holds flat offsets that stay valid across segment growth, and removed
documents leave garbage bytes until compaction rewrites. The per-document
ArenaAllocator and its second full Pair-tree copy are gone.

The matcher walks the stored bytes directly, skipping by length any field
the filter does not name (a new bson byte-walker: element_key, skip_value,
read_value with borrowed leaves, get_at, and a borrowed spine parse). The
byte matcher is differential-tested against the tree matcher on a corpus
and shares its operator logic. Stored documents are never materialized on
the scan path or in aggregate $match; $group reads group keys and sums
straight off the bytes. Sort, projection, findAndModify, updates and
index entry generation use a borrowed spine into the slab (or the byte
collector, which also replaced collect_values in build_entries). The
compaction threshold now counts uncompressed data volume, since a
compressed log would otherwise never trigger.

Measured (tests/e2e/results/phase5.txt): server RSS 1979 -> 539 MB (2.4x
smaller than MongoDB; phase1 baseline 2.0 GB), range-scan 22.5 -> ~12 ms
(parity, best run faster than MongoDB), proj 4.1 -> 3.4 ms, createIndex
parity. Verified: unit suite in all three modes with zero leaks, the
crash pair, e2e6, and the stress/spill programs.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 22:15:07 +03:00
parent b4585106f1
commit 570900a6ef
12 changed files with 985 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
src/stress.zig
View File

@@ -7,8 +7,11 @@ const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
fn doc_of(gpa: std.mem.Allocator, pairs: []const bson.Pair) ![]u8 {
var out: std.ArrayListUnmanaged(u8) = .empty;
defer out.deinit(gpa);
try bson.write_doc(pairs, gpa, &out);
return out.toOwnedSlice(gpa);
}
const Fact = struct { a: i32, b: i32 };
@@ -54,8 +57,8 @@ pub fn main() !void {
try alive.append(gpa, true);
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
try ix.append_doc_entries(gpa, &d, id);
const d = try doc_of(gpa, &pairs);
try ix.append_doc_entries(gpa, d, id);
}
_ = try ix.finish_bulk(gpa, false);
std.debug.print("bulk: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
@@ -82,8 +85,8 @@ pub fn main() !void {
try alive.append(gpa, true);
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
_ = try ix.add_doc(gpa, &d, id, false);
const d = try doc_of(gpa, &pairs);
_ = try ix.add_doc(gpa, d, id, false);
}
std.debug.print("after inserts: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
if (ix.count() != N + M) return error.BadCount;
@@ -106,8 +109,8 @@ pub fn main() !void {
const b = facts.items[i].b;
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
ix.remove_doc(gpa, &d, ids.items[i]);
const d = try doc_of(gpa, &pairs);
ix.remove_doc(gpa, d, ids.items[i]);
alive.items[i] = false;
}
std.debug.print("after deletes: count={d} leaves={d} depth={d} nodes={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.nodes.items.len });
@@ -147,8 +150,8 @@ pub fn main() !void {
const b = facts.items[i].b;
pairs[0] = .{ .key = "a", .value = .{ .int32 = a } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = b } };
const d = doc_of(&pairs);
ix.remove_doc(gpa, &d, ids.items[i]);
const d = try doc_of(gpa, &pairs);
ix.remove_doc(gpa, d, ids.items[i]);
remaining -= 1;
if (ix.count() != remaining) {
std.debug.print("COUNT MISMATCH during drain: {d} != {d}\n", .{ ix.count(), remaining });
@@ -160,8 +163,9 @@ pub fn main() !void {
// The drained tree still accepts and finds entries.
pairs[0] = .{ .key = "a", .value = .{ .int32 = 7 } };
pairs[1] = .{ .key = "b", .value = .{ .int32 = 42 } };
const d2 = doc_of(&pairs);
_ = try ix.add_doc(gpa, &d2, "final", false);
const d2 = try doc_of(gpa, &pairs);
defer gpa.free(d2);
_ = try ix.add_doc(gpa, d2, "final", false);
var out: std.ArrayListUnmanaged([]const u8) = .empty;
defer out.deinit(gpa);
try ix.lookup_eq(gpa, &.{.{ .int32 = 7 }}, &out);