dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

storage: byte documents in a per-collection slab (roadmap item 4)

Browse Source 
Documents live as canonical BSON bytes in a segmented per-collection slab
(fixed 8 MiB segments keep capacity slack under one segment); the docs map
holds flat offsets that stay valid across segment growth, and removed
documents leave garbage bytes until compaction rewrites. The per-document
ArenaAllocator and its second full Pair-tree copy are gone.

The matcher walks the stored bytes directly, skipping by length any field
the filter does not name (a new bson byte-walker: element_key, skip_value,
read_value with borrowed leaves, get_at, and a borrowed spine parse). The
byte matcher is differential-tested against the tree matcher on a corpus
and shares its operator logic. Stored documents are never materialized on
the scan path or in aggregate $match; $group reads group keys and sums
straight off the bytes. Sort, projection, findAndModify, updates and
index entry generation use a borrowed spine into the slab (or the byte
collector, which also replaced collect_values in build_entries). The
compaction threshold now counts uncompressed data volume, since a
compressed log would otherwise never trigger.

Measured (tests/e2e/results/phase5.txt): server RSS 1979 -> 539 MB (2.4x
smaller than MongoDB; phase1 baseline 2.0 GB), range-scan 22.5 -> ~12 ms
(parity, best run faster than MongoDB), proj 4.1 -> 3.4 ms, createIndex
parity. Verified: unit suite in all three modes with zero leaks, the
crash pair, e2e6, and the stress/spill programs.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 22:15:07 +03:00
parent b4585106f1
commit 570900a6ef
12 changed files with 985 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/spill.zig
View File

@@ -6,8 +6,11 @@ const std = @import("std");
const index = @import("index.zig");
const bson = @import("bson.zig");
fn doc_of(pairs: []const bson.Pair) bson.Document {
return .{ .arena = undefined, .pairs = pairs };
fn doc_of(gpa: std.mem.Allocator, pairs: []const bson.Pair) ![]u8 {
var out: std.ArrayListUnmanaged(u8) = .empty;
defer out.deinit(gpa);
try bson.write_doc(pairs, gpa, &out);
return out.toOwnedSlice(gpa);
}
pub fn main() !void {
@@ -22,7 +25,7 @@ pub fn main() !void {
// just over, and one very long. Each id is a short static string.
const lens = [_]usize{ 10, 1023, 1024, 1025, 2000, 100_000 };
var strings: [lens.len][]u8 = undefined;
var docs: [lens.len]bson.Document = undefined;
var docs: [lens.len][]u8 = undefined;
var pairs: [2]bson.Pair = undefined;
for (lens, 0..) |len, i| {
strings[i] = try gpa.alloc(u8, len);
@@ -31,8 +34,8 @@ pub fn main() !void {
std.mem.copyForwards(u8, strings[i][len - 4 ..], &[_]u8{ @intCast(i), 0xff, 0x00, 0x00 });
pairs[0] = .{ .key = "_id", .value = .{ .int32 = @intCast(i) } };
pairs[1] = .{ .key = "tag", .value = .{ .string = strings[i] } };
docs[i] = doc_of(&pairs);
_ = try ix.add_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) }, true);
docs[i] = try doc_of(gpa, &pairs);
_ = try ix.add_doc(gpa, docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) }, true);
}
std.debug.print("count={d} leaves={d} depth={d} overflow={d}\n", .{ ix.count(), ix.leaf_count, ix.depth, ix.overflow.items.len });
if (ix.overflow.items.len < 100_000) return error.NoSpill;
@@ -52,7 +55,7 @@ pub fn main() !void {
// Delete the spilled ones and the inline ones alternately.
for (lens, 0..) |_, i| {
if (i % 2 == 0) continue;
ix.remove_doc(gpa, &docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) });
ix.remove_doc(gpa, docs[i], &[_]u8{ 'i', 'd', @intCast(i + 1) });
}
if (ix.count() != 3) return error.Bad;
for (lens, 0..) |_, i| {