Files
MultiforaDB/src/main.zig
Aleksey Shakhmatov 556ad7dc86 storage/db: XxHash3 record integrity, garbage-ratio compaction
Two streams of work land together: they are interleaved in storage.zig
and db.zig and only build as a unit.

Already in the working tree before this session:
  - ReleaseFast as the default zig build (Debug was 10-200x slower)
  - group commit: one fsync per write command instead of per document
  - plan_id returned a pointer to a stack temporary; ReleaseFast read
    garbage and silently broke findOne({_id: ObjectId})
  - perf suite: big.js, compare.js, compare-run.sh, e2e6.js

Phase 1 performance work:

Record integrity hash CRC32 -> XxHash3. std.hash.Crc32 is the
table-driven byte-at-a-time Crc32IsoHdlc, measured at 408 MB/s against
XxHash3's 31 GB/s: 38us versus 0.5us on a 16 KiB document, which was
about two thirds of the entire bulk-insert cost. The record header
grows from u32 crc to u64 hash (header_len 20 -> 24), a breaking
format change. Bulk insert 260 -> 700 MB/s, reopen 1.1 -> 0.5s.

Compaction fsynced once per live document, because Log.open leaves
defer_sync false and compact never set it. It now issues one sync for
the whole rewrite, before the rename that publishes it.

Compaction triggers on the share of the log that is garbage
(live_docs/dead_docs, maintained at evict_doc, the single point where
a document dies) rather than on bytes appended. A fixed byte count is
wrong in both directions: a 1 GB bulk load holds no garbage at all yet
would compact ~64 times under the 16 MiB default, rewriting 1 GB each
time, while a small collection rewritten in place accumulates garbage
indefinitely without ever reaching the count. Pure inserts now never
compact, and the file stays near 1.25x the live data. Bulk load at the
default threshold: 41.6 -> 702.7 MB/s.

remove() never called maybe_compact, so a delete-heavy workload grew
the log without bound.

e2e6's compaction check required the file to bloat past 30 MiB before
being reclaimed, which encoded the old policy and failed on strictly
better behaviour (ends at 15.8 MB against ~12 MB live, was ~30 MB). It
now asserts the file ends near the live size and peaked well above it,
which does not depend on when the trigger fires. Sampling interval 50
-> 10ms: the operations now finish inside the old window.

Verified: 70 unit tests under both ReleaseFast and ReleaseSafe, e2e
29/16/17/3/2 checks, the crash-a/kill -9/crash-b pair, and e2e6 72/72
across three consecutive runs.
2026-08-02 18:20:40 +03:00

116 lines
4.6 KiB
Zig

const std = @import("std");
const mongo = @import("mongo");
const usage =
\\mongo-lite — lightweight MongoDB-compatible document database
\\
\\usage: mongo-lite [options]
\\ --port <n> listen port (default 27017)
\\ --bind <ip> bind address (default 127.0.0.1)
\\ --db <path> database file (default mongo-lite.log)
\\ --ttl-sweep-secs <n>
\\ seconds between TTL index sweeps (default 60, 0 disables)
\\ --compact-threshold <bytes>
\\ minimum log bytes between compactions; suffixes k/m/g
\\ (default 16m). The actual trigger also scales with the
\\ live data size, so total rewrite traffic stays linear
\\ no matter how large the collection grows.
\\ --help show this help
\\
;
/// Parse a size with k/m/g suffix ("16m", "1g", "512k"), or null.
fn parse_size_suffix(v: []const u8) ?u64 {
const s = std.mem.trim(u8, v, " \t");
if (s.len == 0) return null;
var mult: u64 = 1;
var num_part = s;
switch (s[s.len - 1]) {
'k', 'K' => {
mult = 1024;
num_part = s[0 .. s.len - 1];
},
'm', 'M' => {
mult = 1024 * 1024;
num_part = s[0 .. s.len - 1];
},
'g', 'G' => {
mult = 1024 * 1024 * 1024;
num_part = s[0 .. s.len - 1];
},
else => {},
}
const n = std.fmt.parseInt(u64, num_part, 10) catch return null;
return n * mult;
}
pub fn main(init: std.process.Init) !void {
var port: u16 = 27017;
var bind_ip: []const u8 = "127.0.0.1";
var db_path: []const u8 = "mongo-lite.log";
var ttl_sweep_secs: i64 = 60;
var compact_threshold: u64 = 16 * 1024 * 1024;
var it = std.process.Args.Iterator.init(init.minimal.args);
defer it.deinit();
_ = it.next(); // program name
while (it.next()) |arg| {
if (std.mem.eql(u8, arg, "--port")) {
const v = it.next() orelse return error.MissingValue;
port = std.fmt.parseInt(u16, v, 10) catch {
std.debug.print("mongo-lite: invalid port '{s}'\n", .{v});
return error.InvalidPort;
};
} else if (std.mem.eql(u8, arg, "--bind")) {
bind_ip = it.next() orelse return error.MissingValue;
} else if (std.mem.eql(u8, arg, "--db")) {
db_path = it.next() orelse return error.MissingValue;
} else if (std.mem.eql(u8, arg, "--ttl-sweep-secs")) {
const v = it.next() orelse return error.MissingValue;
// i64 is the width std.Io.Duration.fromSeconds takes, so the
// value reaches the sweeper without a cast; negatives are the
// only thing parseInt would otherwise let through.
ttl_sweep_secs = std.fmt.parseInt(i64, v, 10) catch -1;
if (ttl_sweep_secs < 0) {
std.debug.print("mongo-lite: invalid ttl sweep interval '{s}'\n", .{v});
return error.InvalidTtlSweepSecs;
}
} else if (std.mem.eql(u8, arg, "--compact-threshold")) {
const v = it.next() orelse return error.MissingValue;
const parsed = parse_size_suffix(v) orelse {
std.debug.print("mongo-lite: invalid compact threshold '{s}'\n", .{v});
return error.InvalidCompactThreshold;
};
if (parsed < 1024 * 1024) {
std.debug.print("mongo-lite: compact threshold must be at least 1m\n", .{});
return error.InvalidCompactThreshold;
}
compact_threshold = parsed;
} else if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) {
try std.Io.File.writeStreamingAll(.stdout(), init.io, usage);
return;
} else {
std.debug.print("mongo-lite: unknown option '{s}'\n{s}", .{ arg, usage });
return error.UnknownOption;
}
}
const oid_gen = mongo.bson.ObjectIdGen.init(init.io);
var engine = try mongo.db.Engine.open(init.gpa, init.io, db_path);
defer engine.deinit();
engine.compact_threshold = compact_threshold;
std.debug.print("mongo-lite: opened database '{s}' (compact threshold {d})\n", .{ db_path, compact_threshold });
var server = mongo.server.Server{
.gpa = init.gpa,
.port = port,
.bind_ip = bind_ip,
.oid_gen = oid_gen,
.connection_counter = .init(1),
.engine = &engine,
.start_time = std.Io.Timestamp.now(init.io, .real),
.ttl_sweep_secs = ttl_sweep_secs,
};
try server.run();
}