dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index: ordered _id index (roadmap item 2)

Browse Source 
Give every Collection an implicit _id_ index (a normal Index with keys
[_id: 1]) so _id equality, $in, ranges and sorts stop depending on the
docs-map hash or a full scan. Kept out of the secondary indexes list, so
listIndexes/dropIndexes/createIndex and the log format are unchanged (no
index_create record, no double listing) and e2e3.js passes unmodified.

Maintained in upsert through the same reserve-then-insert protocol as
the secondaries, removed in evict_doc, and rebuilt after replay by
build_all_indexes alongside them (never maintained mid-replay, so a
failed add can't leave the index under-approximating). index.plan now
takes it as a separate argument. Its keys are canonical
(bson.encode_key gives int32 1, int64 1 and double 1.0 identical bytes),
so the serialization-guarded docs-map fast path (plan_id,
value_fast_path_safe and friends) is deleted.

Measured (tests/e2e/results/phase3.txt): sort({_id:-1}).limit(20) 6.2 ->
2.4 ms (2.3x slower than MongoDB -> parity); integer/string _id point
lookups, $in and ranges verified against the tree. Unit suite in all
three optimize modes, the crash pair, e2e3/e2e4/e2e6.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 21:18:40 +03:00
parent 61fe952125
commit 58914a69c3
6 changed files with 267 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
src/db.zig
View File

@@ -12,10 +12,23 @@ const index = @import("index.zig");
pub const Collection = struct {
docs: std.StringHashMapUnmanaged(*bson.Document),
/// Secondary indexes (persisted through the log).
indexes: std.ArrayListUnmanaged(index.Index),
/// The implicit _id_ index: every document has an _id and it is not
/// sparse, so entry count equals document count and a full scan of it
/// cannot miss a document — which is what the sort planner's full-scan
/// plan relies on. Kept out of `indexes` so the listing/drop commands
/// and the log format are unchanged (it is rebuilt on open like
/// everything else). `bson.encode_key` keys are canonical, so it also
/// replaces the old serialization-guarded docs-map fast path for
/// integer/string/etc. _id lookups.
id_index: index.Index,
fn init() Collection {
return .{ .docs = .empty, .indexes = .empty };
fn init(gpa: std.mem.Allocator) !Collection {
var self: Collection = .{ .docs = .empty, .indexes = .empty, .id_index = undefined };
const keys = [_]index.IndexKey{.{ .path = "_id", .descending = false }};
self.id_index = try index.Index.init(gpa, "_id_", &keys, false, false, null);
return self;
}
/// The secondary index with this name, or null. The single by-name
@@ -108,6 +121,7 @@ pub const Engine = struct {
// Dropping a collection turns all of its records into garbage.
self.live_docs -= coll.docs.count();
self.dead_docs += coll.docs.count();
coll.id_index.deinit(self.gpa);
for (coll.indexes.items) |*ix| ix.deinit(self.gpa);
coll.indexes.deinit(self.gpa);
var doc_it = coll.docs.iterator();
@@ -139,6 +153,7 @@ pub const Engine = struct {
/// regenerating them from it, which is far cheaper than scanning.
fn evict_doc(self: *Engine, coll: *Collection, id_key: []const u8) void {
const old = coll.docs.fetchRemove(id_key) orelse return;
coll.id_index.remove_doc(self.gpa, old.value, old.key);
for (coll.indexes.items) |*ix| ix.remove_doc(self.gpa, old.value, old.key);
old.value.*.deinit();
self.gpa.destroy(old.value);
@@ -241,6 +256,15 @@ pub const Engine = struct {
return err;
};
}
{
// The implicit _id_ index, through the same protocol: reserved
// before the log append, inserted infallibly after it.
var built = try coll.id_index.build_entries(self.gpa, owned, id_key);
built_list.append(self.gpa, .{ .built = built, .ix = &coll.id_index }) catch |err| {
built.deinit(self.gpa);
return err;
};
}
// 2. The _id check, mirroring the pre-index behavior.
if (mode == .insert and coll.docs.contains(id_key)) return error.DuplicateKey;
@@ -494,7 +518,9 @@ pub const Engine = struct {
if (db.collections.getPtr(coll_name)) |coll| return coll;
const coll_key = try self.gpa.dupe(u8, coll_name);
errdefer self.gpa.free(coll_key);
try db.collections.put(self.gpa, coll_key, Collection.init());
var new_coll = try Collection.init(self.gpa);
errdefer new_coll.id_index.deinit(self.gpa);
try db.collections.put(self.gpa, coll_key, new_coll);
return db.collections.getPtr(coll_name) orelse unreachable;
}
@@ -624,26 +650,37 @@ pub const Engine = struct {
var coll_it = db_entry.value_ptr.collections.iterator();
while (coll_it.next()) |coll_entry| {
for (coll_entry.value_ptr.indexes.items) |*ix| {
if (ix.count() > 0) continue; // defensive
var doc_it = coll_entry.value_ptr.docs.iterator();
while (doc_it.next()) |doc_entry| {
ix.append_doc_entries(self.gpa, doc_entry.value_ptr.*, doc_entry.key_ptr.*) catch |err| switch (err) {
error.ParallelArrays => {
std.debug.print("mongo-lite: WARNING: index '{s}' cannot index an existing document; entry skipped\n", .{ix.name});
continue;
},
else => return err,
};
}
// Tolerated, not enforced: the database must always open.
if (try ix.finish_bulk(self.gpa, false)) {
std.debug.print("mongo-lite: WARNING: unique index '{s}' has duplicate keys in existing data; duplicates not enforced for existing documents\n", .{ix.name});
}
try self.rebuild_index(coll_entry.value_ptr, ix);
}
try self.rebuild_index(coll_entry.value_ptr, &coll_entry.value_ptr.id_index);
}
}
}
/// Rebuild one index from the live documents. Runs after replay, so it
/// is order-independent; indexes already holding entries (maintained
/// live) are skipped defensively. A duplicate under a unique index logs
/// a loud warning and keeps the index (still correct as a candidate
/// generator; future writes are still enforced) — the database always
/// opens, leaving dropIndexes as an in-band recovery path.
fn rebuild_index(self: *Engine, coll: *Collection, ix: *index.Index) !void {
if (ix.count() > 0) return; // defensive
var doc_it = coll.docs.iterator();
while (doc_it.next()) |doc_entry| {
ix.append_doc_entries(self.gpa, doc_entry.value_ptr.*, doc_entry.key_ptr.*) catch |err| switch (err) {
error.ParallelArrays => {
std.debug.print("mongo-lite: WARNING: index '{s}' cannot index an existing document; entry skipped\n", .{ix.name});
continue;
},
else => return err,
};
}
// Tolerated, not enforced: the database must always open.
if (try ix.finish_bulk(self.gpa, false)) {
std.debug.print("mongo-lite: WARNING: unique index '{s}' has duplicate keys in existing data; duplicates not enforced for existing documents\n", .{ix.name});
}
}
/// Register an (empty) index from a persisted spec document. A repeated
/// create record for the same name is an idempotent no-op.
fn register_index_from_spec(self: *Engine, coll: *Collection, spec_doc: *const bson.Document) !void {
@@ -721,6 +758,8 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an
self.live_docs += 1;
key_owned = true;
stored = true;
// The _id_ entry is added after replay, in build_all_indexes,
// together with the secondary indexes.
},
storage.record_type_delete => self.evict_doc(coll, id_key),
else => {},