dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index: secondary index core — entries, search, planner, _id fast path

Browse Source 
Adds src/index.zig with the full secondary-index machinery: entry
generation mirroring field_matches (array value + elements), BSON-order
sorted entries with binary search, compound prefix and range lookups,
unique/sparse options, the query planner (longest equality/$in run +
optional range, $in cartesian cap, sparse/null bail), and the _id_ fast
path guarded against serialization-ambiguous values (numbers, strings,
symbols, codes, opaque payloads).

query.collect_values is now pub so entry generation can mirror it exactly.
storage.zig gains record_type_index_create/drop; lib.zig exports index.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 12:21:22 +03:00
parent 662df9b121
commit a38ddc2f50
10 changed files with 1824 additions and 782 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
src/db.zig
View File

@@ -55,24 +55,43 @@ pub const Engine = struct {
pub fn deinit(self: *Engine) void {
var db_it = self.dbs.iterator();
while (db_it.next()) |db_entry| {
var coll_it = db_entry.value_ptr.collections.iterator();
while (coll_it.next()) |coll_entry| {
var doc_it = coll_entry.value_ptr.docs.iterator();
while (doc_it.next()) |doc_entry| {
doc_entry.value_ptr.*.deinit();
self.gpa.destroy(doc_entry.value_ptr.*);
self.gpa.free(doc_entry.key_ptr.*);
}
coll_entry.value_ptr.docs.deinit(self.gpa);
self.gpa.free(coll_entry.key_ptr.*);
}
db_entry.value_ptr.collections.deinit(self.gpa);
self.free_db(db_entry.value_ptr);
self.gpa.free(db_entry.key_ptr.*);
}
self.dbs.deinit(self.gpa);
self.log.close();
}
/// Free every document in a collection along with its owned _id keys.
fn free_collection(self: *Engine, coll: *Collection) void {
var doc_it = coll.docs.iterator();
while (doc_it.next()) |doc_entry| {
doc_entry.value_ptr.*.deinit();
self.gpa.destroy(doc_entry.value_ptr.*);
self.gpa.free(doc_entry.key_ptr.*);
}
coll.docs.deinit(self.gpa);
}
/// Free every collection in a database along with its owned name keys.
fn free_db(self: *Engine, db: *Db) void {
var coll_it = db.collections.iterator();
while (coll_it.next()) |coll_entry| {
self.free_collection(coll_entry.value_ptr);
self.gpa.free(coll_entry.key_ptr.*);
}
db.collections.deinit(self.gpa);
}
/// Drop the document stored under `id_key`, freeing it and its key.
/// No-op when the id is absent.
fn evict_doc(self: *Engine, coll: *Collection, id_key: []const u8) void {
const old = coll.docs.fetchRemove(id_key) orelse return;
old.value.*.deinit();
self.gpa.destroy(old.value);
self.gpa.free(old.key);
}
// -- commands (callers must hold the matching lock) ---------------------
/// Exclusive lock: for commands that mutate the engine.
@@ -97,74 +116,76 @@ pub const Engine = struct {
/// Insert a document. Fails with error.DuplicateKey if the _id exists.
/// Generates an ObjectId _id when absent.
pub fn insert(self: *Engine, db_name: []const u8, coll_name: []const u8, doc: *const bson.Document, oid_gen: *bson.ObjectIdGen) !void {
const coll = try self.get_or_create_collection(db_name, coll_name);
const owned = try self.own_with_id(doc, oid_gen);
errdefer {
owned.deinit();
self.gpa.destroy(owned);
}
const id_value = owned.get("_id") orelse unreachable;
const id_key = try bson.serialize_value(self.gpa, id_value);
defer self.gpa.free(id_key);
if (coll.docs.contains(id_key)) return error.DuplicateKey;
const doc_bytes = try serialize_doc(self.gpa, owned);
defer self.gpa.free(doc_bytes);
self.seq += 1;
try self.log.append_upsert(db_name, coll_name, doc_bytes, self.seq);
const key_owned = try self.gpa.dupe(u8, id_key);
try coll.docs.put(self.gpa, key_owned, owned);
try self.maybe_compact();
return self.upsert(db_name, coll_name, doc, oid_gen, .insert);
}
/// Insert or replace a document by _id (upsert without existence check).
pub fn replace(self: *Engine, db_name: []const u8, coll_name: []const u8, doc: *const bson.Document, oid_gen: *bson.ObjectIdGen) !void {
return self.upsert(db_name, coll_name, doc, oid_gen, .replace);
}
/// Shared body of `insert` and `replace`: they differ only in how an
/// existing _id is treated. Logs (and syncs) the new document before it
/// becomes visible in memory.
fn upsert(
self: *Engine,
db_name: []const u8,
coll_name: []const u8,
doc: *const bson.Document,
oid_gen: *bson.ObjectIdGen,
mode: enum { insert, replace },
) !void {
const coll = try self.get_or_create_collection(db_name, coll_name);
const owned = try self.own_with_id(doc, oid_gen);
errdefer {
const id_value = owned.get("_id") orelse unreachable;
// Ownership of the key moves to the map once `stored` is set; until
// then this frame still owns both it and `owned`.
const id_key = try bson.serialize_value(self.gpa, id_value);
var stored = false;
errdefer if (!stored) {
owned.deinit();
self.gpa.destroy(owned);
}
self.gpa.free(id_key);
};
const id_value = owned.get("_id") orelse unreachable;
const id_key = try bson.serialize_value(self.gpa, id_value);
defer self.gpa.free(id_key);
if (mode == .insert and coll.docs.contains(id_key)) return error.DuplicateKey;
const doc_bytes = try serialize_doc(self.gpa, owned);
defer self.gpa.free(doc_bytes);
self.seq += 1;
try self.log.append_upsert(db_name, coll_name, doc_bytes, self.seq);
if (coll.docs.fetchRemove(id_key)) |old| {
old.value.*.deinit();
self.gpa.destroy(old.value);
self.gpa.free(old.key);
}
const key_owned = try self.gpa.dupe(u8, id_key);
try coll.docs.put(self.gpa, key_owned, owned);
if (mode == .replace) self.evict_doc(coll, id_key);
try coll.docs.put(self.gpa, id_key, owned);
stored = true;
try self.maybe_compact();
}
/// Remove a document by _id. Returns true if it existed.
pub fn remove(self: *Engine, db_name: []const u8, coll_name: []const u8, id_key: []const u8) !bool {
/// Remove a document by its `_id` value. Returns true if it existed.
/// The serialized-key encoding stays private to the engine.
pub fn remove_by_id(self: *Engine, db_name: []const u8, coll_name: []const u8, id: bson.Value) !bool {
const id_key = try bson.serialize_value(self.gpa, id);
defer self.gpa.free(id_key);
return self.remove(db_name, coll_name, id_key);
}
fn remove(self: *Engine, db_name: []const u8, coll_name: []const u8, id_key: []const u8) !bool {
const db = self.dbs.get(db_name) orelse return false;
const coll = db.collections.getPtr(coll_name) orelse return false;
const doc = coll.docs.get(id_key) orelse return false;
// Log (and sync) the delete before removing it from memory, so the
// log always describes at least as much as the in-memory state.
const doc_bytes = try serialize_doc(self.gpa, doc);
defer self.gpa.free(doc_bytes);
// Replay only reads _id out of a delete record, so log just that
// rather than a copy of the whole document.
const id_pairs = [_]bson.Pair{.{ .key = "_id", .value = doc.get("_id") orelse unreachable }};
var id_doc: std.ArrayListUnmanaged(u8) = .empty;
defer id_doc.deinit(self.gpa);
try bson.write_doc(&id_pairs, self.gpa, &id_doc);
self.seq += 1;
try self.log.append_delete(db_name, coll_name, doc_bytes, self.seq);
try self.log.append_delete(db_name, coll_name, id_doc.items, self.seq);
const removed = coll.docs.fetchRemove(id_key) orelse unreachable;
removed.value.*.deinit();
self.gpa.destroy(removed.value);
self.gpa.free(removed.key);
self.evict_doc(coll, id_key);
return true;
}
@@ -181,31 +202,14 @@ pub const Engine = struct {
pub fn drop_collection(self: *Engine, db_name: []const u8, coll_name: []const u8) !bool {
const db = self.dbs.getPtr(db_name) orelse return false;
var removed = db.collections.fetchRemove(coll_name) orelse return false;
var doc_it = removed.value.docs.iterator();
while (doc_it.next()) |doc_entry| {
doc_entry.value_ptr.*.deinit();
self.gpa.destroy(doc_entry.value_ptr.*);
self.gpa.free(doc_entry.key_ptr.*);
}
removed.value.docs.deinit(self.gpa);
self.free_collection(&removed.value);
self.gpa.free(removed.key);
return true;
}
pub fn drop_database(self: *Engine, db_name: []const u8) !bool {
var removed = self.dbs.fetchRemove(db_name) orelse return false;
var coll_it = removed.value.collections.iterator();
while (coll_it.next()) |coll_entry| {
var docs_it = coll_entry.value_ptr.docs.iterator();
while (docs_it.next()) |doc_entry| {
doc_entry.value_ptr.*.deinit();
self.gpa.destroy(doc_entry.value_ptr.*);
self.gpa.free(doc_entry.key_ptr.*);
}
coll_entry.value_ptr.docs.deinit(self.gpa);
self.gpa.free(coll_entry.key_ptr.*);
}
removed.value.collections.deinit(self.gpa);
self.free_db(&removed.value);
self.gpa.free(removed.key);
return true;
}
@@ -328,28 +332,19 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an
return;
};
const id_key = try bson.serialize_value(self.gpa, id_value);
defer self.gpa.free(id_key);
var key_owned = false;
defer if (!key_owned) self.gpa.free(id_key);
const coll = self.get_or_create_collection(record.db, record.coll) catch return;
switch (record.type) {
storage.record_type_upsert => {
if (coll.docs.fetchRemove(id_key)) |old| {
old.value.*.deinit();
self.gpa.destroy(old.value);
self.gpa.free(old.key);
}
const key_owned = try self.gpa.dupe(u8, id_key);
try coll.docs.put(self.gpa, key_owned, doc);
self.evict_doc(coll, id_key);
try coll.docs.put(self.gpa, id_key, doc);
key_owned = true;
stored = true;
},
storage.record_type_delete => {
if (coll.docs.fetchRemove(id_key)) |old| {
old.value.*.deinit();
self.gpa.destroy(old.value);
self.gpa.free(old.key);
}
},
storage.record_type_delete => self.evict_doc(coll, id_key),
else => {},
}
}
@@ -360,21 +355,7 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an
const testing = std.testing;
const TmpLog = struct {
tmp: std.testing.TmpDir,
path: []u8,
fn init(gpa: std.mem.Allocator) !TmpLog {
const tmp = std.testing.tmpDir(.{});
const path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.log", .{tmp.sub_path});
return .{ .tmp = tmp, .path = path };
}
fn deinit(self: *TmpLog, gpa: std.mem.Allocator) void {
self.tmp.cleanup();
gpa.free(self.path);
}
};
const TmpLog = storage.TmpLog;
fn test_env(threaded: *std.Io.Threaded) struct { io: std.Io, gen: bson.ObjectIdGen } {
const io = threaded.io();
@@ -426,7 +407,7 @@ test "insert, query, remove" {
try engine.lock();
const found = engine.get_doc("app", "users", id_key).?;
try testing.expectEqualStrings("bob", found.get("name").?.string);
const removed = try engine.remove("app", "users", id_key);
const removed = try engine.remove_by_id("app", "users", .{ .int32 = 2 });
try testing.expect(removed);
engine.unlock();
}
@@ -450,9 +431,7 @@ test "reopen replays log" {
try engine.lock();
try engine.insert("app", "users", &d1, &env.gen);
try engine.insert("app", "users", &d2, &env.gen);
const id_key2 = try bson.serialize_value(gpa, bson.Value{ .int32 = 2 });
defer gpa.free(id_key2);
_ = try engine.remove("app", "users", id_key2);
_ = try engine.remove_by_id("app", "users", .{ .int32 = 2 });
engine.unlock();
}