dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index/db/commands: fold duplicated index logic into single definitions

Browse Source 
Cleanup pass over the secondary-index feature.

add_doc is now the one entry-commit path. create_index and
build_all_indexes each hand-rolled build -> check_unique -> reserve ->
insert, and had already drifted on whether multikey is set before or
after the unique check; add_doc gained an enforce_unique flag so the
rebuild path keeps its tolerate-and-warn behavior. reserve_for and
insert_entries are now the only way the engine touches Index.entries.

One definition each for: prefix comparison and the prefix binary
searches (prefix_order + std.sort), the cartesian-product odometer
(advance_choice), the spec pair list (write_spec builds on spec_pairs,
so the log format and the listIndexes reply share one schema), the _id
clause parser (plan_id reuses analyze_clause), key-pattern direction
(index.descending, which desc_dir already disagreed with on non-numeric
values), option truthiness (query.truthy), the E11000 message, and
index-removal-by-name (Collection.find_index/remove_index). Key-pattern
matching moved out of the dispatcher into index.find_by_key_pattern.

Dead or redundant: ParallelArraysError, the unread `dropped` counter,
insert_entries' discarded gpa, a third pass computing multikey, the
has_id/is_id_index flag pair, Plan.key_len (always lookup_keys[0].len,
now a method), first_match_consumed (now stages = stages[1..]).

Cheaper hot paths: remove_id compacts in one pass instead of an
orderedRemove per hit; Plan.search skips the sort/dedupe when neither
multikey nor multiple lookup keys can produce a repeat; the _id fast
path reuses one scratch key buffer (bson.write_serialized_value); the
plan loop uses the bound collection instead of re-resolving it through
two hash lookups per candidate.

Behavior is unchanged except that dropping plan_id's fixed 16-clause
buffer enables the _id fast path on filters that previously exceeded it.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 13:40:42 +03:00
parent 0d264c6c57
commit 7482042f34
5 changed files with 260 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/bson.zig
View File

@@ -458,11 +458,18 @@ fn write_array(items: []const Value, gpa: std.mem.Allocator, out: *std.ArrayList
pub fn serialize_value(gpa: std.mem.Allocator, v: Value) ![]u8 { pub fn serialize_value(gpa: std.mem.Allocator, v: Value) ![]u8 {
var out: std.ArrayListUnmanaged(u8) = .empty; var out: std.ArrayListUnmanaged(u8) = .empty;
errdefer out.deinit(gpa); errdefer out.deinit(gpa);
try out.append(gpa, v.type_tag()); try write_serialized_value(v, gpa, &out);
try write_value(v, gpa, &out);
return out.toOwnedSlice(gpa); return out.toOwnedSlice(gpa);
} }
/// Append the serialized-key bytes of `v` (type tag + payload) to `out`. The
/// appending form of serialize_value, for callers reusing one scratch buffer
/// across many keys.
pub fn write_serialized_value(v: Value, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) !void {
try out.append(gpa, v.type_tag());
try write_value(v, gpa, out);
}
/// Deep-copy a value into `arena`, so the copy is self-contained. /// Deep-copy a value into `arena`, so the copy is self-contained.
pub fn copy_value(arena: std.mem.Allocator, v: Value) std.mem.Allocator.Error!Value { pub fn copy_value(arena: std.mem.Allocator, v: Value) std.mem.Allocator.Error!Value {
return switch (v) { return switch (v) {

143
src/commands.zig
View File

@@ -336,18 +336,17 @@ fn cmd_create_indexes(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !vo
// {_id: 1} is the implicit index: an idempotent no-op. Any other // {_id: 1} is the implicit index: an idempotent no-op. Any other
// secondary index touching _id is rejected. // secondary index touching _id is rejected.
var has_id = false; var has_id = false;
var is_id_index = false;
for (key_pairs) |p| { for (key_pairs) |p| {
if (std.mem.eql(u8, p.key, "_id")) has_id = true; if (std.mem.eql(u8, p.key, "_id")) has_id = true;
} }
if (key_pairs.len == 1 and has_id) { if (has_id) {
const only = key_pairs[0].value; const only = key_pairs[0].value;
is_id_index = (only == .int32 and only.int32 == 1) or const is_id_index = key_pairs.len == 1 and ((only == .int32 and only.int32 == 1) or
(only == .int64 and only.int64 == 1) or (only == .int64 and only.int64 == 1) or
(only == .double and only.double == 1.0); (only == .double and only.double == 1.0));
if (is_id_index) continue;
return bad_value(reply, "cannot create a secondary index on the _id field");
} }
if (is_id_index) continue;
if (has_id) return bad_value(reply, "cannot create a secondary index on the _id field");
const name = bson.get_pair(spec, "name") orelse bson.Value.null; const name = bson.get_pair(spec, "name") orelse bson.Value.null;
if (name == .string and std.mem.eql(u8, name.string, "_id_")) { if (name == .string and std.mem.eql(u8, name.string, "_id_")) {
return bad_value(reply, "cannot create index with name '_id_'"); return bad_value(reply, "cannot create index with name '_id_'");
@@ -359,11 +358,7 @@ fn cmd_create_indexes(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !vo
error.IndexOptionsConflict => return reply.put_error(@intFromEnum(ErrorCode.index_options_conflict), "IndexOptionsConflict", "index already exists with a different specification"), error.IndexOptionsConflict => return reply.put_error(@intFromEnum(ErrorCode.index_options_conflict), "IndexOptionsConflict", "index already exists with a different specification"),
error.DuplicateKeyIndex => { error.DuplicateKeyIndex => {
const ix_name = if (name == .string) name.string else "index"; const ix_name = if (name == .string) name.string else "index";
const msg_text = try std.fmt.allocPrint( const msg_text = try e11000_message(reply, db_name, coll_name, ix_name, try render_spec_key(reply, key_pairs));
reply.arena_alloc(),
"E11000 duplicate key error collection: {s}.{s} index: {s} dup key: {s}",
.{ db_name, coll_name, ix_name, try render_spec_key(reply, key_pairs) },
);
return reply.put_error(@intFromEnum(ErrorCode.duplicate_key), "DuplicateKey", msg_text); return reply.put_error(@intFromEnum(ErrorCode.duplicate_key), "DuplicateKey", msg_text);
}, },
error.ParallelArrays => return bad_value(reply, "cannot index parallel arrays"), error.ParallelArrays => return bad_value(reply, "cannot index parallel arrays"),
@@ -418,23 +413,18 @@ fn cmd_drop_indexes(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void
const n_indexes_was: i32 = @intCast(coll.indexes.items.len + 1); const n_indexes_was: i32 = @intCast(coll.indexes.items.len + 1);
const arg = msg.body.get("index") orelse return bad_value(reply, "dropIndexes requires index"); const arg = msg.body.get("index") orelse return bad_value(reply, "dropIndexes requires index");
var dropped: usize = 0;
if (arg == .string and std.mem.eql(u8, arg.string, "*")) { if (arg == .string and std.mem.eql(u8, arg.string, "*")) {
// Drop every secondary index. Copy the names first: each drop // Drop every secondary index. Copy the names first: each drop
// mutates the collection's index list. // mutates the collection's index list.
var names: std.ArrayListUnmanaged([]const u8) = .empty; var names: std.ArrayListUnmanaged([]const u8) = .empty;
defer names.deinit(ctx.gpa); defer names.deinit(ctx.gpa);
for (coll.indexes.items) |ix| try names.append(ctx.gpa, ix.name); for (coll.indexes.items) |ix| try names.append(ctx.gpa, ix.name);
for (names.items) |nm| { for (names.items) |nm| _ = try ctx.engine.drop_index(db_name, coll_name, nm);
if (try ctx.engine.drop_index(db_name, coll_name, nm)) dropped += 1;
}
} else if (arg == .string) { } else if (arg == .string) {
if (std.mem.eql(u8, arg.string, "_id_")) { if (std.mem.eql(u8, arg.string, "_id_")) {
return invalid_arg(reply, "cannot drop the _id index"); return invalid_arg(reply, "cannot drop the _id index");
} }
if (try ctx.engine.drop_index(db_name, coll_name, arg.string)) { if (!try ctx.engine.drop_index(db_name, coll_name, arg.string)) {
dropped += 1;
} else {
return reply.put_error(@intFromEnum(ErrorCode.index_not_found), "IndexNotFound", "index not found with name"); return reply.put_error(@intFromEnum(ErrorCode.index_not_found), "IndexNotFound", "index not found with name");
} }
} else if (arg == .doc) { } else if (arg == .doc) {
@@ -444,23 +434,9 @@ fn cmd_drop_indexes(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void
.doc => |p| p, .doc => |p| p,
else => return bad_value(reply, "dropIndexes index must be a name, key document, or '*'"), else => return bad_value(reply, "dropIndexes index must be a name, key document, or '*'"),
}; };
var target: ?[]const u8 = null; const target = index.find_by_key_pattern(coll.indexes.items, key_pairs) orelse
for (coll.indexes.items) |ix| { return reply.put_error(@intFromEnum(ErrorCode.index_not_found), "IndexNotFound", "index not found with key pattern");
if (ix.keys.len != key_pairs.len) continue; _ = try ctx.engine.drop_index(db_name, coll_name, target.name);
var match = true;
for (ix.keys, key_pairs) |k, kp| {
if (!std.mem.eql(u8, k.path, kp.key) or k.descending != (desc_dir(kp.value) < 0)) {
match = false;
break;
}
}
if (match) {
target = ix.name;
break;
}
}
const nm = target orelse return reply.put_error(@intFromEnum(ErrorCode.index_not_found), "IndexNotFound", "index not found with key pattern");
if (try ctx.engine.drop_index(db_name, coll_name, nm)) dropped += 1;
} else { } else {
return bad_value(reply, "dropIndexes index must be a name, key document, or '*'"); return bad_value(reply, "dropIndexes index must be a name, key document, or '*'");
} }
@@ -469,13 +445,14 @@ fn cmd_drop_indexes(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void
try reply.put_ok(); try reply.put_ok();
} }
fn desc_dir(v: bson.Value) i32 { /// The E11000 text drivers parse. One definition for both create-time and
return switch (v) { /// write-time conflicts; they differ only in how the dup key is rendered.
.int32 => |n| if (n < 0) -1 else 1, fn e11000_message(reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, index_name: []const u8, key_text: []const u8) ![]const u8 {
.int64 => |n| if (n < 0) -1 else 1, return std.fmt.allocPrint(
.double => |n| if (n < 0) -1 else 1, reply.arena_alloc(),
else => 1, "E11000 duplicate key error collection: {s}.{s} index: {s} dup key: {s}",
}; .{ db_name, coll_name, index_name, key_text },
);
} }
/// Render the key pattern with placeholder values for a createIndexes /// Render the key pattern with placeholder values for a createIndexes
@@ -588,20 +565,14 @@ fn scan_matching(
// queried value's compare-equivalence class is serialization-ambiguous // queried value's compare-equivalence class is serialization-ambiguous
// (see index.plan_id). // (see index.plan_id).
if (index.plan_id(filter)) |id_plan| { if (index.plan_id(filter)) |id_plan| {
var keys: std.ArrayListUnmanaged([]u8) = .empty; // One scratch key, rebuilt per value: a key is never needed past its
defer { // own lookup.
for (keys.items) |k| ctx.gpa.free(k); var key: std.ArrayListUnmanaged(u8) = .empty;
keys.deinit(ctx.gpa); defer key.deinit(ctx.gpa);
}
for (id_plan.values) |v| { for (id_plan.values) |v| {
const key = try bson.serialize_value(ctx.gpa, v); key.clearRetainingCapacity();
keys.append(ctx.gpa, key) catch |err| { try bson.write_serialized_value(v, ctx.gpa, &key);
ctx.gpa.free(key); const doc = coll.docs.get(key.items) orelse continue;
return err;
};
}
for (keys.items) |key| {
const doc = coll.docs.get(key) orelse continue;
if (!try query.matches(ctx.gpa, &filter_doc, doc)) continue; if (!try query.matches(ctx.gpa, &filter_doc, doc)) continue;
if (out) |list| try list.append(ctx.gpa, doc); if (out) |list| try list.append(ctx.gpa, doc);
n += 1; n += 1;
@@ -619,7 +590,7 @@ fn scan_matching(
defer ids.deinit(ctx.gpa); defer ids.deinit(ctx.gpa);
try plan.search(ctx.gpa, &ids); try plan.search(ctx.gpa, &ids);
for (ids.items) |id| { for (ids.items) |id| {
const doc = ctx.engine.get_doc(db_name, coll_name, id) orelse continue; const doc = coll.docs.get(id) orelse continue;
if (!try query.matches(ctx.gpa, &filter_doc, doc)) continue; if (!try query.matches(ctx.gpa, &filter_doc, doc)) continue;
if (out) |list| try list.append(ctx.gpa, doc); if (out) |list| try list.append(ctx.gpa, doc);
n += 1; n += 1;
@@ -846,7 +817,7 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void {
const coll_name = str_arg(msg.body.get("aggregate")) orelse return bad_value(reply, "aggregate requires a collection name"); const coll_name = str_arg(msg.body.get("aggregate")) orelse return bad_value(reply, "aggregate requires a collection name");
const pipeline_value = msg.body.get("pipeline") orelse return bad_value(reply, "aggregate requires pipeline"); const pipeline_value = msg.body.get("pipeline") orelse return bad_value(reply, "aggregate requires pipeline");
const stages = switch (pipeline_value) { var stages = switch (pipeline_value) {
.array => |a| a, .array => |a| a,
else => return bad_value(reply, "pipeline must be an array"), else => return bad_value(reply, "pipeline must be an array"),
}; };
@@ -857,12 +828,11 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void {
var stream: std.ArrayListUnmanaged(*const bson.Document) = .empty; var stream: std.ArrayListUnmanaged(*const bson.Document) = .empty;
defer stream.deinit(ctx.gpa); defer stream.deinit(ctx.gpa);
// A leading $match is pushed down into an indexed candidate scan; the // A leading $match is pushed down into an indexed candidate scan; the
// stage is then consumed so it is not applied a second time. // stage is then dropped from the pipeline so it is not applied twice.
var first_match_consumed = false;
if (stages.len > 0 and stages[0] == .doc and stages[0].doc.len > 0 and std.mem.eql(u8, stages[0].doc[0].key, "$match")) { if (stages.len > 0 and stages[0] == .doc and stages[0].doc.len > 0 and std.mem.eql(u8, stages[0].doc[0].key, "$match")) {
const filter = doc_arg(stages[0].doc[0].value) orelse return bad_value(reply, "$match requires a document"); const filter = doc_arg(stages[0].doc[0].value) orelse return bad_value(reply, "$match requires a document");
_ = try scan_matching(ctx, db_name, coll_name, filter, 0, &stream); _ = try scan_matching(ctx, db_name, coll_name, filter, 0, &stream);
first_match_consumed = true; stages = stages[1..];
} else if (ctx.engine.get_collection(db_name, coll_name)) |coll| { } else if (ctx.engine.get_collection(db_name, coll_name)) |coll| {
var it = coll.docs.iterator(); var it = coll.docs.iterator();
while (it.next()) |entry| try stream.append(ctx.gpa, entry.value_ptr.*); while (it.next()) |entry| try stream.append(ctx.gpa, entry.value_ptr.*);
@@ -874,10 +844,6 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void {
var proj_pairs: ?[]const bson.Pair = null; var proj_pairs: ?[]const bson.Pair = null;
for (stages) |stage_v| { for (stages) |stage_v| {
if (first_match_consumed) {
first_match_consumed = false;
continue;
}
const stage = switch (stage_v) { const stage = switch (stage_v) {
.doc => |pairs| pairs, .doc => |pairs| pairs,
else => return bad_value(reply, "pipeline stages must be documents"), else => return bad_value(reply, "pipeline stages must be documents"),
@@ -1214,7 +1180,6 @@ fn bad_value(reply: *wire.Reply, msg: []const u8) !void {
/// rejected unique-index write) when the conflict came from a secondary /// rejected unique-index write) when the conflict came from a secondary
/// index; otherwise it is the _id_ index. /// index; otherwise it is the _id_ index.
fn duplicate_key_message(ctx: *Context, reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, doc: *const bson.Document) ![]const u8 { fn duplicate_key_message(ctx: *Context, reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, doc: *const bson.Document) ![]const u8 {
const arena = reply.arena_alloc();
var index_name: []const u8 = "_id_"; var index_name: []const u8 = "_id_";
var key_text: []const u8 = undefined; var key_text: []const u8 = undefined;
if (ctx.engine.dup_index) |name| { if (ctx.engine.dup_index) |name| {
@@ -1223,40 +1188,34 @@ fn duplicate_key_message(ctx: *Context, reply: *wire.Reply, db_name: []const u8,
} else { } else {
key_text = try serialize_value_compact(reply, doc.get("_id") orelse bson.Value.null); key_text = try serialize_value_compact(reply, doc.get("_id") orelse bson.Value.null);
} }
return std.fmt.allocPrint( return e11000_message(reply, db_name, coll_name, index_name, key_text);
arena,
"E11000 duplicate key error collection: {s}.{s} index: {s} dup key: {s}",
.{ db_name, coll_name, index_name, key_text },
);
} }
/// Render the dup key of a secondary index from the offending document: the /// Render the dup key of a secondary index from the offending document: the
/// document's values for the index key pattern. /// document's values for the index key pattern.
fn render_dup_key(ctx: *Context, reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, index_name: []const u8, doc: *const bson.Document) ![]const u8 { fn render_dup_key(ctx: *Context, reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, index_name: []const u8, doc: *const bson.Document) ![]const u8 {
const arena = reply.arena_alloc(); const arena = reply.arena_alloc();
if (ctx.engine.get_collection(db_name, coll_name)) |coll| { const coll = ctx.engine.get_collection(db_name, coll_name) orelse
for (coll.indexes.items) |*ix| { // Index not found (defensive): fall back to the document's _id.
if (std.mem.eql(u8, ix.name, index_name)) { return serialize_value_compact(reply, doc.get("_id") orelse bson.Value.null);
var out: std.ArrayListUnmanaged(u8) = .empty; const ix = coll.find_index(index_name) orelse
defer out.deinit(arena); return serialize_value_compact(reply, doc.get("_id") orelse bson.Value.null);
try out.append(arena, '{');
for (ix.keys, 0..) |k, i| { var out: std.ArrayListUnmanaged(u8) = .empty;
if (i > 0) try out.appendSlice(arena, ", "); defer out.deinit(arena);
try out.appendSlice(arena, k.path); try out.append(arena, '{');
try out.appendSlice(arena, ": "); for (ix.keys, 0..) |k, i| {
var values: std.ArrayListUnmanaged(bson.Value) = .empty; if (i > 0) try out.appendSlice(arena, ", ");
defer values.deinit(arena); try out.appendSlice(arena, k.path);
try query.collect_values(arena, doc.pairs, k.path, &values, 0); try out.appendSlice(arena, ": ");
const v: bson.Value = if (values.items.len > 0) values.items[0] else .null; var values: std.ArrayListUnmanaged(bson.Value) = .empty;
try out.appendSlice(arena, try serialize_value_compact(reply, v)); defer values.deinit(arena);
} try query.collect_values(arena, doc.pairs, k.path, &values, 0);
try out.append(arena, '}'); const v: bson.Value = if (values.items.len > 0) values.items[0] else .null;
return out.toOwnedSlice(arena); try out.appendSlice(arena, try serialize_value_compact(reply, v));
}
}
} }
// Index not found (defensive): fall back to the document's _id. try out.append(arena, '}');
return serialize_value_compact(reply, doc.get("_id") orelse bson.Value.null); return out.toOwnedSlice(arena);
} }
fn duplicate_key_error(ctx: *Context, reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, doc: *const bson.Document) !void { fn duplicate_key_error(ctx: *Context, reply: *wire.Reply, db_name: []const u8, coll_name: []const u8, doc: *const bson.Document) !void {

96
src/db.zig
View File

@@ -17,6 +17,28 @@ pub const Collection = struct {
fn init() Collection { fn init() Collection {
return .{ .docs = .empty, .indexes = .empty }; return .{ .docs = .empty, .indexes = .empty };
} }
/// The secondary index with this name, or null. The single by-name
/// lookup: index lifetime (who calls Index.deinit, and when) is decided
/// here rather than at each caller.
pub fn find_index(self: *Collection, name: []const u8) ?*index.Index {
for (self.indexes.items) |*ix| {
if (std.mem.eql(u8, ix.name, name)) return ix;
}
return null;
}
/// Remove and free the index with this name. Returns whether it existed.
fn remove_index(self: *Collection, gpa: std.mem.Allocator, name: []const u8) bool {
for (self.indexes.items, 0..) |ix, i| {
if (std.mem.eql(u8, ix.name, name)) {
var removed = self.indexes.orderedRemove(i);
removed.deinit(gpa);
return true;
}
}
return false;
}
}; };
pub const Db = struct { pub const Db = struct {
@@ -204,8 +226,7 @@ pub const Engine = struct {
// 4. Reserve entry capacity — the last fallible step, so the entry // 4. Reserve entry capacity — the last fallible step, so the entry
// insertion after the log append is infallible. // insertion after the log append is infallible.
for (built_list.items) |*b| { for (built_list.items) |*b| {
if (b.built.entries.items.len == 0) continue; try b.ix.reserve_for(self.gpa, b.built.entries.items.len);
try b.ix.entries.ensureUnusedCapacity(self.gpa, b.built.entries.items.len);
} }
// 5. Log (and sync) before anything becomes visible. // 5. Log (and sync) before anything becomes visible.
@@ -221,7 +242,7 @@ pub const Engine = struct {
try coll.docs.put(self.gpa, id_key, owned); try coll.docs.put(self.gpa, id_key, owned);
for (built_list.items) |*b| { for (built_list.items) |*b| {
if (b.built.multikey) b.ix.multikey = true; if (b.built.multikey) b.ix.multikey = true;
b.ix.insert_entries(self.gpa, &b.built); b.ix.insert_entries(&b.built);
} }
stored = true; stored = true;
try self.maybe_compact(); try self.maybe_compact();
@@ -293,32 +314,19 @@ pub const Engine = struct {
// parsed spec is only owned by the collection once committed. // parsed spec is only owned by the collection once committed.
defer if (!committed) ix.deinit(self.gpa); defer if (!committed) ix.deinit(self.gpa);
for (coll.indexes.items) |*existing| { if (coll.find_index(ix.name)) |existing| {
if (std.mem.eql(u8, existing.name, ix.name)) { if (index.Index.spec_equal(existing, &ix)) return existing;
if (index.Index.spec_equal(existing, &ix)) return existing; return error.IndexOptionsConflict;
return error.IndexOptionsConflict;
}
} }
// Build entries over the existing documents, checking uniqueness as // Build entries over the existing documents, checking uniqueness as
// we go (the index is not exposed until the end, so mutating it is // we go (the index is not exposed until the end, so mutating it is
// safe). Each batch is inserted into the index immediately (which // safe). Each document's batch is inserted immediately, so on any
// drains it), so on any later failure the errdefer ix.deinit frees // later failure the deferred ix.deinit frees every inserted entry
// every inserted entry key; a batch that fails before insertion is // key. Nothing is persisted on failure.
// freed by its own errdefer. Nothing is persisted on failure.
var doc_it = coll.docs.iterator(); var doc_it = coll.docs.iterator();
while (doc_it.next()) |entry| { while (doc_it.next()) |entry| {
var built = try ix.build_entries(self.gpa, entry.value_ptr.*, entry.key_ptr.*); _ = try ix.add_doc(self.gpa, entry.value_ptr.*, entry.key_ptr.*, true);
// Runs on success too: insert_entries drains the keys, leaving
// only the (now empty) ArrayList buffer to free.
defer built.deinit(self.gpa);
if (built.multikey) ix.multikey = true;
if (ix.unique) {
try ix.check_unique(built.entries.items, entry.key_ptr.*);
}
if (built.entries.items.len == 0) continue;
try ix.entries.ensureUnusedCapacity(self.gpa, built.entries.items.len);
ix.insert_entries(self.gpa, &built);
} }
// Reserve the collection slot, then persist and publish. // Reserve the collection slot, then persist and publish.
@@ -339,14 +347,7 @@ pub const Engine = struct {
pub fn drop_index(self: *Engine, db_name: []const u8, coll_name: []const u8, index_name: []const u8) !bool { pub fn drop_index(self: *Engine, db_name: []const u8, coll_name: []const u8, index_name: []const u8) !bool {
const db = self.dbs.get(db_name) orelse return false; const db = self.dbs.get(db_name) orelse return false;
const coll = db.collections.getPtr(coll_name) orelse return false; const coll = db.collections.getPtr(coll_name) orelse return false;
var found = false; if (coll.find_index(index_name) == null) return false;
for (coll.indexes.items) |ix| {
if (std.mem.eql(u8, ix.name, index_name)) {
found = true;
break;
}
}
if (!found) return false;
const name_pairs = [_]bson.Pair{.{ .key = "name", .value = .{ .string = index_name } }}; const name_pairs = [_]bson.Pair{.{ .key = "name", .value = .{ .string = index_name } }};
var name_doc: std.ArrayListUnmanaged(u8) = .empty; var name_doc: std.ArrayListUnmanaged(u8) = .empty;
@@ -355,13 +356,7 @@ pub const Engine = struct {
self.seq += 1; self.seq += 1;
try self.log.append_index_drop(db_name, coll_name, name_doc.items, self.seq); try self.log.append_index_drop(db_name, coll_name, name_doc.items, self.seq);
var i: usize = 0; _ = coll.remove_index(self.gpa, index_name);
while (i < coll.indexes.items.len) {
if (std.mem.eql(u8, coll.indexes.items[i].name, index_name)) {
var removed = coll.indexes.orderedRemove(i);
removed.deinit(self.gpa);
} else i += 1;
}
return true; return true;
} }
@@ -482,23 +477,16 @@ pub const Engine = struct {
if (ix.entries.items.len > 0) continue; // defensive if (ix.entries.items.len > 0) continue; // defensive
var doc_it = coll_entry.value_ptr.docs.iterator(); var doc_it = coll_entry.value_ptr.docs.iterator();
while (doc_it.next()) |doc_entry| { while (doc_it.next()) |doc_entry| {
var built = ix.build_entries(self.gpa, doc_entry.value_ptr.*, doc_entry.key_ptr.*) catch |err| switch (err) { const duplicate = ix.add_doc(self.gpa, doc_entry.value_ptr.*, doc_entry.key_ptr.*, false) catch |err| switch (err) {
error.ParallelArrays => { error.ParallelArrays => {
std.debug.print("mongo-light: WARNING: index '{s}' cannot index an existing document; entry skipped\n", .{ix.name}); std.debug.print("mongo-light: WARNING: index '{s}' cannot index an existing document; entry skipped\n", .{ix.name});
continue; continue;
}, },
else => return err, else => return err,
}; };
defer built.deinit(self.gpa); if (duplicate) {
if (built.multikey) ix.multikey = true; std.debug.print("mongo-light: WARNING: unique index '{s}' has duplicate keys in existing data; duplicates not enforced for existing documents\n", .{ix.name});
if (ix.unique) {
ix.check_unique(built.entries.items, doc_entry.key_ptr.*) catch {
std.debug.print("mongo-light: WARNING: unique index '{s}' has duplicate keys in existing data; duplicates not enforced for existing documents\n", .{ix.name});
};
} }
if (built.entries.items.len == 0) continue;
try ix.entries.ensureUnusedCapacity(self.gpa, built.entries.items.len);
ix.insert_entries(self.gpa, &built);
} }
} }
} }
@@ -511,9 +499,7 @@ pub const Engine = struct {
var ix = try index.parse_spec(self.gpa, spec_doc); var ix = try index.parse_spec(self.gpa, spec_doc);
var committed = false; var committed = false;
defer if (!committed) ix.deinit(self.gpa); defer if (!committed) ix.deinit(self.gpa);
for (coll.indexes.items) |existing| { if (coll.find_index(ix.name) != null) return;
if (std.mem.eql(u8, existing.name, ix.name)) return;
}
try coll.indexes.append(self.gpa, ix); try coll.indexes.append(self.gpa, ix);
committed = true; committed = true;
} }
@@ -559,13 +545,7 @@ fn apply_record(ctx: *anyopaque, record: storage.Record, doc: *bson.Document) an
.string => |s| s, .string => |s| s,
else => return, else => return,
}; };
var i: usize = 0; _ = coll.remove_index(self.gpa, name);
while (i < coll.indexes.items.len) {
if (std.mem.eql(u8, coll.indexes.items[i].name, name)) {
var removed = coll.indexes.orderedRemove(i);
removed.deinit(self.gpa);
} else i += 1;
}
return; return;
}, },
else => {}, else => {},

332
src/index.zig
View File

@@ -37,8 +37,6 @@ pub const max_index_keys: usize = 32;
/// planner falls back to a scan. /// planner falls back to a scan.
const max_combos: u64 = 100; const max_combos: u64 = 100;
pub const ParallelArraysError = error{ ParallelArrays };
/// One key in an index spec. `path` is owned by the Index. /// One key in an index spec. `path` is owned by the Index.
pub const IndexKey = struct { pub const IndexKey = struct {
path: []const u8, path: []const u8,
@@ -129,16 +127,11 @@ pub const Index = struct {
var i: usize = 0; var i: usize = 0;
while (i < direct) : (i += 1) { while (i < direct) : (i += 1) {
if (values.items[i] == .array) { if (values.items[i] == .array) {
multikey = true;
for (values.items[i].array) |elem| try values.append(gpa, elem); for (values.items[i].array) |elem| try values.append(gpa, elem);
} }
} }
if (direct > 1) multikey = true; if (direct > 1) multikey = true;
for (values.items[0..direct]) |v| {
if (v == .array) {
multikey = true;
break;
}
}
if (values.items.len > 1) multi_paths += 1; if (values.items.len > 1) multi_paths += 1;
if (values.items.len == 0) { if (values.items.len == 0) {
if (self.sparse) return .{ .entries = .empty, .multikey = false }; if (self.sparse) return .{ .entries = .empty, .multikey = false };
@@ -155,6 +148,8 @@ pub const Index = struct {
out.deinit(gpa); out.deinit(gpa);
} }
const nkeys = self.keys.len; const nkeys = self.keys.len;
var limits: [max_index_keys]usize = undefined;
for (0..nkeys) |ci| limits[ci] = per_path.items[ci].items.len;
var choice: [max_index_keys]usize = undefined; var choice: [max_index_keys]usize = undefined;
@memset(choice[0..nkeys], 0); @memset(choice[0..nkeys], 0);
while (true) { while (true) {
@@ -162,14 +157,7 @@ pub const Index = struct {
errdefer gpa.free(key); errdefer gpa.free(key);
for (0..nkeys) |ci| key[ci] = per_path.items[ci].items[choice[ci]]; for (0..nkeys) |ci| key[ci] = per_path.items[ci].items[choice[ci]];
try out.append(gpa, .{ .key = key, .id = id }); try out.append(gpa, .{ .key = key, .id = id });
var ci: usize = nkeys; if (!advance_choice(choice[0..nkeys], limits[0..nkeys])) break;
var carry = true;
while (carry and ci > 0) {
ci -= 1;
choice[ci] += 1;
if (choice[ci] < per_path.items[ci].items.len) carry = false else choice[ci] = 0;
}
if (carry) break;
} }
if (out.items.len > 1) { if (out.items.len > 1) {
std.mem.sort(Entry, out.items, {}, entry_less); std.mem.sort(Entry, out.items, {}, entry_less);
@@ -199,8 +187,7 @@ pub const Index = struct {
/// batch: ownership of each entry's key slice moves into the index, so /// batch: ownership of each entry's key slice moves into the index, so
/// the batch's deinit must not free them. Infallible: capacity must /// the batch's deinit must not free them. Infallible: capacity must
/// already be reserved. /// already be reserved.
pub fn insert_entries(self: *Index, gpa: std.mem.Allocator, built: *BuiltEntries) void { pub fn insert_entries(self: *Index, built: *BuiltEntries) void {
_ = gpa;
for (built.entries.items) |e| { for (built.entries.items) |e| {
const pos = self.insert_pos(e); const pos = self.insert_pos(e);
self.entries.insertAssumeCapacity(pos, e); self.entries.insertAssumeCapacity(pos, e);
@@ -210,26 +197,46 @@ pub const Index = struct {
built.entries.items.len = 0; built.entries.items.len = 0;
} }
/// Build, check, and insert entries for one document; the one-shot form /// Build, check, and insert entries for one document — the whole
/// used when rebuilding an index on open. /// entry-commit protocol in one call, used everywhere a single document
pub fn add_doc(self: *Index, gpa: std.mem.Allocator, doc: *const bson.Document, id: []const u8) !void { /// joins an index (create, rebuild on open). Writes that must reserve
/// capacity before a log append use the split build/reserve/insert form
/// directly.
///
/// With `enforce_unique` false a duplicate is tolerated rather than
/// rejected (the rebuild path keeps the index and warns); the return
/// value reports whether that happened.
pub fn add_doc(self: *Index, gpa: std.mem.Allocator, doc: *const bson.Document, id: []const u8, enforce_unique: bool) !bool {
var built = try self.build_entries(gpa, doc, id); var built = try self.build_entries(gpa, doc, id);
// Runs on success too: insert_entries drains the keys, leaving only
// the (now empty) ArrayList buffer to free.
defer built.deinit(gpa); defer built.deinit(gpa);
if (self.unique) try self.check_unique(built.entries.items, id); var duplicate = false;
if (self.unique) {
self.check_unique(built.entries.items, id) catch |err| {
if (enforce_unique) return err;
duplicate = true;
};
}
if (built.multikey) self.multikey = true; if (built.multikey) self.multikey = true;
try self.entries.ensureUnusedCapacity(gpa, built.entries.items.len); try self.reserve_for(gpa, built.entries.items.len);
self.insert_entries(gpa, &built); self.insert_entries(&built);
return duplicate;
} }
/// Remove every entry for `id` and free its key slices. Infallible. /// Remove every entry for `id` and free its key slices. Infallible.
/// One compaction pass: removing in place would memmove the tail per hit.
pub fn remove_id(self: *Index, gpa: std.mem.Allocator, id: []const u8) void { pub fn remove_id(self: *Index, gpa: std.mem.Allocator, id: []const u8) void {
var i: usize = 0; var w: usize = 0;
while (i < self.entries.items.len) { for (self.entries.items) |e| {
if (std.mem.eql(u8, self.entries.items[i].id, id)) { if (std.mem.eql(u8, e.id, id)) {
gpa.free(self.entries.items[i].key); gpa.free(e.key);
_ = self.entries.orderedRemove(i); } else {
} else i += 1; self.entries.items[w] = e;
w += 1;
}
} }
self.entries.items.len = w;
} }
/// Reject when any of `new_entries` has a key already present under a /// Reject when any of `new_entries` has a key already present under a
@@ -246,13 +253,7 @@ pub const Index = struct {
} }
fn insert_pos(self: *const Index, e: Entry) usize { fn insert_pos(self: *const Index, e: Entry) usize {
var lo: usize = 0; return std.sort.lowerBound(Entry, self.entries.items, e, compare_entries);
var hi: usize = self.entries.items.len;
while (lo < hi) {
const mid = lo + (hi - lo) / 2;
if (compare_entries(self.entries.items[mid], e) == .lt) lo = mid + 1 else hi = mid;
}
return lo;
} }
// -- search ------------------------------------------------------------- // -- search -------------------------------------------------------------
@@ -299,25 +300,13 @@ pub const Index = struct {
/// First entry whose first `prefix.len` components are not less than /// First entry whose first `prefix.len` components are not less than
/// `prefix`. /// `prefix`.
fn lower_bound_prefix(self: *const Index, prefix: []const bson.Value) usize { fn lower_bound_prefix(self: *const Index, prefix: []const bson.Value) usize {
var lo: usize = 0; return std.sort.lowerBound(Entry, self.entries.items, prefix, prefix_order);
var hi: usize = self.entries.items.len;
while (lo < hi) {
const mid = lo + (hi - lo) / 2;
if (prefix_lt(self.entries.items[mid].key, prefix)) lo = mid + 1 else hi = mid;
}
return lo;
} }
/// First entry whose first `prefix.len` components are greater than /// First entry whose first `prefix.len` components are greater than
/// `prefix`. /// `prefix`.
fn upper_bound_prefix(self: *const Index, prefix: []const bson.Value) usize { fn upper_bound_prefix(self: *const Index, prefix: []const bson.Value) usize {
var lo: usize = 0; return std.sort.upperBound(Entry, self.entries.items, prefix, prefix_order);
var hi: usize = self.entries.items.len;
while (lo < hi) {
const mid = lo + (hi - lo) / 2;
if (prefix_le(self.entries.items[mid].key, prefix)) lo = mid + 1 else hi = mid;
}
return lo;
} }
// -- serialization ------------------------------------------------------ // -- serialization ------------------------------------------------------
@@ -325,18 +314,10 @@ pub const Index = struct {
/// The canonical spec document bytes ({v, key, name, unique?, sparse?}) /// The canonical spec document bytes ({v, key, name, unique?, sparse?})
/// stored in the log and used to rebuild the index on replay. /// stored in the log and used to rebuild the index on replay.
pub fn write_spec(self: *const Index, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) !void { pub fn write_spec(self: *const Index, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) !void {
var arena = std.heap.ArenaAllocator.init(gpa);
defer arena.deinit();
var pairs: std.ArrayListUnmanaged(bson.Pair) = .empty; var pairs: std.ArrayListUnmanaged(bson.Pair) = .empty;
defer pairs.deinit(gpa); try self.spec_pairs(arena.allocator(), &pairs);
var key_pairs: std.ArrayListUnmanaged(bson.Pair) = .empty;
defer key_pairs.deinit(gpa);
try pairs.append(gpa, .{ .key = "v", .value = .{ .int32 = 2 } });
for (self.keys) |k| {
try key_pairs.append(gpa, .{ .key = k.path, .value = if (k.descending) .{ .int32 = -1 } else .{ .int32 = 1 } });
}
try pairs.append(gpa, .{ .key = "key", .value = .{ .doc = key_pairs.items } });
try pairs.append(gpa, .{ .key = "name", .value = .{ .string = self.name } });
if (self.unique) try pairs.append(gpa, .{ .key = "unique", .value = .{ .bool = true } });
if (self.sparse) try pairs.append(gpa, .{ .key = "sparse", .value = .{ .bool = true } });
try bson.write_doc(pairs.items, gpa, out); try bson.write_doc(pairs.items, gpa, out);
} }
@@ -366,6 +347,24 @@ pub const Index = struct {
} }
}; };
/// The index whose key pattern is exactly `key_pairs` (same paths, same
/// order, same directions), or null. Keeps the IndexKey layout — and what
/// counts as a match — inside this module.
pub fn find_by_key_pattern(indexes: []const Index, key_pairs: []const bson.Pair) ?*const Index {
for (indexes) |*ix| {
if (ix.keys.len != key_pairs.len) continue;
var match = true;
for (ix.keys, key_pairs) |k, kp| {
if (!std.mem.eql(u8, k.path, kp.key) or k.descending != descending(kp.value)) {
match = false;
break;
}
}
if (match) return ix;
}
return null;
}
pub const SpecError = error{ InvalidIndexSpec, OutOfMemory }; pub const SpecError = error{ InvalidIndexSpec, OutOfMemory };
/// Parse {key: {...}, name?, unique?, sparse?} from a spec document — the /// Parse {key: {...}, name?, unique?, sparse?} from a spec document — the
@@ -390,8 +389,8 @@ pub fn parse_spec(gpa: std.mem.Allocator, spec: *const bson.Document) SpecError!
} }
var unique = false; var unique = false;
var sparse = false; var sparse = false;
if (bson.get_pair(spec.pairs, "unique")) |v| unique = truthy(v); if (bson.get_pair(spec.pairs, "unique")) |v| unique = query.truthy(v);
if (bson.get_pair(spec.pairs, "sparse")) |v| sparse = truthy(v); if (bson.get_pair(spec.pairs, "sparse")) |v| sparse = query.truthy(v);
const name_value = bson.get_pair(spec.pairs, "name") orelse { const name_value = bson.get_pair(spec.pairs, "name") orelse {
const nm = try default_name(gpa, key_pairs); const nm = try default_name(gpa, key_pairs);
defer gpa.free(nm); defer gpa.free(nm);
@@ -404,7 +403,10 @@ pub fn parse_spec(gpa: std.mem.Allocator, spec: *const bson.Document) SpecError!
return Index.init(gpa, name, keys[0..key_pairs.len], unique, sparse); return Index.init(gpa, name, keys[0..key_pairs.len], unique, sparse);
} }
fn descending(v: bson.Value) bool { /// Whether a key-pattern direction value means descending. The single
/// definition of what -1 means in a key pattern, shared with dropIndexes'
/// key-pattern matching.
pub fn descending(v: bson.Value) bool {
return switch (v) { return switch (v) {
.int32 => |n| n < 0, .int32 => |n| n < 0,
.int64 => |n| n < 0, .int64 => |n| n < 0,
@@ -413,16 +415,6 @@ fn descending(v: bson.Value) bool {
}; };
} }
fn truthy(v: bson.Value) bool {
return switch (v) {
.bool => |b| b,
.int32 => |i| i != 0,
.int64 => |i| i != 0,
.double => |d| d != 0,
else => false,
};
}
/// MongoDB's default index name: a_1_b_-1. /// MongoDB's default index name: a_1_b_-1.
fn default_name(gpa: std.mem.Allocator, key_pairs: []const bson.Pair) ![]u8 { fn default_name(gpa: std.mem.Allocator, key_pairs: []const bson.Pair) ![]u8 {
var out: std.ArrayListUnmanaged(u8) = .empty; var out: std.ArrayListUnmanaged(u8) = .empty;
@@ -460,22 +452,29 @@ fn entry_less(_: void, a: Entry, b: Entry) bool {
return compare_entries(a, b) == .lt; return compare_entries(a, b) == .lt;
} }
/// Whether entry key's first `prefix.len` components are less than `prefix`. /// Order of `prefix` against an entry key's leading components — `.eq` when
fn prefix_lt(key: []const bson.Value, prefix: []const bson.Value) bool { /// every prefix component matches (the key may be longer). The search-side
for (key[0..prefix.len], prefix) |a, b| { /// counterpart of compare_entries: same component-wise bson.compare, no
const o = bson.compare(a, b); /// length or id tie-break, so a partial key matches a whole range.
if (o != .eq) return o == .lt; fn prefix_order(prefix: []const bson.Value, e: Entry) std.math.Order {
for (prefix, e.key[0..prefix.len]) |p, k| {
const o = bson.compare(p, k);
if (o != .eq) return o;
} }
return false; return .eq;
} }
/// Whether entry key's first `prefix.len` components are <= `prefix`. /// Advance an odometer of positions, each bounded by the matching `limits`
fn prefix_le(key: []const bson.Value, prefix: []const bson.Value) bool { /// entry. Returns false once it wraps, i.e. the product is exhausted.
for (key[0..prefix.len], prefix) |a, b| { fn advance_choice(choice: []usize, limits: []const usize) bool {
const o = bson.compare(a, b); var i = choice.len;
if (o != .eq) return o == .lt; while (i > 0) {
i -= 1;
choice[i] += 1;
if (choice[i] < limits[i]) return true;
choice[i] = 0;
} }
return true; return false;
} }
fn less_ids(_: void, a: []const u8, b: []const u8) bool { fn less_ids(_: void, a: []const u8, b: []const u8) bool {
@@ -579,7 +578,6 @@ fn analyze_clause(value: bson.Value, info: *CompInfo) void {
pub const Plan = struct { pub const Plan = struct {
index: *const Index, index: *const Index,
lookup_keys: std.ArrayListUnmanaged([]const bson.Value), lookup_keys: std.ArrayListUnmanaged([]const bson.Value),
key_len: usize,
lo: ?bson.Value, lo: ?bson.Value,
lo_incl: bool, lo_incl: bool,
hi: ?bson.Value, hi: ?bson.Value,
@@ -590,10 +588,22 @@ pub const Plan = struct {
self.lookup_keys.deinit(gpa); self.lookup_keys.deinit(gpa);
} }
/// How many leading index components the lookup keys pin down. Every key
/// is built with the same component count, and there is always at least
/// one (a pure range plan appends a single empty key).
pub fn key_len(self: *const Plan) usize {
return self.lookup_keys.items[0].len;
}
/// Collect the candidate ids, sorted and deduplicated. Range scans can /// Collect the candidate ids, sorted and deduplicated. Range scans can
/// return the same id non-adjacently (a doc with {tags: ["a","b"]} /// return the same id non-adjacently (a doc with {tags: ["a","b"]}
/// contributes two entries inside one range), so adjacent-dup skipping /// contributes two entries inside one range), so adjacent-dup skipping
/// would be wrong. /// would be wrong.
///
/// Duplicates are only possible from a multikey index (one document,
/// several entries) or from several lookup keys (whose ranges can be the
/// same key repeated, as in {$in: [1, 1]}); the common single-key lookup
/// on a non-multikey index skips the pass entirely.
pub fn search(self: *const Plan, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged([]const u8)) !void { pub fn search(self: *const Plan, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged([]const u8)) !void {
for (self.lookup_keys.items) |key| { for (self.lookup_keys.items) |key| {
if (self.lo == null and self.hi == null) { if (self.lo == null and self.hi == null) {
@@ -602,7 +612,8 @@ pub const Plan = struct {
try self.index.lookup_range(gpa, key, self.lo, self.lo_incl, self.hi, self.hi_incl, out); try self.index.lookup_range(gpa, key, self.lo, self.lo_incl, self.hi, self.hi_incl, out);
} }
} }
if (out.items.len > 1) { const may_repeat = self.index.multikey or self.lookup_keys.items.len > 1;
if (may_repeat and out.items.len > 1) {
std.mem.sort([]const u8, out.items, {}, less_ids); std.mem.sort([]const u8, out.items, {}, less_ids);
var w: usize = 1; var w: usize = 1;
for (out.items[1..]) |id| { for (out.items[1..]) |id| {
@@ -644,7 +655,7 @@ pub fn plan(gpa: std.mem.Allocator, indexes: []const Index, filter: []const bson
} }
fn plan_better(a: *const Plan, b: *const Plan) bool { fn plan_better(a: *const Plan, b: *const Plan) bool {
if (a.key_len != b.key_len) return a.key_len > b.key_len; if (a.key_len() != b.key_len()) return a.key_len() > b.key_len();
const a_range = a.lo != null or a.hi != null; const a_range = a.lo != null or a.hi != null;
const b_range = b.lo != null or b.hi != null; const b_range = b.lo != null or b.hi != null;
if (a_range != b_range) return a_range; if (a_range != b_range) return a_range;
@@ -716,7 +727,6 @@ fn evaluate_index(gpa: std.mem.Allocator, ix: *const Index, clauses: []const Cla
var pl = Plan{ var pl = Plan{
.index = ix, .index = ix,
.lookup_keys = .empty, .lookup_keys = .empty,
.key_len = run,
.lo = lo, .lo = lo,
.lo_incl = lo_incl, .lo_incl = lo_incl,
.hi = hi, .hi = hi,
@@ -740,14 +750,7 @@ fn evaluate_index(gpa: std.mem.Allocator, ix: *const Index, clauses: []const Cla
key[i] = if (infos[i].eq) |v| v else infos[i].in_values.?[choice[i]]; key[i] = if (infos[i].eq) |v| v else infos[i].in_values.?[choice[i]];
} }
try pl.lookup_keys.append(gpa, key); try pl.lookup_keys.append(gpa, key);
var i: usize = run; if (!advance_choice(choice[0..run], counts[0..run])) break;
var carry = true;
while (carry and i > 0) {
i -= 1;
choice[i] += 1;
if (choice[i] < counts[i]) carry = false else choice[i] = 0;
}
if (carry) break;
} }
} }
return pl; return pl;
@@ -768,67 +771,42 @@ pub const IdPlan = struct {
/// and nested variants), but serialize_value produces different map keys — /// and nested variants), but serialize_value produces different map keys —
/// a hash lookup would then miss documents a scan would match. /// a hash lookup would then miss documents a scan would match.
pub fn plan_id(filter: []const bson.Pair) ?IdPlan { pub fn plan_id(filter: []const bson.Pair) ?IdPlan {
var clauses: [16]Clause = undefined; // The first usable _id clause wins; no flattening buffer is needed
var n: usize = 0; // because nothing is compared across clauses. $and members are searched
if (!flatten_id(filter, &clauses, &n)) return null; // like top-level pairs, every other operator skipped — same rule as
for (clauses[0..n]) |cl| { // flatten_clauses, and safe for the same reason (the full filter is
if (!std.mem.eql(u8, cl.path, "_id")) continue; // re-applied to every candidate).
if (id_lookup_values(cl.value)) |values| return .{ .values = values }; for (filter) |p| {
if (p.key.len > 0 and p.key[0] == '$') {
if (!std.mem.eql(u8, p.key, "$and")) continue;
const members = switch (p.value) {
.array => |a| a,
else => continue,
};
for (members) |m| {
const mp = switch (m) {
.doc => |d| d,
else => continue,
};
if (plan_id(mp)) |found| return found;
}
continue;
}
if (!std.mem.eql(u8, p.key, "_id")) continue;
if (id_lookup_values(p.value)) |values| return .{ .values = values };
} }
return null; return null;
} }
fn flatten_id(pairs: []const bson.Pair, out: *[16]Clause, n: *usize) bool { /// The map-lookup values for one _id clause, or null when it is not a pure
for (pairs) |p| { /// equality/$in of fast-path-safe values. A range is unusable here: the docs
if (p.key.len > 0 and p.key[0] == '$') { /// map is a hash, not an ordered structure.
if (std.mem.eql(u8, p.key, "$and")) {
const members = switch (p.value) {
.array => |a| a,
else => continue,
};
for (members) |m| {
const mp = switch (m) {
.doc => |d| d,
else => continue,
};
if (!flatten_id(mp, out, n)) return false;
}
}
continue;
}
if (n.* >= out.len) return false;
out[n.*] = .{ .path = p.key, .value = p.value };
n.* += 1;
}
return true;
}
fn id_lookup_values(v: bson.Value) ?[]const bson.Value { fn id_lookup_values(v: bson.Value) ?[]const bson.Value {
if (v == .regex) return null; var info = CompInfo{};
if (v != .doc) { analyze_clause(v, &info);
return if (value_fast_path_safe(v)) &.{v} else null; if (info.lo != null or info.hi != null) return null;
} if (info.eq) |e| return if (value_fast_path_safe(e)) &.{e} else null;
const pairs = v.doc; if (info.in_values) |list| {
if (pairs.len > 0 and !query.all_operator_keys(pairs)) {
// Bare document equality (compare the whole doc).
return if (value_fast_path_safe(v)) &.{v} else null;
}
var eq: ?bson.Value = null;
var in_list: ?[]const bson.Value = null;
for (pairs) |p| {
if (std.mem.eql(u8, p.key, "$eq")) {
eq = p.value;
} else if (std.mem.eql(u8, p.key, "$in")) {
in_list = switch (p.value) {
.array => |a| a,
else => return null,
};
} else return null;
}
if (eq) |e| {
return if (value_fast_path_safe(e)) &.{e} else null;
}
if (in_list) |list| {
for (list) |m| { for (list) |m| {
if (!value_fast_path_safe(m)) return null; if (!value_fast_path_safe(m)) return null;
} }
@@ -897,11 +875,11 @@ test "entries sort across numeric types and string/null/objectid" {
const d_str = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 3 } }, .{ .key = "a", .value = .{ .string = "b" } } }); const d_str = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 3 } }, .{ .key = "a", .value = .{ .string = "b" } } });
const d_nul = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 4 } }, .{ .key = "a", .value = .null } }); const d_nul = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 4 } }, .{ .key = "a", .value = .null } });
const d_oid = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 5 } }, .{ .key = "a", .value = .{ .object_id = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 } } } }); const d_oid = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 5 } }, .{ .key = "a", .value = .{ .object_id = [_]u8{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 } } } });
try ix.add_doc(gpa, &d_int, "i1"); _ = try ix.add_doc(gpa, &d_int, "i1", true);
try ix.add_doc(gpa, &d_dbl, "i2"); _ = try ix.add_doc(gpa, &d_dbl, "i2", true);
try ix.add_doc(gpa, &d_str, "i3"); _ = try ix.add_doc(gpa, &d_str, "i3", true);
try ix.add_doc(gpa, &d_nul, "i4"); _ = try ix.add_doc(gpa, &d_nul, "i4", true);
try ix.add_doc(gpa, &d_oid, "i5"); _ = try ix.add_doc(gpa, &d_oid, "i5", true);
// An int64 query finds both the int32 and double entries: compare-equal. // An int64 query finds both the int32 and double entries: compare-equal.
try expect_ids(gpa, &ix, &.{.{ .int64 = 5 }}, &.{ "i1", "i2" }); try expect_ids(gpa, &ix, &.{.{ .int64 = 5 }}, &.{ "i1", "i2" });
@@ -920,13 +898,13 @@ test "missing field is indexed as null; sparse skips the document" {
var ix = try simple_index(gpa, &.{"a"}, false, false); var ix = try simple_index(gpa, &.{"a"}, false, false);
defer ix.deinit(gpa); defer ix.deinit(gpa);
const d = doc_of(&.{.{ .key = "_id", .value = .{ .int32 = 1 } }}); const d = doc_of(&.{.{ .key = "_id", .value = .{ .int32 = 1 } }});
try ix.add_doc(gpa, &d, "m1"); _ = try ix.add_doc(gpa, &d, "m1", true);
try testing.expectEqual(@as(usize, 1), ix.entries.items.len); try testing.expectEqual(@as(usize, 1), ix.entries.items.len);
try expect_ids(gpa, &ix, &.{.null}, &.{"m1"}); try expect_ids(gpa, &ix, &.{.null}, &.{"m1"});
var sp = try simple_index(gpa, &.{"a"}, false, true); var sp = try simple_index(gpa, &.{"a"}, false, true);
defer sp.deinit(gpa); defer sp.deinit(gpa);
try sp.add_doc(gpa, &d, "m2"); _ = try sp.add_doc(gpa, &d, "m2", true);
try testing.expectEqual(@as(usize, 0), sp.entries.items.len); try testing.expectEqual(@as(usize, 0), sp.entries.items.len);
} }
@@ -936,7 +914,7 @@ test "multikey expansion indexes the array and its elements" {
defer ix.deinit(gpa); defer ix.deinit(gpa);
const d = doc_of(&.{.{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "tags", .value = .{ .array = &.{ .{ .string = "a" }, .{ .string = "b" } } } }}); const d = doc_of(&.{.{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "tags", .value = .{ .array = &.{ .{ .string = "a" }, .{ .string = "b" } } } }});
try ix.add_doc(gpa, &d, "mk1"); _ = try ix.add_doc(gpa, &d, "mk1", true);
// 3 entries: the array itself, "a", "b". // 3 entries: the array itself, "a", "b".
try testing.expectEqual(@as(usize, 3), ix.entries.items.len); try testing.expectEqual(@as(usize, 3), ix.entries.items.len);
@@ -953,7 +931,7 @@ test "per-document dedup keeps {a: [1,1]} under a unique index" {
var ix = try simple_index(gpa, &.{"a"}, true, false); var ix = try simple_index(gpa, &.{"a"}, true, false);
defer ix.deinit(gpa); defer ix.deinit(gpa);
const d = doc_of(&.{.{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 1 } } } }}); const d = doc_of(&.{.{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 1 } } } }});
try ix.add_doc(gpa, &d, "d1"); _ = try ix.add_doc(gpa, &d, "d1", true);
// Entries after dedup: the array itself and one element. // Entries after dedup: the array itself and one element.
try testing.expectEqual(@as(usize, 2), ix.entries.items.len); try testing.expectEqual(@as(usize, 2), ix.entries.items.len);
} }
@@ -967,7 +945,7 @@ test "parallel arrays are rejected" {
.{ .key = "a", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 2 } } } }, .{ .key = "a", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 2 } } } },
.{ .key = "b", .value = .{ .array = &.{ .{ .int32 = 3 }, .{ .int32 = 4 } } } }, .{ .key = "b", .value = .{ .array = &.{ .{ .int32 = 3 }, .{ .int32 = 4 } } } },
}); });
try testing.expectError(error.ParallelArrays, ix.add_doc(gpa, &d, "p1")); try testing.expectError(error.ParallelArrays, ix.add_doc(gpa, &d, "p1", true));
// One array path is fine. // One array path is fine.
const ok = doc_of(&.{ const ok = doc_of(&.{
@@ -975,7 +953,7 @@ test "parallel arrays are rejected" {
.{ .key = "a", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 2 } } } }, .{ .key = "a", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 2 } } } },
.{ .key = "b", .value = .{ .int32 = 3 } }, .{ .key = "b", .value = .{ .int32 = 3 } },
}); });
try ix.add_doc(gpa, &ok, "p2"); _ = try ix.add_doc(gpa, &ok, "p2", true);
try testing.expectEqual(@as(usize, 3), ix.entries.items.len); try testing.expectEqual(@as(usize, 3), ix.entries.items.len);
} }
@@ -985,16 +963,16 @@ test "unique conflict across documents, replace of own entries allowed" {
defer ix.deinit(gpa); defer ix.deinit(gpa);
const d1 = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .int32 = 10 } } }); const d1 = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .int32 = 10 } } });
try ix.add_doc(gpa, &d1, "u1"); _ = try ix.add_doc(gpa, &d1, "u1", true);
const d2 = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 2 } }, .{ .key = "a", .value = .{ .int32 = 10 } } }); const d2 = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 2 } }, .{ .key = "a", .value = .{ .int32 = 10 } } });
try testing.expectError(error.DuplicateKeyIndex, ix.add_doc(gpa, &d2, "u2")); try testing.expectError(error.DuplicateKeyIndex, ix.add_doc(gpa, &d2, "u2", true));
// A replace keeps its own key: remove old entries first (the engine's // A replace keeps its own key: remove old entries first (the engine's
// evict_doc does this), then add the new ones. // evict_doc does this), then add the new ones.
const d1b = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .int32 = 20 } } }); const d1b = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .int32 = 20 } } });
ix.remove_id(gpa, "u1"); ix.remove_id(gpa, "u1");
try ix.add_doc(gpa, &d1b, "u1"); _ = try ix.add_doc(gpa, &d1b, "u1", true);
try expect_ids(gpa, &ix, &.{.{ .int32 = 20 }}, &.{"u1"}); try expect_ids(gpa, &ix, &.{.{ .int32 = 20 }}, &.{"u1"});
try expect_ids(gpa, &ix, &.{.{ .int32 = 10 }}, &.{}); try expect_ids(gpa, &ix, &.{.{ .int32 = 10 }}, &.{});
} }
@@ -1012,7 +990,7 @@ test "range bounds inclusive and exclusive" {
}; };
for (docs) |s| { for (docs) |s| {
const d = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = s.a } }, .{ .key = "a", .value = .{ .int32 = s.a } } }); const d = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = s.a } }, .{ .key = "a", .value = .{ .int32 = s.a } } });
try ix.add_doc(gpa, &d, s.id); _ = try ix.add_doc(gpa, &d, s.id, true);
} }
var out: std.ArrayListUnmanaged([]const u8) = .empty; var out: std.ArrayListUnmanaged([]const u8) = .empty;
@@ -1043,7 +1021,7 @@ test "empty index and remove_id" {
try testing.expectEqual(@as(usize, 0), out.items.len); try testing.expectEqual(@as(usize, 0), out.items.len);
const d = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .int32 = 1 } } }); const d = doc_of(&.{ .{ .key = "_id", .value = .{ .int32 = 1 } }, .{ .key = "a", .value = .{ .int32 = 1 } } });
try ix.add_doc(gpa, &d, "e1"); _ = try ix.add_doc(gpa, &d, "e1", true);
ix.remove_id(gpa, "e1"); ix.remove_id(gpa, "e1");
try testing.expectEqual(@as(usize, 0), ix.entries.items.len); try testing.expectEqual(@as(usize, 0), ix.entries.items.len);
try ix.lookup_eq(gpa, &.{.{ .int32 = 1 }}, &out); try ix.lookup_eq(gpa, &.{.{ .int32 = 1 }}, &out);
@@ -1069,7 +1047,7 @@ test "compound index prefix search and range on the next key" {
.{ .key = "a", .value = .{ .int32 = s.a } }, .{ .key = "a", .value = .{ .int32 = s.a } },
.{ .key = "b", .value = .{ .int32 = s.b } }, .{ .key = "b", .value = .{ .int32 = s.b } },
}); });
try ix.add_doc(gpa, &d, s.id); _ = try ix.add_doc(gpa, &d, s.id, true);
} }
// Prefix on a only. // Prefix on a only.
@@ -1140,7 +1118,7 @@ test "planner picks eq run, ranges, and bails on sparse null" {
}; };
var p = (try plan(gpa, &.{ix}, &f)).?; var p = (try plan(gpa, &.{ix}, &f)).?;
defer p.deinit(gpa); defer p.deinit(gpa);
try testing.expectEqual(@as(usize, 2), p.key_len); try testing.expectEqual(@as(usize, 2), p.key_len());
try testing.expect(p.lo == null and p.hi == null); try testing.expect(p.lo == null and p.hi == null);
} }
// {a: 1, b: {$gt: 2}} → equality run of 1 + range on the next key. // {a: 1, b: {$gt: 2}} → equality run of 1 + range on the next key.
@@ -1151,7 +1129,7 @@ test "planner picks eq run, ranges, and bails on sparse null" {
}; };
var p = (try plan(gpa, &.{ix}, &f)).?; var p = (try plan(gpa, &.{ix}, &f)).?;
defer p.deinit(gpa); defer p.deinit(gpa);
try testing.expectEqual(@as(usize, 1), p.key_len); try testing.expectEqual(@as(usize, 1), p.key_len());
try testing.expect(p.hi == null and p.lo != null and !p.lo_incl); try testing.expect(p.hi == null and p.lo != null and !p.lo_incl);
} }
// {a: 1} only → prefix run of 1. // {a: 1} only → prefix run of 1.
@@ -1159,14 +1137,14 @@ test "planner picks eq run, ranges, and bails on sparse null" {
const f = [_]bson.Pair{.{ .key = "a", .value = .{ .int32 = 1 } }}; const f = [_]bson.Pair{.{ .key = "a", .value = .{ .int32 = 1 } }};
var p = (try plan(gpa, &.{ix}, &f)).?; var p = (try plan(gpa, &.{ix}, &f)).?;
defer p.deinit(gpa); defer p.deinit(gpa);
try testing.expectEqual(@as(usize, 1), p.key_len); try testing.expectEqual(@as(usize, 1), p.key_len());
} }
// Pure range on the first key → key_len 0 with a bound. // Pure range on the first key → key_len 0 with a bound.
{ {
const f = [_]bson.Pair{.{ .key = "a", .value = .{ .doc = &.{.{ .key = "$gte", .value = .{ .int32 = 1 } }} } }}; const f = [_]bson.Pair{.{ .key = "a", .value = .{ .doc = &.{.{ .key = "$gte", .value = .{ .int32 = 1 } }} } }};
var p = (try plan(gpa, &.{ix}, &f)).?; var p = (try plan(gpa, &.{ix}, &f)).?;
defer p.deinit(gpa); defer p.deinit(gpa);
try testing.expectEqual(@as(usize, 0), p.key_len); try testing.expectEqual(@as(usize, 0), p.key_len());
try testing.expect(p.lo != null and p.lo_incl); try testing.expect(p.lo != null and p.lo_incl);
} }
// Unusable filter → no plan. // Unusable filter → no plan.
@@ -1185,7 +1163,7 @@ test "planner picks eq run, ranges, and bails on sparse null" {
// Non-sparse is fine with null. // Non-sparse is fine with null.
var p = (try plan(gpa, &.{ix}, &f)).?; var p = (try plan(gpa, &.{ix}, &f)).?;
defer p.deinit(gpa); defer p.deinit(gpa);
try testing.expect(p.key_len == 1); try testing.expect(p.key_len() == 1);
// A null inside $in bails too. // A null inside $in bails too.
const fin = [_]bson.Pair{.{ .key = "a", .value = .{ .doc = &.{.{ .key = "$in", .value = .{ .array = &.{ .{ .int32 = 1 }, .null } } }} } }}; const fin = [_]bson.Pair{.{ .key = "a", .value = .{ .doc = &.{.{ .key = "$in", .value = .{ .array = &.{ .{ .int32 = 1 }, .null } } }} } }};
try testing.expect((try plan(gpa, &.{sp}, &fin)) == null); try testing.expect((try plan(gpa, &.{sp}, &fin)) == null);

12
src/query.zig
View File

@@ -604,7 +604,7 @@ pub fn project(arena: std.mem.Allocator, doc: *const bson.Document, proj: *const
for (proj.pairs) |p| { for (proj.pairs) |p| {
if (std.mem.eql(u8, p.key, "_id")) continue; if (std.mem.eql(u8, p.key, "_id")) continue;
non_id_count += 1; non_id_count += 1;
const flag = projection_flag(p.value); const flag = truthy(p.value);
inclusion = if (inclusion == null) flag else inclusion; inclusion = if (inclusion == null) flag else inclusion;
} }
@@ -614,7 +614,7 @@ pub fn project(arena: std.mem.Allocator, doc: *const bson.Document, proj: *const
// Inclusion list: _id unless excluded, plus listed paths. // Inclusion list: _id unless excluded, plus listed paths.
var include_id = true; var include_id = true;
if (bson.get_pair(proj.pairs, "_id")) |idv| { if (bson.get_pair(proj.pairs, "_id")) |idv| {
include_id = projection_flag(idv); include_id = truthy(idv);
} }
if (include_id) { if (include_id) {
if (bson.get_pair(doc.pairs, "_id")) |idv| { if (bson.get_pair(doc.pairs, "_id")) |idv| {
@@ -623,7 +623,7 @@ pub fn project(arena: std.mem.Allocator, doc: *const bson.Document, proj: *const
} }
for (proj.pairs) |p| { for (proj.pairs) |p| {
if (std.mem.eql(u8, p.key, "_id")) continue; if (std.mem.eql(u8, p.key, "_id")) continue;
if (!projection_flag(p.value)) continue; if (!truthy(p.value)) continue;
try project_path(arena, doc.pairs, p.key, out); try project_path(arena, doc.pairs, p.key, out);
} }
} else { } else {
@@ -631,7 +631,7 @@ pub fn project(arena: std.mem.Allocator, doc: *const bson.Document, proj: *const
for (doc.pairs) |p| { for (doc.pairs) |p| {
if (std.mem.eql(u8, p.key, "_id")) { if (std.mem.eql(u8, p.key, "_id")) {
var excluded = false; var excluded = false;
if (bson.get_pair(proj.pairs, "_id")) |idv| excluded = !projection_flag(idv); if (bson.get_pair(proj.pairs, "_id")) |idv| excluded = !truthy(idv);
if (excluded) continue; if (excluded) continue;
} }
if (is_excluded(proj, p.key)) continue; if (is_excluded(proj, p.key)) continue;
@@ -673,7 +673,9 @@ fn has_deeper_exclusion(proj: *const bson.Document, key: []const u8) bool {
return false; return false;
} }
fn projection_flag(v: bson.Value) bool { /// MongoDB's truthiness for an option/flag value (projection flags, index
/// spec options).
pub fn truthy(v: bson.Value) bool {
return switch (v) { return switch (v) {
.bool => |b| b, .bool => |b| b,
.int32 => |i| i != 0, .int32 => |i| i != 0,