dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index/commands: stream whole-index scans; add a reverse leaf iterator

Browse Source 
A whole-index read used to materialize every candidate before the caller saw
the first one. At the tens-of-GB target that is a list of every offset in the
collection -- ~160 MB for a countDocuments({}) over 20 million documents --
which defeats the point of moving storage to disk. Cursors are M1, but
*streaming a scan* has to exist now.

`Candidates` is the one loop candidates arrive through, whatever produced them:
a plan's materialized lookups, or the index read end to end. That keeps this
file's governing invariant -- an index only generates candidates, the full
filter is re-applied to every one -- in a single place. A narrowed plan still
materializes, because its multikey/$in dedupe genuinely needs the whole set and
is bounded by selectivity.

`RevIter` walks `Node.prev`, which has always been maintained and which nothing
had ever read: a descending scan materialized the whole index and reversed the
list. `find({}).sort({_id:-1}).limit(20)` becomes O(20).

The unfiltered fallback now walks the _id_ index instead of the docs map. That
is ordered rather than hash-ordered, and it does not depend on a structure that
is about to be deleted.

`Plan.full_scan()` refuses multikey indexes, since one document contributes
several entries there and a stream cannot dedupe what `search` did. The check is
currently redundant -- the planner refuses to order a multikey index anyway --
and is kept because the two guards protect different things. Stated precisely
in both places after checking: the commands.zig test reddens only when *both*
guards are removed, which is what that test actually pins.

--

This also broke e2e6's compaction check, and the fix there is the more
interesting half.

The check required peak/final > 1.4 and got 1.28. The final size was identical
to the byte (2,398,065 vs 2,398,064) -- compaction reclaimed exactly as before
-- and only the peak moved. Isolated to one variable: changing just the order
updateMany({}) walks its matches moves peak/final between 1.65 and 1.28, because
compaction can also fire from the once-per-second TTL monitor and whether one
lands inside the batch shifts the peak a long way while leaving the outcome
unchanged. The threshold was measuring the schedule.

Replaced with `peak > final`, which measures the shape instead: an append-only
log grows monotonically, so its maximum *is* its final size, and a file that
was ever larger than it ended can only have been rewritten.

Worth recording why the obvious alternative does not work. An absolute size
bound cannot distinguish a working compactor here: the payload is one repeated
character, so ~48 MB of records LZ4-compress to ~3 MB whether or not anything is
reclaimed -- with compaction disabled entirely the file still ends at 3.1 MB. I
first wrote the comment claiming that bound was the strong one, then measured it
and found the opposite; `peak > final` is what goes red.
This commit is contained in:
Alexey Shakhmatov
2026-08-03 20:05:38 +03:00
parent 491a4d0a6a
commit 9390021b1e
3 changed files with 321 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
src/commands.zig
View File

@@ -760,27 +760,41 @@ fn scan_sorted(
// superseded-but-parseable bytes that the re-applied filter might accept.
// Loud beats silent: a wrong answer a test can see beats a missing
// candidate nothing can.
if (try index.plan(ctx.gpa, &coll.id_index, coll.indexes.items, filter, sort)) |p| {
var plan = p;
defer plan.deinit(ctx.gpa);
var offs: std.ArrayListUnmanaged(u64) = .empty;
defer offs.deinit(ctx.gpa);
try plan.search(ctx.gpa, &offs);
// Candidates arrive as a stream so that a whole-index read never
// materializes: at the tens-of-GB target a `countDocuments({})` would
// otherwise build a list of every offset in the collection before the
// first one is examined. A narrowed plan still materializes, because its
// multikey/$in dedupe genuinely needs the whole set, and it is bounded by
// selectivity.
var offs: std.ArrayListUnmanaged(u64) = .empty;
defer offs.deinit(ctx.gpa);
var cands: index.Candidates = undefined;
var plan_opt = try index.plan(ctx.gpa, &coll.id_index, coll.indexes.items, filter, sort);
defer if (plan_opt) |*p| p.deinit(ctx.gpa);
if (plan_opt) |*plan| {
if (sorted) |flag| flag.* = plan.provides_sort;
if (plan.provides_sort) lim = limit;
for (offs.items) |off| {
if (!try query.matches_bytes(ctx.gpa, filter, coll.doc_bytes(off))) continue;
if (out) |list| try list.append(ctx.gpa, off);
n += 1;
if (lim != 0 and n >= lim) break;
if (plan.full_scan()) {
cands = if (plan.backward)
.{ .scan_rev = plan.index.iter_reverse() }
else
.{ .scan = plan.index.iter() };
} else {
try plan.search(ctx.gpa, &offs);
cands = .{ .list = .{ .items = offs.items } };
}
return n;
} else {
// No usable predicate: every document, in _id order. The docs map was
// the fallback here, and its iteration order was the hash's; walking
// the _id_ index instead is ordered, streams, and does not depend on a
// structure that is going away.
cands = .{ .scan = coll.id_index.iter() };
}
var it = coll.docs.iterator();
while (it.next()) |entry| {
if (!try query.matches_bytes(ctx.gpa, filter, coll.doc_bytes(entry.value_ptr.*))) continue;
if (out) |list| try list.append(ctx.gpa, entry.value_ptr.*);
while (cands.next()) |off| {
if (!try query.matches_bytes(ctx.gpa, filter, coll.doc_bytes(off))) continue;
if (out) |list| try list.append(ctx.gpa, off);
n += 1;
if (lim != 0 and n >= lim) break;
}
@@ -1096,8 +1110,13 @@ fn cmd_aggregate(ctx: *Context, msg: *wire.Message, reply: *wire.Reply) !void {
_ = try scan_matching(ctx, db_name, coll_name, filter, 0, &offs);
stages = stages[1..];
} else {
var it = coll.docs.iterator();
while (it.next()) |entry| try offs.append(ctx.gpa, entry.value_ptr.*);
// Every document, in _id order. A pipeline materializes its stream
// anyway (stages need random access to the window), so this one stays a
// list -- but it comes from the _id_ index rather than the docs map,
// which is ordered and does not depend on a structure that is going
// away. Streaming the whole pipeline is M1's cursor work.
var it = coll.id_index.iter();
while (it.next()) |e| try offs.append(ctx.gpa, e.off);
}
var start: usize = 0;
@@ -2384,6 +2403,65 @@ test "count_only_pipeline accepts only shapes a count can answer" {
try testing.expect(try count_only_pipeline(&reply, &.{}) == null);
}
test "a sorted full scan over a multikey index returns each document once" {
// scan_sorted streams a whole-index read instead of materializing it, which
// is what keeps a countDocuments({}) from building a list of every offset in
// the collection. But one document contributes several entries to a multikey
// index, so walking that index end to end yields it once per array element.
// The materializing path deduped; a stream cannot, so Plan.full_scan()
// refuses multikey indexes and this shape keeps materializing.
//
// The shape is `find({}).sort({tags: 1})`: no filter, so the only reason to
// use an index at all is that it supplies the order.
//
// Mutation check, and it is worth stating precisely because the obvious
// version of it does nothing: two independent guards refuse this, so
// removing either one alone leaves the test green. `index_provides_sort`
// returns null for a multikey index, and `Plan.full_scan` refuses one
// again. Remove *both* and each document comes back three times. So this
// test pins the pair, not either guard -- which is the useful property,
// since it is the behaviour that matters rather than which check delivers
// it.
var threaded: std.Io.Threaded = .init_single_threaded;
defer threaded.deinit();
const io = threaded.io();
var tdb = try TestDb.init(io);
defer tdb.deinit();
try dispatch_insert(&tdb, io, "mk", &.{
.{ .doc = &.{
.{ .key = "_id", .value = .{ .int32 = 1 } },
.{ .key = "tags", .value = .{ .array = &.{ .{ .int32 = 1 }, .{ .int32 = 2 }, .{ .int32 = 3 } } } },
} },
.{ .doc = &.{
.{ .key = "_id", .value = .{ .int32 = 2 } },
.{ .key = "tags", .value = .{ .array = &.{ .{ .int32 = 4 }, .{ .int32 = 5 }, .{ .int32 = 6 } } } },
} },
});
try dispatch_create_index(&tdb, io, "mk", .{ .doc = &.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "tags", .value = .{ .int32 = 1 } }} } },
.{ .key = "name", .value = .{ .string = "tags_1" } },
} });
var ctx = tdb.ctx(io);
var msg = try parse_fake_msg("find", .{ .string = "mk" }, &.{
.{ .key = "filter", .value = .{ .doc = &.{} } },
.{ .key = "sort", .value = .{ .doc = &.{.{ .key = "tags", .value = .{ .int32 = 1 } }} } },
});
defer msg.deinit();
var reply = wire.Reply.init(testing.allocator);
defer reply.deinit();
try dispatch(&ctx, &msg, &reply);
try testing.expectEqual(@as(f64, 1.0), bson.get_pair(reply.pairs.items, "ok").?.double);
const cursor = bson.get_pair(reply.pairs.items, "cursor") orelse return error.TestUnexpectedResult;
const batch = switch (bson.get_pair(cursor.doc, "firstBatch") orelse return error.TestUnexpectedResult) {
.array => |arr| arr,
else => return error.TestUnexpectedResult,
};
// Two documents, not six.
try testing.expectEqual(@as(usize, 2), batch.len);
}
test "a command with no collection name errors and holds no lock" {
// Regression for a leaked catalog lock. dispatch resolved the namespace
// *after* taking the catalog lock, with `orelse return` -- and a plain