dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index: secondary index core — entries, search, planner, _id fast path

Browse Source 
Adds src/index.zig with the full secondary-index machinery: entry
generation mirroring field_matches (array value + elements), BSON-order
sorted entries with binary search, compound prefix and range lookups,
unique/sparse options, the query planner (longest equality/$in run +
optional range, $in cartesian cap, sparse/null bail), and the _id_ fast
path guarded against serialization-ambiguous values (numbers, strings,
symbols, codes, opaque payloads).

query.collect_values is now pub so entry generation can mirror it exactly.
storage.zig gains record_type_index_create/drop; lib.zig exports index.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 12:21:22 +03:00
parent 662df9b121
commit a38ddc2f50
10 changed files with 1824 additions and 782 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
src/bson.zig
View File

@@ -128,14 +128,17 @@ pub const Document = struct {
};
pub fn get_pair(pairs: []const Pair, key: []const u8) ?Value {
for (pairs) |p| {
if (std.mem.eql(u8, p.key, key)) return p.value;
}
return null;
const i = get_pair_index(pairs, key) orelse return null;
return pairs[i].value;
}
pub fn parse_doc(allocator: std.mem.Allocator, bytes: []const u8) !Document {
return Document.parse(allocator, bytes);
/// Position of `key` in `pairs` — for callers that need to replace or remove
/// the pair in place rather than just read it.
pub fn get_pair_index(pairs: []const Pair, key: []const u8) ?usize {
for (pairs, 0..) |p, i| {
if (std.mem.eql(u8, p.key, key)) return i;
}
return null;
}
// ---------------------------------------------------------------------------
@@ -145,10 +148,6 @@ pub fn parse_doc(allocator: std.mem.Allocator, bytes: []const u8) !Document {
const Parser = struct {
arena: *std.heap.ArenaAllocator,
bytes: []const u8,
fn fail() error{InvalidBson} {
return error.InvalidBson;
}
};
const ParseError = error{ InvalidBson, OutOfMemory };
@@ -162,14 +161,21 @@ fn ensure_available(bytes: []const u8, idx: usize, n: usize) error{InvalidBson}!
if (bytes.len -| idx < n) return error.InvalidBson;
}
/// Validate the length-prefixed frame starting at `start` — documents and
/// arrays share it — and return the offset just past its terminating byte.
fn doc_extent(p: Parser, start: usize) ParseError!usize {
try ensure_available(p.bytes, start, 4);
const total: u32 = std.mem.readInt(u32, p.bytes[start..][0..4], .little);
if (total < 5) return error.InvalidBson;
if (p.bytes.len - start < total) return error.InvalidBson;
const end = start + total;
if (p.bytes[end - 1] != 0x00) return error.InvalidBson;
return end;
}
fn parse_doc_inner(p: Parser, idx: *usize) ParseError![]const Pair {
const start = idx.*;
ensure_available(p.bytes, start, 4) catch return Parser.fail();
const total: u32 = std.mem.readInt(u32, p.bytes[start..][0..4], .little);
if (total < 5) return Parser.fail();
if (p.bytes.len - start < total) return Parser.fail();
const end = start + total;
if (p.bytes[end - 1] != 0x00) return Parser.fail();
const end = try doc_extent(p, start);
const gpa = p.arena.allocator();
var pairs: std.ArrayListUnmanaged(Pair) = .empty;
@@ -180,13 +186,13 @@ fn parse_doc_inner(p: Parser, idx: *usize) ParseError![]const Pair {
const value = try parse_element(p, idx);
try pairs.append(gpa, value);
}
if (idx.* != end - 1) return Parser.fail();
if (idx.* != end - 1) return error.InvalidBson;
idx.* = end;
return pairs.toOwnedSlice(gpa);
}
fn parse_element(p: Parser, idx: *usize) ParseError!Pair {
ensure_available(p.bytes, idx.*, 1) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 1);
const tag = p.bytes[idx.*];
idx.* += 1;
const key = try parse_cstring(p, idx);
@@ -197,7 +203,7 @@ fn parse_element(p: Parser, idx: *usize) ParseError!Pair {
fn parse_cstring(p: Parser, idx: *usize) ParseError![]const u8 {
const start = idx.*;
while (idx.* < p.bytes.len and p.bytes[idx.*] != 0) idx.* += 1;
if (idx.* >= p.bytes.len) return Parser.fail();
if (idx.* >= p.bytes.len) return error.InvalidBson;
idx.* += 1;
// Strings are copied into the arena so documents are self-contained and
// outlive the input buffer (wire messages and log records are transient).
@@ -205,11 +211,11 @@ fn parse_cstring(p: Parser, idx: *usize) ParseError![]const u8 {
}
fn parse_string(p: Parser, idx: *usize) ParseError![]const u8 {
ensure_available(p.bytes, idx.*, 4) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 4);
const len: u32 = std.mem.readInt(u32, p.bytes[idx.*..][0..4], .little);
if (len == 0 or len > p.bytes.len - (idx.* + 4)) return Parser.fail();
if (len == 0 or len > p.bytes.len - (idx.* + 4)) return error.InvalidBson;
const str = p.bytes[idx.* + 4 .. idx.* + 4 + len];
if (str[len - 1] != 0) return Parser.fail();
if (str[len - 1] != 0) return error.InvalidBson;
idx.* += 4 + len;
return p.arena.allocator().dupe(u8, str[0 .. len - 1]);
}
@@ -217,7 +223,7 @@ fn parse_string(p: Parser, idx: *usize) ParseError![]const u8 {
fn parse_value(p: Parser, tag: u8, idx: *usize) ParseError!Value {
return switch (tag) {
0x01 => blk: {
ensure_available(p.bytes, idx.*, 8) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 8);
const v: f64 = @bitCast(std.mem.readInt(u64, p.bytes[idx.*..][0..8], .little));
idx.* += 8;
break :blk .{ .double = v };
@@ -226,30 +232,30 @@ fn parse_value(p: Parser, tag: u8, idx: *usize) ParseError!Value {
0x03 => .{ .doc = try parse_doc_inner(p, idx) },
0x04 => .{ .array = try parse_array(p, idx) },
0x05 => blk: {
ensure_available(p.bytes, idx.*, 5) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 5);
const len: u32 = std.mem.readInt(u32, p.bytes[idx.*..][0..4], .little);
const subtype = p.bytes[idx.* + 4];
ensure_available(p.bytes, idx.* + 5, len) catch return Parser.fail();
try ensure_available(p.bytes, idx.* + 5, len);
const data = p.bytes[idx.* + 5 .. idx.* + 5 + len];
idx.* += 5 + len;
break :blk .{ .binary = .{ .subtype = subtype, .data = try p.arena.allocator().dupe(u8, data) } };
},
0x06 => .{ .opaque_val = .{ .kind = 0x06, .data = &.{} } },
0x07 => blk: {
ensure_available(p.bytes, idx.*, 12) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 12);
const oid: ObjectId = p.bytes[idx.*..][0..12].*;
idx.* += 12;
break :blk .{ .object_id = oid };
},
0x08 => blk: {
ensure_available(p.bytes, idx.*, 1) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 1);
const v = p.bytes[idx.*];
if (v > 1) return Parser.fail();
if (v > 1) return error.InvalidBson;
idx.* += 1;
break :blk .{ .bool = v == 1 };
},
0x09 => blk: {
ensure_available(p.bytes, idx.*, 8) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 8);
const v: i64 = std.mem.readInt(i64, p.bytes[idx.*..][0..8], .little);
idx.* += 8;
break :blk .{ .datetime = v };
@@ -262,7 +268,7 @@ fn parse_value(p: Parser, tag: u8, idx: *usize) ParseError!Value {
0x0C => blk: {
const start = idx.*;
_ = try parse_string(p, idx);
ensure_available(p.bytes, idx.*, 12) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 12);
idx.* += 12;
// Like all other types, the payload is copied into the arena so
// documents stay valid after the input buffer is reused.
@@ -271,49 +277,44 @@ fn parse_value(p: Parser, tag: u8, idx: *usize) ParseError!Value {
0x0D => .{ .code = try parse_string(p, idx) },
0x0E => .{ .symbol = try parse_string(p, idx) },
0x0F => blk: {
ensure_available(p.bytes, idx.*, 4) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 4);
const total: u32 = std.mem.readInt(u32, p.bytes[idx.*..][0..4], .little);
if (total < 4 or total > p.bytes.len - idx.*) return Parser.fail();
if (total < 4 or total > p.bytes.len - idx.*) return error.InvalidBson;
const data = p.bytes[idx.* .. idx.* + total];
idx.* += total;
break :blk .{ .opaque_val = .{ .kind = 0x0F, .data = try p.arena.allocator().dupe(u8, data) } };
},
0x10 => blk: {
ensure_available(p.bytes, idx.*, 4) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 4);
const v: i32 = std.mem.readInt(i32, p.bytes[idx.*..][0..4], .little);
idx.* += 4;
break :blk .{ .int32 = v };
},
0x11 => blk: {
ensure_available(p.bytes, idx.*, 8) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 8);
const v: u64 = std.mem.readInt(u64, p.bytes[idx.*..][0..8], .little);
idx.* += 8;
break :blk .{ .timestamp = v };
},
0x12 => blk: {
ensure_available(p.bytes, idx.*, 8) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 8);
const v: i64 = std.mem.readInt(i64, p.bytes[idx.*..][0..8], .little);
idx.* += 8;
break :blk .{ .int64 = v };
},
0x13 => blk: {
ensure_available(p.bytes, idx.*, 16) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 16);
const v: [16]u8 = p.bytes[idx.*..][0..16].*;
idx.* += 16;
break :blk .{ .decimal128 = v };
},
else => return Parser.fail(),
else => return error.InvalidBson,
};
}
fn parse_array(p: Parser, idx: *usize) ParseError![]const Value {
const start = idx.*;
ensure_available(p.bytes, start, 4) catch return Parser.fail();
const total: u32 = std.mem.readInt(u32, p.bytes[start..][0..4], .little);
if (total < 5) return Parser.fail();
if (p.bytes.len - start < total) return Parser.fail();
const end = start + total;
if (p.bytes[end - 1] != 0x00) return Parser.fail();
const end = try doc_extent(p, start);
const gpa = p.arena.allocator();
var values: std.ArrayListUnmanaged(Value) = .empty;
@@ -321,13 +322,13 @@ fn parse_array(p: Parser, idx: *usize) ParseError![]const Value {
idx.* = start + 4;
while (idx.* < end - 1) {
ensure_available(p.bytes, idx.*, 1) catch return Parser.fail();
try ensure_available(p.bytes, idx.*, 1);
const tag = p.bytes[idx.*];
idx.* += 1;
_ = try parse_cstring(p, idx);
try values.append(gpa, try parse_value(p, tag, idx));
}
if (idx.* != end - 1) return Parser.fail();
if (idx.* != end - 1) return error.InvalidBson;
idx.* = end;
return values.toOwnedSlice(gpa);
}
@@ -417,22 +418,31 @@ pub fn write_element(pair: Pair, gpa: std.mem.Allocator, out: *std.ArrayListUnma
try write_value(pair.value, gpa, out);
}
/// Write a length-prefixed document. Length is patched in after the body.
pub fn write_doc(pairs: []const Pair, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) SerializeError!void {
/// Reserve the 4-byte length prefix of a document or array frame. Returns the
/// offset to hand back to `end_frame`.
fn begin_frame(gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) SerializeError!usize {
const len_pos = out.items.len;
var zero: [4]u8 = [4]u8{ 0, 0, 0, 0 };
try out.appendSlice(gpa, &zero);
for (pairs) |p| try write_element(p, gpa, out);
try out.appendSlice(gpa, &[4]u8{ 0, 0, 0, 0 });
return len_pos;
}
/// Terminate the frame opened at `len_pos` and patch in its total length.
fn end_frame(gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8), len_pos: usize) SerializeError!void {
try out.append(gpa, 0);
const total = out.items.len - len_pos;
if (total > std.math.maxInt(u32)) return error.BsonTooLarge;
std.mem.writeInt(u32, out.items[len_pos..][0..4], @intCast(total), .little);
}
fn write_array(items: []const Value, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) !void {
const len_pos = out.items.len;
var zero: [4]u8 = [4]u8{ 0, 0, 0, 0 };
try out.appendSlice(gpa, &zero);
/// Write a length-prefixed document. Length is patched in after the body.
pub fn write_doc(pairs: []const Pair, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) SerializeError!void {
const len_pos = try begin_frame(gpa, out);
for (pairs) |p| try write_element(p, gpa, out);
try end_frame(gpa, out, len_pos);
}
fn write_array(items: []const Value, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) SerializeError!void {
const len_pos = try begin_frame(gpa, out);
var buf: [16]u8 = undefined;
for (items, 0..) |item, i| {
try out.append(gpa, item.type_tag());
@@ -440,10 +450,7 @@ fn write_array(items: []const Value, gpa: std.mem.Allocator, out: *std.ArrayList
try write_cstring(key, gpa, out);
try write_value(item, gpa, out);
}
try out.append(gpa, 0);
const total = out.items.len - len_pos;
if (total > std.math.maxInt(u32)) return error.BsonTooLarge;
std.mem.writeInt(u32, out.items[len_pos..][0..4], @intCast(total), .little);
try end_frame(gpa, out, len_pos);
}
/// Serialize a single value with its type byte (no key) — used for `_id`