dev/MultiforaDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

index/db/commands/server: TTL indexes

Browse Source 
createIndex({expireAt: 1}, {expireAfterSeconds: 60}) now deletes a
document once its indexed date is that many seconds old.

index.zig carries the option: Index.ttl (?i64), parsed from
expireAfterSeconds (int32/int64/integral double, within MongoDB's
[0, 2147483647]; 0 means "expire at the stored instant"), emitted by
spec_pairs and so persisted through the log and reported by listIndexes,
and compared by spec_equal — a same-name re-create with a different
expiry stays IndexOptionsConflict, as in MongoDB, since collMod does not
exist here. The bound keeps the value an int32 on the wire and makes the
emission cast unconditional. TTL is single-field only (a compound key is
error.TtlOnCompoundIndex); an absent option parses to null, so every
existing log record reparses unchanged.

db.zig sweeps: Engine.ttl_sweep(now_ms) walks each TTL index's entries
and deletes through the ordinary remove path, so an expiry is logged and
fsynced like any other write and holds across a restart. Expired ids are
duped before removal — remove frees the docs-map key that Entry.id
aliases — then sorted and deduped, because one document can be expired
by several entries (an array of dates expires on its earliest member,
which multikey expansion gives for free) or by several TTL indexes.
Selecting entries is a type test rather than a range lookup: bson
compare order ranks datetime above null, numbers and strings, so a
datetime upper bound would also select every value of a lesser type. A
sweep that deleted something checks the compaction threshold, since a
TTL-only workload never reaches the one in upsert.

commands.zig maps the new spec errors: CannotCreateIndex (67) for a
compound key or an out-of-range expiry, and InvalidIndexSpecificationOption
(197) for an expiry on {_id: 1}, which the idempotent _id no-op would
otherwise swallow.

server.zig runs the monitor as a member of the connection group, so the
existing group.cancel tears it down; it sleeps first, then sweeps under
the write lock, and logs rather than dies on a sweep failure.
--ttl-sweep-secs sets the interval (default 60, 0 leaves it unspawned).

Expiry is coarse by design, as in MongoDB: a document stays visible
until the next sweep, and a non-date value at the indexed path never
expires. Unit tests cover the spec round-trip and every rejection, the
sweep (inclusive cutoff, string/missing/future values untouched, second
sweep a no-op) and its survival of a reopen, and two TTL indexes over
one collection. e2e4.js exercises it through the Node driver.
This commit is contained in:
Alexey Shakhmatov
2026-08-02 14:36:45 +03:00
parent 7482042f34
commit 3c1ab6f656
8 changed files with 721 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
src/index.zig
View File

@@ -66,15 +66,22 @@ pub const Index = struct {
keys: []const IndexKey,
unique: bool,
sparse: bool,
/// expireAfterSeconds when this is a TTL index, else null. Documents
/// whose indexed value is a datetime older than this many seconds are
/// deleted by Engine.ttl_sweep (src/db.zig); the index itself only
/// carries the setting. Always within
/// [0, max_expire_after_seconds] — parse_spec is the only producer.
ttl: ?i64,
multikey: bool,
entries: std.ArrayListUnmanaged(Entry),
pub fn init(gpa: std.mem.Allocator, name: []const u8, keys: []const IndexKey, unique: bool, sparse: bool) !Index {
pub fn init(gpa: std.mem.Allocator, name: []const u8, keys: []const IndexKey, unique: bool, sparse: bool, ttl: ?i64) !Index {
var self: Index = .{
.name = undefined,
.keys = undefined,
.unique = unique,
.sparse = sparse,
.ttl = ttl,
.multikey = false,
.entries = .empty,
};
@@ -311,8 +318,9 @@ pub const Index = struct {
// -- serialization ------------------------------------------------------
/// The canonical spec document bytes ({v, key, name, unique?, sparse?})
/// stored in the log and used to rebuild the index on replay.
/// The canonical spec document bytes
/// ({v, key, name, unique?, sparse?, expireAfterSeconds?}) stored in the
/// log and used to rebuild the index on replay.
pub fn write_spec(self: *const Index, gpa: std.mem.Allocator, out: *std.ArrayListUnmanaged(u8)) !void {
var arena = std.heap.ArenaAllocator.init(gpa);
defer arena.deinit();
@@ -333,11 +341,15 @@ pub const Index = struct {
try out.append(arena, .{ .key = "name", .value = .{ .string = self.name } });
if (self.unique) try out.append(arena, .{ .key = "unique", .value = .{ .bool = true } });
if (self.sparse) try out.append(arena, .{ .key = "sparse", .value = .{ .bool = true } });
// int32 like MongoDB: parse_spec caps the value at
// max_expire_after_seconds, so the cast always fits.
if (self.ttl) |secs| try out.append(arena, .{ .key = "expireAfterSeconds", .value = .{ .int32 = @intCast(secs) } });
}
pub fn spec_equal(a: *const Index, b: *const Index) bool {
if (!std.mem.eql(u8, a.name, b.name)) return false;
if (a.unique != b.unique or a.sparse != b.sparse) return false;
if (!std.meta.eql(a.ttl, b.ttl)) return false;
if (a.keys.len != b.keys.len) return false;
for (a.keys, b.keys) |ka, kb| {
if (!std.mem.eql(u8, ka.path, kb.path)) return false;
@@ -365,10 +377,10 @@ pub fn find_by_key_pattern(indexes: []const Index, key_pairs: []const bson.Pair)
return null;
}
pub const SpecError = error{ InvalidIndexSpec, OutOfMemory };
pub const SpecError = error{ InvalidIndexSpec, TtlOnCompoundIndex, InvalidExpireAfterSeconds, OutOfMemory };
/// Parse {key: {...}, name?, unique?, sparse?} from a spec document — the
/// form drivers send and the form the log stores.
/// Parse {key: {...}, name?, unique?, sparse?, expireAfterSeconds?} from a
/// spec document — the form drivers send and the form the log stores.
pub fn parse_spec(gpa: std.mem.Allocator, spec: *const bson.Document) SpecError!Index {
const key_value = bson.get_pair(spec.pairs, "key") orelse return error.InvalidIndexSpec;
const key_pairs = switch (key_value) {
@@ -391,16 +403,48 @@ pub fn parse_spec(gpa: std.mem.Allocator, spec: *const bson.Document) SpecError!
var sparse = false;
if (bson.get_pair(spec.pairs, "unique")) |v| unique = query.truthy(v);
if (bson.get_pair(spec.pairs, "sparse")) |v| sparse = query.truthy(v);
var ttl: ?i64 = null;
if (bson.get_pair(spec.pairs, "expireAfterSeconds")) |v| {
ttl = try expire_after_seconds(v);
// MongoDB's rule: TTL is a single-field index option. A compound key
// would leave it ambiguous which component dates the document.
if (key_pairs.len != 1) return error.TtlOnCompoundIndex;
}
const name_value = bson.get_pair(spec.pairs, "name") orelse {
const nm = try default_name(gpa, key_pairs);
defer gpa.free(nm);
return Index.init(gpa, nm, keys[0..key_pairs.len], unique, sparse);
return Index.init(gpa, nm, keys[0..key_pairs.len], unique, sparse, ttl);
};
const name = switch (name_value) {
.string => |s| s,
else => return error.InvalidIndexSpec,
};
return Index.init(gpa, name, keys[0..key_pairs.len], unique, sparse);
return Index.init(gpa, name, keys[0..key_pairs.len], unique, sparse, ttl);
}
/// MongoDB's bound on expireAfterSeconds. Keeping it means a TTL always
/// round-trips as an int32, exactly as a real server reports it.
pub const max_expire_after_seconds: i64 = 2147483647;
/// The seconds in an `expireAfterSeconds` option: integral and within
/// [0, max_expire_after_seconds]. 0 is legal (expire at the stored instant);
/// a negative, fractional, out-of-range or non-numeric value is not. A
/// double is accepted because that is what a JavaScript driver sends for a
/// plain number — and the range check runs before the conversion, so
/// @intFromFloat is always defined.
fn expire_after_seconds(v: bson.Value) error{InvalidExpireAfterSeconds}!i64 {
const secs: i64 = switch (v) {
.int32 => |n| n,
.int64 => |n| n,
.double => |d| blk: {
if (!std.math.isFinite(d) or @trunc(d) != d) return error.InvalidExpireAfterSeconds;
if (d < 0 or d > @as(f64, @floatFromInt(max_expire_after_seconds))) return error.InvalidExpireAfterSeconds;
break :blk @intFromFloat(d);
},
else => return error.InvalidExpireAfterSeconds,
};
if (secs < 0 or secs > max_expire_after_seconds) return error.InvalidExpireAfterSeconds;
return secs;
}
/// Whether a key-pattern direction value means descending. The single
@@ -851,7 +895,7 @@ fn doc_of(pairs: []const bson.Pair) bson.Document {
fn simple_index(gpa: std.mem.Allocator, paths: []const []const u8, unique: bool, sparse: bool) !Index {
var keys: [max_index_keys]IndexKey = undefined;
for (paths, 0..) |p, i| keys[i] = .{ .path = p, .descending = false };
return Index.init(gpa, "test", keys[0..paths.len], unique, sparse);
return Index.init(gpa, "test", keys[0..paths.len], unique, sparse, null);
}
/// Look up ids and compare with the expected set. Entry ids alias the
@@ -1103,6 +1147,119 @@ test "id fast path guards and $in" {
try testing.expect(plan_id(&doc_safe) != null);
}
test "TTL spec round-trips through write_spec and compares in spec_equal" {
const gpa = testing.allocator;
const spec = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "expireAt", .value = .{ .int32 = 1 } }} } },
.{ .key = "name", .value = .{ .string = "expireAt_1" } },
// A driver sends a plain JS number as a double.
.{ .key = "expireAfterSeconds", .value = .{ .double = 60.0 } },
});
var ix = try parse_spec(gpa, &spec);
defer ix.deinit(gpa);
try testing.expectEqual(@as(?i64, 60), ix.ttl);
// Serialize and reparse: the log/compaction path.
var bytes: std.ArrayListUnmanaged(u8) = .empty;
defer bytes.deinit(gpa);
try ix.write_spec(gpa, &bytes);
var reparsed_doc = try bson.Document.parse(gpa, bytes.items);
defer reparsed_doc.deinit();
try testing.expectEqual(@as(i32, 60), reparsed_doc.get("expireAfterSeconds").?.int32);
var ix2 = try parse_spec(gpa, &reparsed_doc);
defer ix2.deinit(gpa);
try testing.expect(Index.spec_equal(&ix, &ix2));
// Same name and key, different expiry: not the same spec (the command
// layer turns this into IndexOptionsConflict).
const other = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "expireAt", .value = .{ .int32 = 1 } }} } },
.{ .key = "name", .value = .{ .string = "expireAt_1" } },
.{ .key = "expireAfterSeconds", .value = .{ .int32 = 90 } },
});
var ix3 = try parse_spec(gpa, &other);
defer ix3.deinit(gpa);
try testing.expect(!Index.spec_equal(&ix, &ix3));
// The largest legal expiry, arriving as an int64, still round-trips as
// an int32 — the only type this ever emits.
const big = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "expireAt", .value = .{ .int32 = 1 } }} } },
.{ .key = "expireAfterSeconds", .value = .{ .int64 = max_expire_after_seconds } },
});
var ix_big = try parse_spec(gpa, &big);
defer ix_big.deinit(gpa);
bytes.clearRetainingCapacity();
try ix_big.write_spec(gpa, &bytes);
var big_doc = try bson.Document.parse(gpa, bytes.items);
defer big_doc.deinit();
try testing.expectEqual(@as(i32, 2147483647), big_doc.get("expireAfterSeconds").?.int32);
var ix_big2 = try parse_spec(gpa, &big_doc);
defer ix_big2.deinit(gpa);
try testing.expectEqual(@as(?i64, max_expire_after_seconds), ix_big2.ttl);
// No option at all: ttl null, and nothing emitted (old logs reparse
// unchanged).
const plain = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "expireAt", .value = .{ .int32 = 1 } }} } },
});
var ix4 = try parse_spec(gpa, &plain);
defer ix4.deinit(gpa);
try testing.expect(ix4.ttl == null);
bytes.clearRetainingCapacity();
try ix4.write_spec(gpa, &bytes);
var plain_doc = try bson.Document.parse(gpa, bytes.items);
defer plain_doc.deinit();
try testing.expect(plain_doc.get("expireAfterSeconds") == null);
}
test "TTL spec rejects compound keys and bad expireAfterSeconds" {
const gpa = testing.allocator;
const compound = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{
.{ .key = "a", .value = .{ .int32 = 1 } },
.{ .key = "b", .value = .{ .int32 = 1 } },
} } },
.{ .key = "expireAfterSeconds", .value = .{ .int32 = 60 } },
});
try testing.expectError(error.TtlOnCompoundIndex, parse_spec(gpa, &compound));
const bad = [_]bson.Value{
.{ .int32 = -1 },
.{ .int64 = -1 },
.{ .double = -0.5 },
.{ .double = 1.5 },
// Past MongoDB's bound, as an int and as a double (1e19 would also
// overflow the conversion, which the range check runs before).
.{ .int64 = max_expire_after_seconds + 1 },
.{ .double = 3e9 },
.{ .double = 1e19 },
.{ .double = std.math.inf(f64) },
.{ .double = std.math.nan(f64) },
.{ .string = "60" },
.{ .bool = true },
.null,
};
for (bad) |v| {
const spec = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "expireAt", .value = .{ .int32 = 1 } }} } },
.{ .key = "expireAfterSeconds", .value = v },
});
try testing.expectError(error.InvalidExpireAfterSeconds, parse_spec(gpa, &spec));
}
// 0 is legal: expire at exactly the stored instant.
const zero = doc_of(&.{
.{ .key = "key", .value = .{ .doc = &.{.{ .key = "expireAt", .value = .{ .int32 = 1 } }} } },
.{ .key = "expireAfterSeconds", .value = .{ .int32 = 0 } },
});
var ix = try parse_spec(gpa, &zero);
defer ix.deinit(gpa);
try testing.expectEqual(@as(?i64, 0), ix.ttl);
// The default name still comes from the key pattern.
try testing.expectEqualStrings("expireAt_1", ix.name);
}
test "planner picks eq run, ranges, and bails on sparse null" {
const gpa = testing.allocator;
var ix = try simple_index(gpa, &.{ "a", "b" }, false, false);