baseline: mongo-light working tree before concurrency refactor
This commit is contained in:
mongo-light
329
src/storage.zig
Normal file
329
src/storage.zig
Normal file
@@ -0,0 +1,329 @@
|
||||
//! Append-only record log. Each record is:
|
||||
//! [0..4) u32 len — total record bytes
|
||||
//! [4..8) u32 crc32 over bytes [8..len)
|
||||
//! [8..16) u64 seq
|
||||
//! [16] u8 type
|
||||
//! [17..20) reserved
|
||||
//! [20..) db\0 coll\0 bson doc
|
||||
//! All integers little-endian. Reads and writes are positional, so the fd
|
||||
//! offset never matters. A torn tail record (crash mid-append) is detected
|
||||
//! during replay and skipped.
|
||||
|
||||
const std = @import("std");
|
||||
const bson = @import("bson.zig");
|
||||
|
||||
pub const record_type_upsert: u8 = 1;
|
||||
pub const record_type_delete: u8 = 2;
|
||||
|
||||
pub const header_len: usize = 20; // len + crc + seq + type + reserved
|
||||
|
||||
/// Largest record payload we will accept during replay. Matches the
|
||||
/// announced maxBsonObjectSize with room for names and header.
|
||||
pub const max_record_payload: usize = 16 * 1024 * 1024 + 64 * 1024;
|
||||
|
||||
pub const Record = struct {
|
||||
seq: u64,
|
||||
type: u8,
|
||||
db: []const u8, // transient: valid only during replay callback
|
||||
coll: []const u8,
|
||||
doc: []const u8, // raw bson bytes
|
||||
};
|
||||
|
||||
pub const Error = error{
|
||||
InvalidLog, // corrupt interior record (bad CRC or impossible length)
|
||||
};
|
||||
|
||||
/// Callback receives transient slices and a heap-allocated, freshly parsed
|
||||
/// document. The callback takes ownership of the document (must deinit).
|
||||
pub const ReplayFn = *const fn (ctx: *anyopaque, record: Record, doc: *bson.Document) anyerror!void;
|
||||
|
||||
pub const Log = struct {
|
||||
gpa: std.mem.Allocator,
|
||||
io: std.Io,
|
||||
file: std.Io.File,
|
||||
path: []const u8,
|
||||
end_pos: u64,
|
||||
log_bytes: u64, // bytes written since the log was last rewritten
|
||||
|
||||
pub fn open(gpa: std.mem.Allocator, io: std.Io, path: []const u8) !Log {
|
||||
// Resolve to an absolute path so compaction can rename the file
|
||||
// without depending on the caller's working directory.
|
||||
const abs_path = blk: {
|
||||
if (path.len > 0 and path[0] == '/') break :blk try gpa.dupe(u8, path);
|
||||
const cwd = try std.process.currentPathAlloc(io, gpa);
|
||||
defer gpa.free(cwd);
|
||||
break :blk try std.fmt.allocPrint(gpa, "{s}/{s}", .{ cwd, path });
|
||||
};
|
||||
errdefer gpa.free(abs_path);
|
||||
|
||||
const dir = std.Io.Dir.cwd();
|
||||
const file: std.Io.File = dir.openFile(io, abs_path, .{ .mode = .read_write }) catch |err| switch (err) {
|
||||
error.FileNotFound => try dir.createFile(io, abs_path, .{ .read = true }),
|
||||
else => return err,
|
||||
};
|
||||
return .{
|
||||
.gpa = gpa,
|
||||
.io = io,
|
||||
.file = file,
|
||||
.path = abs_path,
|
||||
.end_pos = 0,
|
||||
.log_bytes = 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn close(self: *Log) void {
|
||||
self.file.close(self.io);
|
||||
self.gpa.free(self.path);
|
||||
}
|
||||
|
||||
/// Replay all valid records from the beginning of the file.
|
||||
pub fn replay(self: *Log, ctx: *anyopaque, callback: ReplayFn) !void {
|
||||
var chunk: [64 * 1024]u8 = undefined;
|
||||
var heap_buf: []u8 = &.{};
|
||||
defer if (heap_buf.len > 0) self.gpa.free(heap_buf);
|
||||
var pos: u64 = 0;
|
||||
|
||||
while (true) {
|
||||
const len_read = self.file.readPositionalAll(self.io, chunk[0..4], pos) catch |err| {
|
||||
std.debug.print("mongo-light: log read error at {d}: {s}\n", .{ pos, @errorName(err) });
|
||||
return error.InvalidLog;
|
||||
};
|
||||
if (len_read == 0) return; // clean end
|
||||
if (len_read < 4) return; // torn tail
|
||||
const total: u32 = std.mem.readInt(u32, chunk[0..4], .little);
|
||||
if (total < header_len) {
|
||||
std.debug.print("mongo-light: corrupt record length {d} at {d}\n", .{ total, pos });
|
||||
return error.InvalidLog;
|
||||
}
|
||||
const payload_len: usize = total - 4;
|
||||
// Documents up to maxBsonObjectSize are legal; anything larger is
|
||||
// corruption. Covers a hostile length prefix from a truncated file.
|
||||
if (payload_len > max_record_payload) {
|
||||
std.debug.print("mongo-light: record too large at {d}\n", .{pos});
|
||||
return error.InvalidLog;
|
||||
}
|
||||
const payload = if (payload_len <= chunk.len) chunk[0..payload_len] else blk: {
|
||||
if (heap_buf.len < payload_len) {
|
||||
if (heap_buf.len > 0) self.gpa.free(heap_buf);
|
||||
heap_buf = try self.gpa.alloc(u8, payload_len);
|
||||
}
|
||||
break :blk heap_buf[0..payload_len];
|
||||
};
|
||||
const payload_read = self.file.readPositionalAll(self.io, payload, pos + 4) catch return error.InvalidLog;
|
||||
if (payload_read < payload_len) return; // torn tail — crash during append
|
||||
const crc_stored: u32 = std.mem.readInt(u32, payload[0..4], .little);
|
||||
const crc_actual = std.hash.Crc32.hash(payload[4..payload_len]);
|
||||
if (crc_stored != crc_actual) return error.InvalidLog;
|
||||
|
||||
var idx: usize = 16; // after crc + seq + type + reserved
|
||||
const seq: u64 = std.mem.readInt(u64, payload[4..12], .little);
|
||||
const rtype = payload[12];
|
||||
const db = read_cstring(payload, &idx) orelse return error.InvalidLog;
|
||||
const coll = read_cstring(payload, &idx) orelse return error.InvalidLog;
|
||||
if (idx > payload_len) return error.InvalidLog;
|
||||
const doc_bytes = payload[idx..payload_len];
|
||||
|
||||
const doc = try self.gpa.create(bson.Document);
|
||||
doc.* = bson.Document.parse(self.gpa, doc_bytes) catch {
|
||||
self.gpa.destroy(doc);
|
||||
std.debug.print("mongo-light: unparseable doc in log at {d}\n", .{pos});
|
||||
return error.InvalidLog;
|
||||
};
|
||||
try callback(ctx, .{
|
||||
.seq = seq,
|
||||
.type = rtype,
|
||||
.db = db,
|
||||
.coll = coll,
|
||||
.doc = doc_bytes,
|
||||
}, doc);
|
||||
|
||||
pos += total;
|
||||
self.end_pos = pos;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn append_upsert(self: *Log, db: []const u8, coll: []const u8, doc: []const u8, seq: u64) !void {
|
||||
try self.append(record_type_upsert, db, coll, doc, seq);
|
||||
}
|
||||
|
||||
pub fn append_delete(self: *Log, db: []const u8, coll: []const u8, doc: []const u8, seq: u64) !void {
|
||||
try self.append(record_type_delete, db, coll, doc, seq);
|
||||
}
|
||||
|
||||
fn append(self: *Log, rtype: u8, db: []const u8, coll: []const u8, doc: []const u8, seq: u64) !void {
|
||||
if (std.mem.indexOfScalar(u8, db, 0) != null or std.mem.indexOfScalar(u8, coll, 0) != null) {
|
||||
return error.NulInName;
|
||||
}
|
||||
var buf: std.ArrayListUnmanaged(u8) = .empty;
|
||||
defer buf.deinit(self.gpa);
|
||||
try buf.appendNTimes(self.gpa, 0, header_len);
|
||||
std.mem.writeInt(u64, buf.items[8..16], seq, .little);
|
||||
buf.items[16] = rtype;
|
||||
try buf.appendSlice(self.gpa, db);
|
||||
try buf.append(self.gpa, 0);
|
||||
try buf.appendSlice(self.gpa, coll);
|
||||
try buf.append(self.gpa, 0);
|
||||
try buf.appendSlice(self.gpa, doc);
|
||||
if (buf.items.len > std.math.maxInt(u32)) return error.LogTooLarge;
|
||||
const total: u32 = @intCast(buf.items.len);
|
||||
std.mem.writeInt(u32, buf.items[0..4], total, .little);
|
||||
std.mem.writeInt(u32, buf.items[4..8], std.hash.Crc32.hash(buf.items[8..]), .little);
|
||||
try self.file.writePositionalAll(self.io, buf.items, self.end_pos);
|
||||
self.end_pos += buf.items.len;
|
||||
self.log_bytes += buf.items.len;
|
||||
try self.file.sync(self.io);
|
||||
}
|
||||
|
||||
fn read_cstring(bytes: []const u8, idx: *usize) ?[]const u8 {
|
||||
const start = idx.*;
|
||||
while (idx.* < bytes.len and bytes[idx.*] != 0) idx.* += 1;
|
||||
if (idx.* >= bytes.len) return null;
|
||||
idx.* += 1;
|
||||
return bytes[start .. idx.* - 1];
|
||||
}
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const testing = std.testing;
|
||||
|
||||
const TmpLog = struct {
|
||||
tmp: std.testing.TmpDir,
|
||||
path: []u8,
|
||||
|
||||
fn init(gpa: std.mem.Allocator) !TmpLog {
|
||||
const tmp = std.testing.tmpDir(.{});
|
||||
const path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.log", .{tmp.sub_path});
|
||||
return .{ .tmp = tmp, .path = path };
|
||||
}
|
||||
|
||||
fn deinit(self: *TmpLog, gpa: std.mem.Allocator) void {
|
||||
self.tmp.cleanup();
|
||||
gpa.free(self.path);
|
||||
}
|
||||
};
|
||||
|
||||
test "append, replay, torn tail" {
|
||||
var threaded: std.Io.Threaded = .init_single_threaded;
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
const gpa = testing.allocator;
|
||||
|
||||
var tmp = try TmpLog.init(gpa);
|
||||
defer tmp.deinit(gpa);
|
||||
var log = try Log.open(gpa, io, tmp.path);
|
||||
defer log.close();
|
||||
|
||||
const doc_bytes = [_]u8{
|
||||
0x0E, 0x00, 0x00, 0x00, // len 14
|
||||
0x10, '_', 'i', 'd', 0, 0x2A, 0x00, 0x00, 0x00, // _id: 42
|
||||
0x00,
|
||||
};
|
||||
try log.append_upsert("db1", "coll1", &doc_bytes, 1);
|
||||
try log.append_delete("db1", "coll1", &doc_bytes, 2);
|
||||
|
||||
var seen: std.ArrayListUnmanaged(u8) = .empty;
|
||||
defer seen.deinit(gpa);
|
||||
const Ctx = struct {
|
||||
seen: *std.ArrayListUnmanaged(u8),
|
||||
gpa: std.mem.Allocator,
|
||||
fn apply(ctx: *anyopaque, record: Record, doc: *bson.Document) anyerror!void {
|
||||
const self: *@This() = @ptrCast(@alignCast(ctx));
|
||||
try self.seen.append(self.gpa, record.type);
|
||||
try self.seen.append(self.gpa, @intCast(record.seq));
|
||||
doc.deinit();
|
||||
self.gpa.destroy(doc);
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .seen = &seen, .gpa = gpa };
|
||||
try log.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
|
||||
try testing.expectEqualSlices(u8, &[_]u8{ record_type_upsert, 1, record_type_delete, 2 }, seen.items);
|
||||
try testing.expectEqual(@as(u64, log.end_pos), log.end_pos);
|
||||
}
|
||||
|
||||
test "record larger than the read chunk replays" {
|
||||
var threaded: std.Io.Threaded = .init_single_threaded;
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
const gpa = testing.allocator;
|
||||
|
||||
var tmp = try TmpLog.init(gpa);
|
||||
defer tmp.deinit(gpa);
|
||||
var log = try Log.open(gpa, io, tmp.path);
|
||||
defer log.close();
|
||||
|
||||
// Build a doc whose payload pushes the record past the 64 KiB stack chunk.
|
||||
const big = try gpa.alloc(u8, 80 * 1024);
|
||||
defer gpa.free(big);
|
||||
@memset(big, 'x');
|
||||
const pairs = [_]bson.Pair{
|
||||
.{ .key = "_id", .value = .{ .int32 = 1 } },
|
||||
.{ .key = "blob", .value = .{ .binary = .{ .subtype = 0, .data = big } } },
|
||||
};
|
||||
var out: std.ArrayListUnmanaged(u8) = .empty;
|
||||
defer out.deinit(gpa);
|
||||
try bson.write_doc(&pairs, gpa, &out);
|
||||
try log.append_upsert("db", "big", out.items, 1);
|
||||
|
||||
var count: usize = 0;
|
||||
const Ctx = struct {
|
||||
count: *usize,
|
||||
gpa: std.mem.Allocator,
|
||||
fn apply(ctx: *anyopaque, _: Record, doc: *bson.Document) anyerror!void {
|
||||
const self: *@This() = @ptrCast(@alignCast(ctx));
|
||||
try testing.expectEqual(@as(usize, 80 * 1024), doc.get("blob").?.binary.data.len);
|
||||
self.count.* += 1;
|
||||
doc.deinit();
|
||||
self.gpa.destroy(doc);
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .count = &count, .gpa = gpa };
|
||||
try log.replay(@ptrCast(&ctx), Ctx.apply);
|
||||
try testing.expectEqual(@as(usize, 1), count);
|
||||
}
|
||||
|
||||
test "reject corrupt interior record" {
|
||||
var threaded: std.Io.Threaded = .init_single_threaded;
|
||||
defer threaded.deinit();
|
||||
const io = threaded.io();
|
||||
const gpa = testing.allocator;
|
||||
|
||||
var tmp = try TmpLog.init(gpa);
|
||||
defer tmp.deinit(gpa);
|
||||
const path = tmp.path;
|
||||
|
||||
var log = try Log.open(gpa, io, path);
|
||||
const doc_bytes = [_]u8{ 0x0E, 0, 0, 0, 0x10, '_', 'i', 'd', 0, 42, 0, 0, 0, 0 };
|
||||
try log.append_upsert("db", "c", &doc_bytes, 1);
|
||||
log.close();
|
||||
|
||||
// Corrupt the file: flip a byte in the middle of the record.
|
||||
const dir = std.Io.Dir.cwd();
|
||||
var f = try dir.openFile(io, path, .{ .mode = .read_write });
|
||||
var buf: [64]u8 = undefined;
|
||||
const n = try f.readPositionalAll(io, &buf, 0);
|
||||
_ = n;
|
||||
buf[25] ^= 0xFF;
|
||||
try f.writePositionalAll(io, buf[0..64], 0);
|
||||
f.close(io);
|
||||
|
||||
var log2 = try Log.open(gpa, io, path);
|
||||
defer log2.close();
|
||||
defer dir.deleteFile(io, path) catch {};
|
||||
var count: usize = 0;
|
||||
const Ctx = struct {
|
||||
count: *usize,
|
||||
gpa: std.mem.Allocator,
|
||||
fn apply(ctx: *anyopaque, _: Record, doc: *bson.Document) anyerror!void {
|
||||
const self: *@This() = @ptrCast(@alignCast(ctx));
|
||||
self.count.* += 1;
|
||||
doc.deinit();
|
||||
self.gpa.destroy(doc);
|
||||
}
|
||||
};
|
||||
var ctx = Ctx{ .count = &count, .gpa = gpa };
|
||||
try testing.expectError(error.InvalidLog, log2.replay(@ptrCast(&ctx), Ctx.apply));
|
||||
}
|
||||
Reference in New Issue
Block a user