//! The data file: a checkpoint of the engine's structures, mapped rather than //! parsed. The log (storage.zig) remains the WAL and the source of truth; this //! file is a snapshot at some sequence number, and on open the log replays //! whatever happened after it (PLAN D4). //! //! File layout — an array of 4 KiB pages: //! page 0 header. Written at create, never rewritten. //! [0..4) u32 magic "MFDB" (0x4D464442) //! [4] u8 format version (1) //! [5] u8 page_shift (12) //! [6..8) u16 reserved (0) //! [8..24) u128 database uuid — must match the log's, so a data file //! can never be paired with a foreign log //! [24..32) u64 xxhash3 over [0..24) //! [32..4096) zero //! pages 1,2 watermark slots A and B. Generation N writes slot (N & 1), //! so a torn write can never leave zero valid slots -- writing //! a new generation over the only copy of the old one could. //! The newer slot whose hash validates is authoritative. //! [0..8) u64 generation (monotonic, >= 1) //! [8..16) u64 seq -- the log sequence this image covers //! [16..24) u64 alloc_tail -- pages [0, alloc_tail) are allocated //! [24..32) u64 file_pages //! [32..40) u64 catalog_page -- first page of the catalog stream //! [40..48) u64 catalog_len -- bytes of it //! [48..56) u64 freelist_page //! [56..64) u64 freelist_len //! [64..72) u64 prev_generation -- kept intact for fallback //! [72..80) u64 live_docs -- cached hint //! [80..88) u64 dead_bytes -- cached, drives the rebuild trigger //! [88..4088) reserved (zero) //! [4088..4096) u64 xxhash3 over [0..4088) //! pages 3.. data, handed out by a tail-bump extent allocator //! //! The catalog stream and the free list are written *wholesale into freshly //! allocated pages* at every checkpoint, never mutated in place. That is what //! makes them untearable: the previous copy stays intact and referenced by the //! previous watermark until the new watermark switches over. //! //! Every persisted reference is a page number (u32) or an absolute file byte //! offset (u64) — never a pointer — so where the file happens to be mapped is //! irrelevant. Node pages are raw host memory (D4: on-disk format == in-memory //! format, no serialization on page-in), which makes the data file //! little-endian-only; the log, framed field by field, stays portable. //! //! PLAN D6.1 called for a region table. It cannot be that: there is one node //! arena per index and one document slab per collection, so the region count is //! dynamic and unbounded, and N contiguous regions cannot all grow at the tail. //! With one page array and extent allocation there is exactly one growth path, //! therefore exactly one place where the write-then-extend discipline lives, //! and `ls`/`du` stay honest for the backup story (D6.6). //! //! Why this reaches for std.posix instead of std.Io, against the house style: //! `std.Io.File.MemoryMap` prefaults by default (`populate = true`), which is //! the opposite of "RSS = working set"; it exposes no NORESERVE, FIXED or //! address hint, so it cannot express an address-space reservation; and its //! `setLength` is mremap on Linux and unsupported on darwin, which this project //! develops on. Everything else here goes through std.Io.File. const std = @import("std"); const builtin = @import("builtin"); const assert = @import("assert.zig").assert; const assert_msg = @import("assert.zig").assert_msg; /// Test-only: record which pages have been written since the last sync, so a /// test can simulate the one failure a `kill -9` cannot produce -- writeback /// that never happened. /// /// This exists because the ordering it guards is the whole crash-recovery /// argument and is otherwise untestable. Killing a process does not lose /// page-cache writes, so removing the msync before the watermark leaves every /// test green while making a real power loss unrecoverable. Compiled out /// entirely outside tests, so `page_mut` keeps no branch in a real build. const track_dirty = builtin.is_test; /// Also make the published image *hardware* read-only, in builds that can /// afford it. /// /// The assert in `page_mut` catches a caller that asks for a writable page below /// the stable mark. It cannot catch one that holds a pointer obtained *before* /// the mark moved and writes through it afterwards -- and that is the realistic /// mistake, because tree code holds `*Node` across calls. mprotect catches it, /// as an immediate segfault at the offending store rather than as a corrupt /// database discovered after a power loss. /// /// Off in ReleaseFast so the hot path keeps no extra syscall, on everywhere /// else. One mprotect per checkpoint over one range is the whole cost. const protect_stable = builtin.mode != .ReleaseFast; pub const page_shift: u6 = 12; pub const page_size: usize = 1 << page_shift; /// Page numbers of the fixed-position pages. pub const page_header: u32 = 0; pub const page_watermark_a: u32 = 1; pub const page_watermark_b: u32 = 2; /// The first page the allocator may hand out. pub const page_first_data: u32 = 3; const magic: u32 = 0x4D464442; // "MFDB" const format_version: u8 = 1; const header_hashed_len: usize = 24; /// The alignment mmap requires, which is the *system* page size and not ours. /// 16 KiB on Apple Silicon against a 4 KiB logical page, so four logical pages /// share one system page there — which is why the checkpoint rounds its append /// cursors up to this rather than to page_size. pub const map_align = std.heap.page_size_min; /// Growth granularity. Large enough that growth is rare and each `setLength` /// covers many allocations, and a multiple of every supported system page size. const grow_chunk_pages: u32 = 2048; // 8 MiB /// Address space reserved by default. Reserved, not committed: a PROT_NONE /// anonymous NORESERVE mapping is one VMA and zero pages. Sized for the /// tens-of-GB target (D3) with room to spare. pub const default_reserve_bytes: usize = 64 << 30; comptime { assert(page_size == 4096); assert(page_first_data == 3); assert(grow_chunk_pages * page_size % map_align == 0); assert(map_align % page_size == 0 or page_size % map_align == 0); // Node pages are raw host memory in this file (see the header comment). assert(@import("builtin").cpu.arch.endian() == .little); } /// A run of contiguous pages. pub const Extent = struct { first: u32, pages: u32, }; /// What a checkpoint publishes. Everything here is authoritative except the /// two cached counters, which are hints the engine recomputes if they look /// wrong. pub const Watermark = struct { generation: u64 = 0, /// The log sequence this image covers. The crash-recovery invariant is that /// this never exceeds the durable log tail (PLAN D6), so replay after open /// is exactly the records above it. seq: u64 = 0, alloc_tail: u32 = page_first_data, file_pages: u32 = page_first_data, catalog_page: u32 = 0, catalog_len: u64 = 0, freelist_page: u32 = 0, freelist_len: u64 = 0, prev_generation: u64 = 0, live_docs: u64 = 0, dead_bytes: u64 = 0, }; const wm_hashed_len: usize = page_size - 8; pub const Error = error{ /// Not a data file, or a version this build cannot read. InvalidDataFile, /// The file belongs to a different database than the log beside it. DatabaseMismatch, /// Growth would exceed the reserved address space. DatabaseTooLarge, }; pub const OpenOptions = struct { /// Must match the log's uuid. Zero means "adopt whatever the file has", /// which is only for tests that do not care. uuid: u128 = 0, reserve_bytes: usize = default_reserve_bytes, }; pub const Pager = struct { gpa: std.mem.Allocator, io: std.Io, file: std.Io.File, path: []u8, uuid: u128, /// The address-space reservation. Never moves for the life of the process, /// so a pointer obtained from `page`/`page_mut` stays valid across growth — /// which the old ArrayList-backed arena could not promise, and which is the /// reason a whole class of dangling-pointer bugs disappears here. reserve: []align(map_align) u8, /// Pages actually mapped to the file. Always >= alloc_tail. mapped_pages: u32, /// Pages the file is long enough to hold. file_pages: u32, /// Pages [0, alloc_tail) have been handed out. alloc_tail: u32, /// Pages promised by `reserve_pages` and not yet handed out. /// /// A count rather than a tail mark, because there is more than one consumer: /// an upsert reserves tree pages for every index *and* slab room for the /// document, all before the log append. A tail mark cannot express that -- /// the second reserver overwrites the first one's promise, and then the /// first one's allocation asserts. Which is exactly what happened, on a /// 512 MB load, with the tripwire in alloc_node catching it. reserved_pages: u32, /// True when this file was created by this open (no checkpoint to load). fresh: bool, /// The watermark this pager opened from, or a zero one when fresh. The /// engine reads `seq` to decide where replay starts. loaded: Watermark, /// The generation the next checkpoint will publish. generation: u64, /// Pages below this belong to the last published image and must never be /// stored into (PLAN amendment A1). Zero until a checkpoint publishes one, /// which is why copy-on-write is inert before then. stable_pages: u32, /// Free pages, in three stages. `ready` may be handed out now; `hold` was /// freed one generation ago; `pending` was freed during this generation. /// /// The two-generation delay is not caution, it is what makes generation N-1 /// remain a usable image: its pages are still allocated while N is current, /// so a torn watermark or a bad catalog can fall back to it instead of /// discarding the database. Releasing one generation early would silently /// take that away. free_ready: std.ArrayListUnmanaged(Extent), free_hold: std.ArrayListUnmanaged(Extent), free_pending: std.ArrayListUnmanaged(Extent), /// See `track_dirty`. Pages written since the last sync. dirty: if (track_dirty) std.AutoHashMapUnmanaged(u32, void) else void, /// Whether the last `protect_image` actually took effect. Checked by a test: /// an mprotect that silently fails would leave the belt looking present and /// doing nothing, which is worse than not having it. protect_ok: bool, pub fn open( gpa: std.mem.Allocator, io: std.Io, path: []const u8, opts: OpenOptions, ) !Pager { // Absolute, for the same reason storage.Log resolves its own path: the // rebuild renames this file and must not depend on the caller's working // directory. const owned_path = blk: { if (path.len > 0 and path[0] == '/') break :blk try gpa.dupe(u8, path); const cwd = try std.process.currentPathAlloc(io, gpa); defer gpa.free(cwd); break :blk try std.fmt.allocPrint(gpa, "{s}/{s}", .{ cwd, path }); }; errdefer gpa.free(owned_path); const dir = std.Io.Dir.cwd(); var created = false; var file = dir.openFile(io, owned_path, .{ .mode = .read_write }) catch |err| switch (err) { error.FileNotFound => blk: { created = true; break :blk try dir.createFile(io, owned_path, .{ .read = true }); }, else => return err, }; errdefer file.close(io); // A zero-length file is as good as absent: a create that died before // its header landed must not look like a corrupt database. const existing_len = try file.length(io); if (existing_len == 0) created = true; const reserve = try reserve_address_space(opts.reserve_bytes); errdefer std.posix.munmap(reserve); var self: Pager = .{ .gpa = gpa, .io = io, .file = file, .path = owned_path, .uuid = opts.uuid, .reserve = reserve, .mapped_pages = 0, .file_pages = @intCast(existing_len / page_size), .alloc_tail = page_first_data, .reserved_pages = 0, .fresh = created, .loaded = .{}, .generation = 0, .stable_pages = 0, .free_ready = .empty, .free_hold = .empty, .free_pending = .empty, .dirty = if (track_dirty) .empty else {}, .protect_ok = false, }; if (created) { try self.grow_to(page_first_data); try self.write_header(); } else { try self.grow_to(@max(page_first_data, self.file_pages)); try self.read_header(); // Everything already in the file is allocated until a watermark // narrows it down. self.alloc_tail = @max(page_first_data, self.file_pages); try self.load_watermark(); } return self; } pub fn deinit(self: *Pager) void { self.free_ready.deinit(self.gpa); self.free_hold.deinit(self.gpa); self.free_pending.deinit(self.gpa); if (track_dirty) self.dirty.deinit(self.gpa); std.posix.munmap(self.reserve); self.file.close(self.io); self.gpa.free(self.path); } // -- page access -------------------------------------------------------- /// The page's bytes, for reading. pub inline fn page(self: *const Pager, p: u32) *align(page_size) const [page_size]u8 { assert_msg(p < self.mapped_pages, "read of a page past the mapped end of the data file"); return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } /// The page's bytes, for writing. Callers holding a page number that can be /// *updated* use `page_mut_cow`; append-only consumers pass a page they /// allocated themselves. /// /// There is deliberately no `p >= stable_pages` assert here, and the reason /// is worth recording because it looks like an obvious check to add. A page /// recycled off the free list *is* below the mark and *is* legitimately /// writable -- it was freed two generations ago and no live image references /// it any more -- so the page number alone cannot distinguish a violation /// from a reuse, and a per-write set membership test would cost more than it /// is worth on the hot path. /// /// The invariant is enforced the two ways PLAN amendment A1 describes /// instead: structurally, because every write to a tree node goes through /// `page_mut_cow`, and mechanically, because `protect_stable` builds make the /// image hardware read-only, which `unprotect` lifts for exactly the pages /// recycling hands back. pub inline fn page_mut(self: *Pager, p: u32) *align(page_size) [page_size]u8 { assert_msg(p < self.mapped_pages, "write to a page past the mapped end of the data file"); if (track_dirty) self.dirty.put(self.gpa, p, {}) catch {}; return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } /// A writable page for an owner that can be told where the page moved. /// /// `slot` must be the *single* owner of this page number. That is the /// property the whole scheme rests on: copy-on-write relocates the page, and /// if a second reference existed it would still point at the abandoned copy. /// For the B+tree that owner is the id->page table entry, which is precisely /// why node ids are not page numbers (PLAN amendment A1). /// /// The victim goes on the free list, which withholds it for two generations, /// so the image that still references it stays intact and usable. pub fn page_mut_cow(self: *Pager, slot: *u32) !*align(page_size) [page_size]u8 { if (slot.* >= self.stable_pages) return self.page_mut(slot.*); const fresh = try self.alloc_pages(1); @memcpy( @as(*[page_size]u8, @ptrCast(self.page_mut(fresh))), @as(*const [page_size]u8, @ptrCast(self.page(slot.*))), ); try self.free_pages(slot.*, 1); slot.* = fresh; return self.page_mut(fresh); } /// The one page a write may legitimately land on below the stable mark: a /// watermark slot. Overwriting the *inactive* slot is the whole mechanism -- /// alternating by generation parity is what makes it safe, where every other /// page below the mark is part of the image a crash must find unchanged. /// /// Deliberately not public and deliberately narrow: it takes the slot number /// and asserts it is one, so it cannot become a general escape hatch from /// the check in `page_mut`. inline fn page_mut_slot(self: *Pager, p: u32) *align(page_size) [page_size]u8 { assert(p == page_watermark_a or p == page_watermark_b); assert_msg(p < self.mapped_pages, "write to a watermark slot past the mapped end"); if (track_dirty) self.dirty.put(self.gpa, p, {}) catch {}; return @ptrCast(@alignCast(self.reserve.ptr + (@as(usize, p) << page_shift))); } /// Bytes above the stable mark, i.e. the first byte a write may touch. pub inline fn stable_bytes(self: *const Pager) u64 { return @as(u64, self.stable_pages) << page_shift; } /// Bytes at an absolute file offset, for the consumers whose references are /// byte offsets rather than page numbers: the document slab and the /// overflow slab. pub inline fn bytes(self: *const Pager, off: u64, len: usize) []const u8 { assert_msg(off + len <= @as(u64, self.mapped_pages) << page_shift, "read past the mapped end of the data file"); return self.reserve[@intCast(off)..][0..len]; } pub inline fn bytes_mut(self: *Pager, off: u64, len: usize) []u8 { assert_msg(off + len <= @as(u64, self.mapped_pages) << page_shift, "write past the mapped end of the data file"); if (track_dirty) { var pg_i: u32 = @intCast(off >> page_shift); const last: u32 = @intCast((off + len - 1) >> page_shift); while (pg_i <= last) : (pg_i += 1) self.dirty.put(self.gpa, pg_i, {}) catch {}; } return self.reserve[@intCast(off)..][0..len]; } // -- allocation --------------------------------------------------------- /// Hand out `n` contiguous pages, growing the file if needed. pub fn alloc_pages(self: *Pager, n: u32) !u32 { assert(n > 0); try self.reserve_pages(n); return self.alloc_pages_assume_reserved(n); } /// Guarantee that the next `n` pages can be handed out without failing. /// Fallible, and meant to run *before* the log append on a write path, so /// that publishing afterwards cannot fail — the invariant that keeps a /// document from ever being live but unindexed. /// /// Nothing is dirtied here, so nothing is allocated on disk: `setLength` /// leaves a sparse file, `ls -l` grows and `du` does not. That is what makes /// a generous reservation cheap. pub fn reserve_pages(self: *Pager, n: u32) !void { // Additive: room for what is already promised *plus* this. Two // consumers reserving before the same log append must both be able to // rely on their promise. try self.grow_to(self.alloc_tail + self.reserved_pages + n); self.reserved_pages += n; } /// Drop whatever is still promised but unclaimed. /// /// A reservation is scoped to one write: it is taken before the log append /// so the publish afterwards cannot fail, and once the publish is done /// anything unclaimed is dead. Without this the promise accumulates -- a /// tree reservation covers the worst case of several splits and a typical /// insert causes none, so `reserved_pages` grew by a handful per write and /// dragged the file up with it. It showed as a 1.89 GB data file for 512 MB /// of documents. pub fn release_reservation(self: *Pager) void { self.reserved_pages = 0; } /// Hand out `n` pages against a previous `reserve_pages`. Infallible. /// Take a run from the free list if one fits, else bump the tail. A recycled /// page was inside a published image once, so its protection has to be /// lifted before it is handed out again. fn take_free(self: *Pager, n: u32) ?u32 { for (self.free_ready.items, 0..) |e, i| { if (e.pages < n) continue; const first = e.first; if (e.pages == n) { _ = self.free_ready.swapRemove(i); } else { self.free_ready.items[i] = .{ .first = e.first + n, .pages = e.pages - n }; } self.unprotect(first, n); return first; } return null; } pub fn alloc_pages_assume_reserved(self: *Pager, n: u32) u32 { assert(n > 0); assert_msg( n <= self.reserved_pages, "page allocation overran reserve_pages' promise", ); assert_msg( self.alloc_tail + n <= self.mapped_pages, "page allocation past the mapped end of the data file", ); self.reserved_pages -= n; // Reuse before growing. Without this the free list is decorative and the // file grows without bound under churn, because copy-on-write abandons // every page it touches in every generation (PLAN amendment A2). if (self.take_free(n)) |recycled| return recycled; const first = self.alloc_tail; self.alloc_tail += n; return first; } /// Bytes currently allocated, i.e. the extent of the live image. pub fn allocated_bytes(self: *const Pager) u64 { return @as(u64, self.alloc_tail) << page_shift; } // -- durability --------------------------------------------------------- /// Flush every allocated page and the inode. Only the checkpoint calls /// this: between checkpoints dirty pages may sit in the page cache /// indefinitely, because recovery is `image + replay(seq > watermark)` and /// the image's pages are never written *differently*. That is what keeps /// the write path at exactly one fsync — the WAL's (PLAN amendment A1). pub fn sync(self: *Pager) !void { const len = std.mem.alignForward(usize, @intCast(self.allocated_bytes()), map_align); if (len > 0) { try std.posix.msync(self.reserve[0..len], std.posix.MSF.SYNC); } try self.file.sync(self.io); if (track_dirty) self.dirty.clearRetainingCapacity(); } /// Make the published image read-only at the hardware level. See /// `protect_stable`. The watermark slots stay writable: overwriting the /// inactive one is the publication mechanism, not a violation. fn protect_image(self: *Pager) void { if (!protect_stable) return; const first = std.mem.alignForward(usize, @as(usize, page_first_data) << page_shift, map_align); const end = std.mem.alignBackward(usize, @as(usize, self.stable_pages) << page_shift, map_align); if (end <= first) return; // Best effort as far as the database is concerned -- losing the check // leaves a weaker test build, not wrong data -- but recorded, so a test // can tell the difference between a belt that is working and one that is // silently doing nothing. std.posix has no mprotect wrapper in Zig 0.16, // hence the libc call. const rc = std.c.mprotect(@ptrCast(@alignCast(self.reserve.ptr + first)), end - first, .{ .READ = true }); self.protect_ok = rc == 0; } /// Lift the protection over a range that is about to leave the image -- /// which only copy-on-write does, when it recycles a freed page. fn unprotect(self: *Pager, first_page: u32, pages: u32) void { if (!protect_stable) return; const first = std.mem.alignBackward(usize, @as(usize, first_page) << page_shift, map_align); const end = std.mem.alignForward(usize, @as(usize, first_page + pages) << page_shift, map_align); _ = std.c.mprotect(@ptrCast(@alignCast(self.reserve.ptr + first)), end - first, .{ .READ = true, .WRITE = true }); } /// Test-only: overwrite, in the *file*, every page written since the last /// sync -- the state a power loss can leave behind and a `kill -9` cannot. /// Call it with the mapping already closed. fn simulate_lost_writeback(gpa: std.mem.Allocator, io: std.Io, path: []const u8, lost: []const u32) !void { var f = try std.Io.Dir.cwd().openFile(io, path, .{ .mode = .read_write }); defer f.close(io); const junk = try gpa.alloc(u8, page_size); defer gpa.free(junk); // Not zeroes: zeroes are a plausible value, and a test that passes // because the lost page happened to read as zero proves nothing. @memset(junk, 0xDD); for (lost) |p| try f.writePositionalAll(io, junk, @as(u64, p) << page_shift); } /// Test-only: the pages `simulate_lost_writeback` would discard. fn dirty_pages(self: *const Pager, gpa: std.mem.Allocator) ![]u32 { var out: std.ArrayListUnmanaged(u32) = .empty; errdefer out.deinit(gpa); var it = self.dirty.keyIterator(); while (it.next()) |k| try out.append(gpa, k.*); return out.toOwnedSlice(gpa); } // -- growth ------------------------------------------------------------- /// Make at least `want_pages` pages mapped and file-backed. /// /// The order is the whole point: extend the file, *then* map the new /// suffix. A store into a mapped page past end-of-file raises SIGBUS, which /// no Zig error path can catch, so the file must be long enough before any /// page in the range is reachable. The accessors assert against /// `mapped_pages` so a violation is a panic with a message rather than a /// signal. fn grow_to(self: *Pager, want_pages: u32) !void { if (want_pages <= self.mapped_pages) return; // Round up to a growth chunk, and to the system page size, so a // 16 KiB-page host never gets a partial mapping request. const chunk = @max(grow_chunk_pages, self.mapped_pages / 8); var new_pages = std.mem.alignForward(u32, want_pages, chunk); const sys_pages: u32 = @intCast(map_align / page_size); if (sys_pages > 1) new_pages = std.mem.alignForward(u32, new_pages, sys_pages); if (@as(u64, new_pages) << page_shift > self.reserve.len) { // One more try at exactly what was asked for: a reservation the // chunking would overshoot is still usable up to its end. new_pages = want_pages; if (sys_pages > 1) new_pages = std.mem.alignForward(u32, new_pages, sys_pages); if (@as(u64, new_pages) << page_shift > self.reserve.len) return Error.DatabaseTooLarge; } const new_len: u64 = @as(u64, new_pages) << page_shift; if (new_len > self.file_pages_bytes()) { try self.file.setLength(self.io, new_len); self.file_pages = new_pages; } const off: usize = @as(usize, self.mapped_pages) << page_shift; const len: usize = @intCast(new_len - (@as(u64, self.mapped_pages) << page_shift)); _ = try std.posix.mmap( @alignCast(self.reserve.ptr + off), len, .{ .READ = true, .WRITE = true }, .{ .TYPE = .SHARED, .FIXED = true }, self.file.handle, off, ); self.mapped_pages = new_pages; } fn file_pages_bytes(self: *const Pager) u64 { return @as(u64, self.file_pages) << page_shift; } // -- watermark ---------------------------------------------------------- /// Adopt the newest valid watermark, or none. Never fails on a damaged /// watermark: a data file whose slots are both unreadable is treated as /// having no checkpoint, which means a full replay -- the database must /// always open (ground rule 4). The caller learns which happened from /// `loaded.generation == 0`. fn load_watermark(self: *Pager) !void { const a = self.read_slot(page_watermark_a); const b = self.read_slot(page_watermark_b); // Validate first, compare generations second. The other order picks a // torn slot whenever its garbage generation happens to be larger. const pick: ?Watermark = if (a != null and b != null) (if (a.?.generation >= b.?.generation) a else b) else if (a != null) a else b; if (pick) |wm| { self.loaded = wm; self.generation = wm.generation; self.alloc_tail = wm.alloc_tail; self.reserved_pages = 0; // Everything the published image references is off limits to // writes from here on. self.stable_pages = wm.alloc_tail; try self.read_freelist(wm); } else if (self.watermark_attempted()) { // Only worth saying when a watermark was *written* and cannot be // read: a data file that simply never reached its first checkpoint // is the normal state of a young database, and warning about it // trains people to ignore the warning that matters. std.debug.print( "multiforadb: WARNING: data file '{s}' has a damaged watermark; " ++ "treating it as having no checkpoint and replaying the log in full\n", .{self.path}, ); } } /// Whether either slot holds anything at all. A never-checkpointed file has /// both slots as written by create: all zeroes. fn watermark_attempted(self: *const Pager) bool { for ([_]u32{ page_watermark_a, page_watermark_b }) |p| { if (p >= self.mapped_pages) continue; for (self.page(p)) |byte| if (byte != 0) return true; } return false; } fn read_slot(self: *const Pager, p: u32) ?Watermark { if (p >= self.mapped_pages) return null; const b = self.page(p); const want = std.mem.readInt(u64, b[wm_hashed_len..][0..8], .little); if (header_hash(b[0..wm_hashed_len]) != want) return null; const wm: Watermark = .{ .generation = std.mem.readInt(u64, b[0..8], .little), .seq = std.mem.readInt(u64, b[8..16], .little), .alloc_tail = @intCast(std.mem.readInt(u64, b[16..24], .little)), .file_pages = @intCast(std.mem.readInt(u64, b[24..32], .little)), .catalog_page = @intCast(std.mem.readInt(u64, b[32..40], .little)), .catalog_len = std.mem.readInt(u64, b[40..48], .little), .freelist_page = @intCast(std.mem.readInt(u64, b[48..56], .little)), .freelist_len = std.mem.readInt(u64, b[56..64], .little), .prev_generation = std.mem.readInt(u64, b[64..72], .little), .live_docs = std.mem.readInt(u64, b[72..80], .little), .dead_bytes = std.mem.readInt(u64, b[80..88], .little), }; // A slot whose hash matches but whose contents are impossible is worse // than a torn one, because it would be trusted. // // The generation check is belt-and-braces: an all-zero slot is already // rejected by the hash, so no test reddens when it is removed. Kept // because it costs nothing and it is the field every comparison below // depends on. The alloc_tail check is *not* redundant -- a slot can be // rehashed to validate with nonsense in it, which is what the // "hash-valid but impossible" test does. if (wm.generation == 0) return null; if (wm.alloc_tail < page_first_data) return null; return wm; } /// Publish `wm` as the new durable image. /// /// The ordering here is the entire crash-recovery argument, so it is worth /// stating plainly: every page the watermark describes is made durable /// *before* the watermark that describes them. Reverse the two and a crash /// in between leaves a watermark pointing at pages that were never written, /// with the log already truncated below it -- unrecoverable. /// /// The slot alternates by generation parity, so the previous watermark /// survives this write and a torn slot leaves the older one intact. pub fn publish(self: *Pager, wm_in: Watermark) !void { var wm = wm_in; wm.generation = self.generation + 1; wm.prev_generation = self.generation; // 1. every data page durable. try self.sync(); // 2. the free list, into fresh pages so the old one stays intact. const fl = try self.write_freelist(); wm.freelist_page = fl.first; wm.freelist_len = fl.len; try self.sync(); // 3. only now record the extent, *after* the free list has taken its // pages. Capturing alloc_tail before this left those pages outside // the published image, so the next open would hand them out again // while the watermark still pointed at them. wm.alloc_tail = self.alloc_tail; wm.file_pages = self.file_pages; // 4. the watermark itself, last. const slot = if (wm.generation & 1 == 1) page_watermark_a else page_watermark_b; const b = self.page_mut_slot(slot); @memset(b, 0); std.mem.writeInt(u64, b[0..8], wm.generation, .little); std.mem.writeInt(u64, b[8..16], wm.seq, .little); std.mem.writeInt(u64, b[16..24], wm.alloc_tail, .little); std.mem.writeInt(u64, b[24..32], wm.file_pages, .little); std.mem.writeInt(u64, b[32..40], wm.catalog_page, .little); std.mem.writeInt(u64, b[40..48], wm.catalog_len, .little); std.mem.writeInt(u64, b[48..56], wm.freelist_page, .little); std.mem.writeInt(u64, b[56..64], wm.freelist_len, .little); std.mem.writeInt(u64, b[64..72], wm.prev_generation, .little); std.mem.writeInt(u64, b[72..80], wm.live_docs, .little); std.mem.writeInt(u64, b[80..88], wm.dead_bytes, .little); std.mem.writeInt(u64, b[wm_hashed_len..][0..8], header_hash(b[0..wm_hashed_len]), .little); const off = @as(usize, slot) << page_shift; const end = std.mem.alignForward(usize, off + page_size, map_align); const start = std.mem.alignBackward(usize, off, map_align); try std.posix.msync(@alignCast(self.reserve[start..end]), std.posix.MSF.SYNC); try self.file.sync(self.io); // Deliberately does *not* clear the dirty record. After a correct // publish the only page left in it is this slot, which this msync just // made durable -- so a test can discard everything else and assert the // image still loads. Clearing it here would make that check vacuous. // 5. only now is the new image current: advance the stable mark and // rotate the free lists by one generation. self.generation = wm.generation; self.loaded = wm; self.stable_pages = self.alloc_tail; self.protect_image(); try self.free_ready.appendSlice(self.gpa, self.free_hold.items); self.free_hold.clearRetainingCapacity(); try self.free_hold.appendSlice(self.gpa, self.free_pending.items); self.free_pending.clearRetainingCapacity(); } // -- free list ---------------------------------------------------------- /// Give back a run of pages. They become reusable two generations later -- /// see the field comment on `free_ready`. pub fn free_pages(self: *Pager, first: u32, pages: u32) !void { if (pages == 0) return; try self.free_pending.append(self.gpa, .{ .first = first, .pages = pages }); } /// Pages available for immediate reuse. pub fn free_ready_pages(self: *const Pager) u32 { var n: u32 = 0; for (self.free_ready.items) |e| n += e.pages; return n; } fn write_freelist(self: *Pager) !struct { first: u32, len: u64 } { const count = self.free_ready.items.len + self.free_hold.items.len + self.free_pending.items.len; const len: u64 = 8 + @as(u64, count) * 8 + 8; const pages: u32 = @intCast((len + page_size - 1) / page_size); const first = try self.alloc_pages(pages); const buf = self.bytes_mut(@as(u64, first) << page_shift, @intCast(len)); @memset(buf, 0); std.mem.writeInt(u64, buf[0..8], count, .little); var at: usize = 8; for ([_][]const Extent{ self.free_ready.items, self.free_hold.items, self.free_pending.items, }) |list| { for (list) |e| { std.mem.writeInt(u32, buf[at..][0..4], e.first, .little); std.mem.writeInt(u32, buf[at + 4 ..][0..4], e.pages, .little); at += 8; } } std.mem.writeInt(u64, buf[at..][0..8], header_hash(buf[0..at]), .little); return .{ .first = first, .len = len }; } /// Load the free list a watermark points at. A damaged one is dropped with a /// warning rather than refused: losing it wastes space, refusing to open /// loses the database. fn read_freelist(self: *Pager, wm: Watermark) !void { if (wm.freelist_len == 0 or wm.freelist_page == 0) return; if (wm.freelist_page >= self.mapped_pages) return; const buf = self.bytes(@as(u64, wm.freelist_page) << page_shift, @intCast(wm.freelist_len)); const count = std.mem.readInt(u64, buf[0..8], .little); const want_len = 8 + count * 8 + 8; if (want_len != wm.freelist_len) return; const at: usize = @intCast(8 + count * 8); if (header_hash(buf[0..at]) != std.mem.readInt(u64, buf[at..][0..8], .little)) { std.debug.print("multiforadb: WARNING: data file free list is corrupt; its pages stay in use\n", .{}); return; } // Everything the previous generations freed is reusable now: this open // *is* a new generation, and nothing from before it is referenced. var i: usize = 0; while (i < count) : (i += 1) { const o = 8 + i * 8; try self.free_ready.append(self.gpa, .{ .first = std.mem.readInt(u32, buf[o..][0..4], .little), .pages = std.mem.readInt(u32, buf[o + 4 ..][0..4], .little), }); } } // -- header ------------------------------------------------------------- fn write_header(self: *Pager) !void { const h = self.page_mut(page_header); @memset(h, 0); std.mem.writeInt(u32, h[0..4], magic, .little); h[4] = format_version; h[5] = page_shift; std.mem.writeInt(u128, h[8..24], self.uuid, .little); std.mem.writeInt(u64, h[24..32], header_hash(h[0..header_hashed_len]), .little); } fn read_header(self: *Pager) !void { const h = self.page(page_header); if (std.mem.readInt(u32, h[0..4], .little) != magic) return Error.InvalidDataFile; if (h[4] != format_version) return Error.InvalidDataFile; if (h[5] != page_shift) return Error.InvalidDataFile; const want = std.mem.readInt(u64, h[24..32], .little); if (header_hash(h[0..header_hashed_len]) != want) return Error.InvalidDataFile; const file_uuid = std.mem.readInt(u128, h[8..24], .little); if (self.uuid == 0) { self.uuid = file_uuid; } else if (file_uuid != self.uuid) { return Error.DatabaseMismatch; } } }; fn header_hash(b: []const u8) u64 { return std.hash.XxHash3.hash(0, b); } /// Reserve address space without committing memory: PROT_NONE, anonymous and /// NORESERVE is one VMA and zero pages, and is not charged even under strict /// overcommit. Nothing is mapped to the file yet, so nothing here can fault. /// /// Halves on refusal rather than failing outright, so a container with a tight /// address-space limit still opens — with a smaller ceiling, which `grow_to` /// reports as DatabaseTooLarge if it is ever actually reached. fn reserve_address_space(want: usize) ![]align(map_align) u8 { var len = std.mem.alignForward(usize, want, map_align); while (true) { if (std.posix.mmap( null, len, .{}, // PROT_NONE .{ .TYPE = .PRIVATE, .ANONYMOUS = true, .NORESERVE = true }, -1, 0, )) |m| { return m; } else |err| { if (len <= 1 << 30) return err; len /= 2; } } } // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- const testing = std.testing; /// A pager over a temp file, with a reservation small enough that the growth /// ceiling is reachable in a test. const TmpPager = struct { tmp: std.testing.TmpDir, path: []u8, /// Optional so a test can close it early -- several deliberately break the /// file and must not have to reopen one just to satisfy teardown. pager: ?Pager, fn init(io: std.Io, reserve_bytes: usize) !TmpPager { const tmp = std.testing.tmpDir(.{}); const path = try std.fmt.allocPrint( testing.allocator, ".zig-cache/tmp/{s}/data", .{tmp.sub_path}, ); return .{ .tmp = tmp, .path = path, .pager = try Pager.open(testing.allocator, io, path, .{ .uuid = 7, .reserve_bytes = reserve_bytes }), }; } fn deinit(self: *TmpPager) void { if (self.pager) |*p| p.deinit(); self.pager = null; self.tmp.cleanup(); testing.allocator.free(self.path); } fn close(self: *TmpPager) void { if (self.pager) |*p| p.deinit(); self.pager = null; } fn pg(self: *TmpPager) *Pager { return &self.pager.?; } }; test "pages survive growth and the mapping base never moves" { // The reason the fixed reservation exists: a pointer handed out before a // growth must still be valid after it. The ArrayList-backed arena this // replaces could not promise that, and the workarounds for it (copying // promoted keys into a scratch buffer) exist only because of it. // // Mutation check: make grow_to munmap and re-map the whole prefix at a // kernel-chosen address and the base assertion goes red. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 256 << 20); defer tp.deinit(); const pg = tp.pg(); const base = @intFromPtr(pg.reserve.ptr); const first = try pg.alloc_pages(1); const first_ptr = pg.page_mut(first); @memset(first_ptr, 0xAB); // Grow well past several chunk boundaries. var written: std.ArrayListUnmanaged(u32) = .empty; defer written.deinit(testing.allocator); for (0..6000) |i| { const p = try pg.alloc_pages(1); const bytes = pg.page_mut(p); @memset(bytes, @truncate(i)); try written.append(testing.allocator, p); } try testing.expectEqual(base, @intFromPtr(pg.reserve.ptr)); // The pointer taken before all that growth still addresses its page. for (first_ptr) |b| try testing.expectEqual(@as(u8, 0xAB), b); for (written.items, 0..) |p, i| { const want: u8 = @truncate(i); try testing.expectEqual(want, pg.page(p)[0]); try testing.expectEqual(want, pg.page(p)[page_size - 1]); } } test "the file is extended before any page in the range is reachable" { // Write-then-extend, asserted as an ordering rather than by provoking the // failure: a store past end-of-file is SIGBUS, which cannot be caught // in-process, so the check is that the file is always long enough for // everything mapped, and everything allocated is always mapped. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 256 << 20); defer tp.deinit(); const pg = tp.pg(); for (0..40) |_| { _ = try pg.alloc_pages(97); // not a chunk divisor, so growth is ragged try testing.expect(pg.alloc_tail <= pg.mapped_pages); try testing.expect(pg.mapped_pages <= pg.file_pages); const on_disk = try pg.file.length(io); try testing.expect(on_disk >= @as(u64, pg.mapped_pages) << page_shift); } } test "two consumers reserving before one commit both keep their promise" { // The reservation is a count, not a tail mark, and this is why. An upsert // reserves tree pages for every index *and* slab room for the document, // all before the log append, and both allocations happen after it. With a // tail mark the second reserver overwrote the first one's promise and the // first one's allocation then asserted -- which is how this was found, on a // 512 MB load, by the tripwire in alloc_node. // // Mutation check: make reserve_pages assign `alloc_tail + n` instead of // accumulating, and this goes red. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 256 << 20); defer tp.deinit(); const pg = tp.pg(); // Consumer A reserves a few pages, then consumer B reserves a large extent // and takes it -- exactly the order upsert uses. try pg.reserve_pages(8); try pg.reserve_pages(2048); const b_first = pg.alloc_pages_assume_reserved(2048); // A's promise must have survived B's reservation *and* B's allocation. const a_first = pg.alloc_pages_assume_reserved(8); try testing.expect(a_first >= b_first + 2048); try testing.expectEqual(@as(u32, 0), pg.reserved_pages); } test "a reservation makes the following allocation infallible" { var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 256 << 20); defer tp.deinit(); const pg = tp.pg(); try pg.reserve_pages(64); const before = pg.alloc_tail; // Exactly the promised amount, one page at a time. for (0..64) |_| _ = pg.alloc_pages_assume_reserved(1); try testing.expectEqual(before + 64, pg.alloc_tail); } test "growth past the reservation is an error, not a crash" { // A bounded reservation must fail cleanly at its ceiling: this is the path // a container with a tight address-space limit takes. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 1 << 30); defer tp.deinit(); const pg = tp.pg(); const capacity: u32 = @intCast(pg.reserve.len / page_size); try testing.expectError(Error.DatabaseTooLarge, pg.alloc_pages(capacity + 1)); // And the pager is still usable afterwards. const p = try pg.alloc_pages(1); @memset(pg.page_mut(p), 1); } test "header round-trips and rejects a foreign database" { var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); try testing.expect(tp.pg().fresh); const p = try tp.pg().alloc_pages(3); @memset(tp.pg().page_mut(p), 0x5A); try tp.pg().sync(); const path = try testing.allocator.dupe(u8, tp.path); defer testing.allocator.free(path); tp.close(); // Reopening finds the same header and is not fresh. var again = try Pager.open(testing.allocator, io, path, .{ .uuid = 7, .reserve_bytes = 64 << 20 }); try testing.expect(!again.fresh); try testing.expectEqual(@as(u128, 7), again.uuid); try testing.expectEqual(@as(u8, 0x5A), again.page(p)[0]); again.deinit(); // A different uuid is a different database; pairing them would silently // replay one database's log onto another's checkpoint. // Mutation check: drop the uuid comparison in read_header and this passes. try testing.expectError( Error.DatabaseMismatch, Pager.open(testing.allocator, io, path, .{ .uuid = 8, .reserve_bytes = 64 << 20 }), ); } test "a corrupt header is rejected rather than misread" { var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const path = try testing.allocator.dupe(u8, tp.path); defer testing.allocator.free(path); try tp.pg().sync(); tp.close(); // Flip a byte inside the hashed region. { var f = try std.Io.Dir.cwd().openFile(io, path, .{ .mode = .read_write }); defer f.close(io); var b: [1]u8 = undefined; _ = try f.readPositionalAll(io, &b, 6); b[0] ^= 0xFF; try f.writePositionalAll(io, &b, 6); } // Mutation check: drop the hash comparison in read_header and this passes, // which is worse than failing — a header believed on faith describes where // every structure in the file lives. try testing.expectError( Error.InvalidDataFile, Pager.open(testing.allocator, io, path, .{ .uuid = 7, .reserve_bytes = 64 << 20 }), ); } /// Reopen the same file, so a test can check what survived a publish. fn reopen(io: std.Io, path: []const u8) !Pager { return Pager.open(testing.allocator, io, path, .{ .uuid = 7, .reserve_bytes = 64 << 20 }); } test "a published watermark is what the next open loads" { var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const p = try tp.pg().alloc_pages(4); @memset(tp.pg().page_mut(p), 0xC3); try tp.pg().publish(.{ .seq = 4242, .catalog_page = p, .catalog_len = 99, .live_docs = 7 }); // After publish, not before: publish allocates the free list's own pages, // and the watermark has to cover them. const tail_at_publish = tp.pg().alloc_tail; tp.close(); var again = try reopen(io, tp.path); defer again.deinit(); try testing.expect(!again.fresh); try testing.expectEqual(@as(u64, 1), again.loaded.generation); try testing.expectEqual(@as(u64, 4242), again.loaded.seq); try testing.expectEqual(p, again.loaded.catalog_page); try testing.expectEqual(@as(u64, 99), again.loaded.catalog_len); try testing.expectEqual(@as(u64, 7), again.loaded.live_docs); try testing.expectEqual(tail_at_publish, again.alloc_tail); // Everything the image references is off limits to writes. try testing.expectEqual(tail_at_publish, again.stable_pages); try testing.expectEqual(@as(u8, 0xC3), again.page(p)[0]); } test "slots alternate by generation, so a torn write leaves the older one" { // Writing each new generation over the only copy of the previous one risks // having *neither* valid after a crash. Alternating is what makes a torn // slot survivable, so both halves are asserted: the parity, and that // corrupting the newer slot falls back rather than failing. // // Mutation checks: always write slot A, and the fallback finds nothing; // compare generations before validating the hash, and the torn slot wins. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); // Three generations: A, B, A. try tp.pg().publish(.{ .seq = 10 }); try tp.pg().publish(.{ .seq = 20 }); try tp.pg().publish(.{ .seq = 30 }); try testing.expectEqual(@as(u64, 3), tp.pg().generation); // Generation 3 is odd, so slot A. Generation 2 must still be in slot B. const a_gen = std.mem.readInt(u64, tp.pg().page(page_watermark_a)[0..8], .little); const b_gen = std.mem.readInt(u64, tp.pg().page(page_watermark_b)[0..8], .little); try testing.expectEqual(@as(u64, 3), a_gen); try testing.expectEqual(@as(u64, 2), b_gen); tp.close(); // Tear the newer slot: the open must fall back to generation 2, not fail // and not believe the garbage. { var f = try std.Io.Dir.cwd().openFile(io, tp.path, .{ .mode = .read_write }); defer f.close(io); var junk: [64]u8 = undefined; @memset(&junk, 0xFF); try f.writePositionalAll(io, &junk, page_size); } var again = try reopen(io, tp.path); defer again.deinit(); try testing.expectEqual(@as(u64, 2), again.loaded.generation); try testing.expectEqual(@as(u64, 20), again.loaded.seq); } test "both slots unreadable opens anyway, with no checkpoint" { // Ground rule 4: the database must always open. Once the log is truncated // this costs data, which is why it warns -- but refusing to start would // cost all of it. // // Mutation check: return an error from load_watermark instead and this // fails outright. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); try tp.pg().publish(.{ .seq = 5 }); try tp.pg().publish(.{ .seq = 6 }); tp.close(); { var f = try std.Io.Dir.cwd().openFile(io, tp.path, .{ .mode = .read_write }); defer f.close(io); var junk: [page_size * 2]u8 = undefined; @memset(&junk, 0xA5); try f.writePositionalAll(io, &junk, page_size); } var again = try reopen(io, tp.path); defer again.deinit(); try testing.expectEqual(@as(u64, 0), again.loaded.generation); try testing.expectEqual(@as(u64, 0), again.loaded.seq); } test "a hash-valid but impossible slot is refused" { // A slot that validates is *trusted*, so it has to be checked for sense as // well as integrity: a generation of 0 or an alloc_tail below the first data // page would be believed and would describe a file that does not exist. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); try tp.pg().publish(.{ .seq = 11 }); tp.close(); { var f = try std.Io.Dir.cwd().openFile(io, tp.path, .{ .mode = .read_write }); defer f.close(io); var b: [page_size]u8 = undefined; _ = try f.readPositionalAll(io, &b, page_size); // slot A holds gen 1 // alloc_tail = 1, below page_first_data, and re-hash so it validates. std.mem.writeInt(u64, b[16..24], 1, .little); std.mem.writeInt(u64, b[wm_hashed_len..][0..8], header_hash(b[0..wm_hashed_len]), .little); try f.writePositionalAll(io, &b, page_size); } var again = try reopen(io, tp.path); defer again.deinit(); // Refused, so no checkpoint rather than a nonsensical one. try testing.expectEqual(@as(u64, 0), again.loaded.generation); } test "freed pages are withheld for two generations and survive a reopen" { // The delay is what keeps generation N-1 usable as a fallback image: its // pages must still be allocated while N is current. // // Mutation check: move free_pending straight into free_ready at publish and // the withholding assertions go red. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const victim = try tp.pg().alloc_pages(2); @memset(tp.pg().page_mut(victim), 0x11); try tp.pg().publish(.{ .seq = 1 }); try tp.pg().free_pages(victim, 2); try testing.expectEqual(@as(u32, 0), tp.pg().free_ready_pages()); try tp.pg().publish(.{ .seq = 2 }); // pending -> hold try testing.expectEqual(@as(u32, 0), tp.pg().free_ready_pages()); // Still intact, which is the point: generation 1 references it. try testing.expectEqual(@as(u8, 0x11), tp.pg().page(victim)[0]); try tp.pg().publish(.{ .seq = 3 }); // hold -> ready try testing.expectEqual(@as(u32, 2), tp.pg().free_ready_pages()); tp.close(); // And the list itself is durable. var again = try reopen(io, tp.path); defer again.deinit(); try testing.expectEqual(@as(u32, 2), again.free_ready_pages()); } test "the watermark is never published before the pages it describes" { // The load-bearing ordering of the whole design: every page a watermark // describes is durable before the watermark that describes it. Reverse them // and a crash in between leaves a watermark pointing at pages that were // never written, with the log already truncated below it. // // A kill -9 cannot produce that state -- the page cache survives the // process -- so this simulates the writeback that never happened, using the // pager's own record of which pages were dirtied since the last sync. That // is the only honest way to make the ordering mutation-checkable, and // without it the mutation "delete the sync before the watermark" leaves // every other test in this file green. // // Mutation checks, stated precisely because the obvious one is not a // violation: publish syncs twice before the watermark (data pages, then the // free list), so removing *either* call alone is harmless and leaves this // green -- correctly. What it catches is removing both, or moving them after // the watermark write. The invariant is "at least one full sync between the // last data write and the watermark", not any particular call. const gpa = testing.allocator; var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); // A page the checkpoint will reference, written and then published. const cat = try tp.pg().alloc_pages(2); @memset(tp.pg().page_mut(cat), 0x77); @memset(tp.pg().page_mut(cat + 1), 0x77); try tp.pg().publish(.{ .seq = 900, .catalog_page = cat, .catalog_len = 2 * page_size }); // After a correct publish the only page still pending is the watermark slot // itself, which its own msync made durable. Everything else the watermark // references was flushed first, so discarding whatever is still pending -- // minus that slot -- must leave a loadable image. const pending = try tp.pg().dirty_pages(gpa); defer gpa.free(pending); const slot: u32 = if (tp.pg().generation & 1 == 1) page_watermark_a else page_watermark_b; var lost: std.ArrayListUnmanaged(u32) = .empty; defer lost.deinit(gpa); for (pending) |pg_no| if (pg_no != slot) try lost.append(gpa, pg_no); tp.close(); try Pager.simulate_lost_writeback(gpa, io, tp.path, lost.items); var again = try reopen(io, tp.path); defer again.deinit(); try testing.expectEqual(@as(u64, 1), again.loaded.generation); try testing.expectEqual(@as(u64, 900), again.loaded.seq); try testing.expectEqual(cat, again.loaded.catalog_page); // The referenced page survived, because publish synced it first. for (again.page(cat)) |byte| try testing.expectEqual(@as(u8, 0x77), byte); for (again.page(cat + 1)) |byte| try testing.expectEqual(@as(u8, 0x77), byte); } test "copy-on-write leaves the published image byte-identical" { // The single most important property in this milestone. Recovery is // `image + replay(seq > watermark)`, and that is only correct because no // page the image references is ever written *differently* after it is // published. Everything else about the crash story follows from this. // // Mutation checks: make page_mut_cow return the page without copying, or // without updating the slot, and the snapshot comparison goes red. Remove // the stable-mark assert in page_mut and this still passes -- that assert is // the mechanical belt for callers that bypass COW entirely, and its own test // is "a direct write inside the image is refused" below. const gpa = testing.allocator; var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 256 << 20); defer tp.deinit(); const pg = tp.pg(); // Build an image of distinguishable pages, each owned by a slot, and // publish it. var slots: [64]u32 = undefined; for (&slots, 0..) |*slot, i| { slot.* = try pg.alloc_pages(1); @memset(pg.page_mut(slot.*), @truncate(i + 1)); } try pg.publish(.{ .seq = 1 }); // Snapshot every byte the image covers. const snapshot = try gpa.alloc(u8, @intCast(pg.stable_bytes())); defer gpa.free(snapshot); @memcpy(snapshot, pg.reserve[0..snapshot.len]); // Now rewrite every one of those pages through COW. for (&slots, 0..) |*slot, i| { const before = slot.*; const p = try pg.page_mut_cow(slot); @memset(p, @truncate(0x80 + i)); // The page moved out of the image, and the slot followed it. try testing.expect(slot.* != before); try testing.expect(slot.* >= pg.stable_pages); } // The published image is untouched, byte for byte. try testing.expectEqualSlices(u8, snapshot, pg.reserve[0..snapshot.len]); // And the new contents are readable through the updated slots. for (slots, 0..) |slot, i| { const want: u8 = @truncate(0x80 + i); try testing.expectEqual(want, pg.page(slot)[0]); } } test "a page already above the mark is written in place, not copied" { // COW must not copy what it does not have to: a page allocated since the // last publish is not part of any image, so writing it is free. Getting this // wrong would double the file on every write. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const pg = tp.pg(); try pg.publish(.{ .seq = 1 }); var slot = try pg.alloc_pages(1); // fresh, above the mark const before = slot; for (0..10) |_| { const p = try pg.page_mut_cow(&slot); @memset(p, 0x42); } try testing.expectEqual(before, slot); } test "freed pages are recycled rather than growing the file" { // Copy-on-write abandons every page it touches, in every generation, so // without reuse a write-heavy workload grows the file by // `generations x touched_set` without bound. That is why PLAN amendment A2 // makes the free list a prerequisite rather than the defense-in-depth D6.2 // assumed. // // Mutation check: make alloc_pages_assume_reserved skip take_free and the // tail assertion goes red. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const pg = tp.pg(); var slots: [32]u32 = undefined; for (&slots, 0..) |*slot, i| { slot.* = try pg.alloc_pages(1); @memset(pg.page_mut(slot.*), @truncate(i + 1)); } try pg.publish(.{ .seq = 1 }); // Three rounds of rewriting every page. The first COWs each one and frees // the original; two publishes later those originals come back. var tail_after_first: u32 = 0; for (0..3) |round| { for (&slots) |*slot| { const p = try pg.page_mut_cow(slot); @memset(p, @truncate(round)); } try pg.publish(.{ .seq = round + 2 }); if (round == 0) tail_after_first = pg.alloc_tail; } // By round three the freed pages are being handed back, so the tail has // stopped climbing by a full working set per round. const grew = pg.alloc_tail - tail_after_first; try testing.expect(grew < slots.len * 2); // And the data is still correct through the current slots. for (slots) |slot| try testing.expectEqual(@as(u8, 2), pg.page(slot)[0]); } test "the published image is made hardware read-only where the build allows it" { // The belt described in `protect_stable`. It exists for the mistake the // structural mechanism cannot catch: a caller writing through a `*Node` // obtained *before* the stable mark moved. // // What is asserted here is that the protection is really applied, not that a // violating write faults -- a SIGSEGV cannot be caught in-process, so // verifying the fault would need a child process, and the fault itself is // the OS's behaviour rather than this code's. An mprotect that silently // failed would leave a belt that looks present and does nothing, which is // the failure worth guarding against here. if (!protect_stable) return; // ReleaseFast: deliberately off var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const pg = tp.pg(); // Enough pages that the protected range spans at least one system page even // where those are 16 KiB. var i: usize = 0; while (i < 64) : (i += 1) { const p = try pg.alloc_pages(1); @memset(pg.page_mut(p), 0x31); } try pg.publish(.{ .seq = 1 }); try testing.expect(pg.protect_ok); // Reads through the protection still work. try testing.expectEqual(@as(u8, 0x31), pg.page(page_first_data)[0]); } test "an empty file is treated as absent, not as corruption" { // A create that died before its header landed must not look like a corrupt // database — the engine has to be able to open. var threaded: std.Io.Threaded = .init_single_threaded; defer threaded.deinit(); const io = threaded.io(); var tp = try TmpPager.init(io, 64 << 20); defer tp.deinit(); const path = try testing.allocator.dupe(u8, tp.path); defer testing.allocator.free(path); tp.close(); { var f = try std.Io.Dir.cwd().openFile(io, path, .{ .mode = .read_write }); defer f.close(io); try f.setLength(io, 0); } tp.pager = try Pager.open(testing.allocator, io, path, .{ .uuid = 7, .reserve_bytes = 64 << 20 }); try testing.expect(tp.pg().fresh); }