From 319a515b89fdc383ef6b60ca2f38b37743edc3fc Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Tue, 4 Aug 2026 00:40:30 +0300 Subject: [PATCH] pager: reuse the data file when there is no checkpoint to honour `Pager.open` set `alloc_tail` to the end of the existing file -- "everything already in the file is allocated until a watermark narrows it down". Safe when a watermark exists. When one does not, it is the opposite of safe: nothing in the file is referenced, the log is the whole truth and replay is about to rebuild the slab, the trees and the overflow from it, so every reopen started allocating *above* the previous copy. With no watermark there is also no free list, so the old copy was never given back. Linear growth per open, unbounded. It does not need a crash. A database small enough never to reach the 32 MB checkpoint threshold never publishes a watermark at all, so *every* clean reopen took this path: 20 documents inserted per cycle, 12 reopen cycles before 17, 34, 50, 67, 85, 102, 118, 135, 168, 201, 236, 269 MB after 17 MB, flat 240 documents in a 269 MB file, heading for `DatabaseTooLarge`. The crash fuzzer shows the same thing under a real workload -- 60 crash/reopen cycles with ~460 documents ended at 2735 MB before, 17 MB after, with the prefix invariant holding either way. That number was sitting in its own output as `data=2735MB` and reads as normal until you divide it by the document count. The file is deliberately not truncated. The mapping already covers these pages and `grow_to` extends the file only when the mapping is too small, so shortening the file behind a mapping that still spans it would turn a later write into SIGBUS. Reusing from the front is what the unbounded growth needed; giving the disk back is a separate change to the same function. Mutation: leave `alloc_tail` at the file end -- red on the new test, which opens, writes and closes three times without a checkpoint and requires the third tail to be within one slab extent of the first. --- src/db.zig | 44 ++++++++++++++++++++++++++++++++++++++++++++ src/pager.zig | 24 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/db.zig b/src/db.zig index cf74735..4ac2479 100644 --- a/src/db.zig +++ b/src/db.zig @@ -2106,6 +2106,50 @@ test "compaction reclaims garbage but leaves a garbage-free log alone" { try testing.expect(engine.log.data_bytes < after_insert * 2); } +test "reopening without a checkpoint reuses the data file instead of appending to it" { + // Mutation check: delete the `loaded.generation == 0` reset of `alloc_tail` + // in `Pager.open`. Red -- each reopen starts allocating above the previous + // file end, so the file grows by a slab extent every time and no watermark + // exists to put the abandoned copy on a free list. Unbounded, and it needs no + // crash: a database too small to reach the checkpoint threshold never + // publishes a watermark, so every clean reopen took that path. Measured + // through the server at 20 documents per cycle: +17 MB per reopen. + var threaded: std.Io.Threaded = .init_single_threaded; + defer threaded.deinit(); + var env = test_env(&threaded); + const io = env.io; + const gpa = testing.allocator; + + var tmp = try TmpLog.init(gpa); + defer tmp.deinit(gpa); + + // Three rounds of open, write a little, close -- with no checkpoint, so the + // data file never gets a watermark and replay rebuilds everything each time. + var tails: [3]u32 = undefined; + for (0..3) |round| { + var engine = try Engine.open(gpa, io, tmp.path); + defer engine.deinit(); + try testing.expectEqual(@as(u64, 0), engine.pager.loaded.generation); + try engine.lock(); + for (0..5) |i| { + var d = try make_doc(gpa, @intCast(round * 10 + i), "x"); + defer d.deinit(); + try engine.insert("app", "c", &d, &env.gen); + } + try engine.commit(); + engine.unlock(); + tails[round] = engine.pager.alloc_tail; + // Every document written so far is readable, so reuse is not data loss. + try testing.expectEqual(@as(u64, (round + 1) * 5), engine.live_docs); + } + + // The third open must not have allocated a third copy of the arena. Reuse + // makes the tail essentially flat; appending makes it grow by a slab extent + // (2048 pages) per round. + try testing.expect(tails[2] < tails[0] + slab_extent_pages); + try testing.expect(tails[1] < tails[0] + slab_extent_pages); +} + test "an append after a checkpoint keeps its extent instead of abandoning it" { // Mutation check: delete the `resumed` branch in `slab_reserve`. Red on the // extent count -- every checkpoint would take a fresh 8 MiB extent per diff --git a/src/pager.zig b/src/pager.zig index 738f08f..f2f67b1 100644 --- a/src/pager.zig +++ b/src/pager.zig @@ -352,6 +352,30 @@ pub const Pager = struct { // narrows it down. self.alloc_tail = @max(page_first_data, self.file_pages); try self.load_watermark(); + if (self.loaded.generation == 0) { + // No usable checkpoint: either none was ever published, or both + // watermark slots were unreadable. Then *nothing* in this file is + // referenced -- the log is the whole truth and replay is about to + // rebuild the slab, the trees and the overflow from it -- so the + // file is free space, not allocated space. + // + // Leaving `alloc_tail` at the file end instead made every reopen + // append a fresh copy above the last one, with no watermark and + // therefore no free list to ever give the old copy back. Linear + // growth per open, unbounded, and it does not need a crash: a + // database small enough never to reach the checkpoint threshold + // never publishes a watermark at all, so *every* clean reopen + // took this path. Measured at 20 documents per cycle: +17 MB per + // reopen, 269 MB after twelve, on course for DatabaseTooLarge. + // + // The file is not truncated here on purpose. The mapping is + // already established over these pages, and `grow_to` extends the + // file only when the mapping is too small -- so shortening the + // file behind a mapping that still covers it would turn a later + // write into SIGBUS. Reusing from the front bounds the file at + // its high-water mark, which is what the unbounded growth needed. + self.alloc_tail = page_first_data; + } } return self; }