From 6c7f1f2e77497de3b8c38871957de40ed0aaf9b8 Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Mon, 3 Aug 2026 23:38:23 +0300 Subject: [PATCH] tests/fuzz: crash-consistency fuzzer (crash-fuzz.js + README) Black-box SIGKILL fuzzer for the M0 mmap+WAL crash story. Random write workload through the official driver, kill -9 at a random point, reopen the same log, verify the recovered state against an in-memory model: - prefix invariant: recovered state == history[0..m) for some m in [acked, sent]; every acked write durable, in-flight commands all-or-nothing (group commit), nothing after them may survive - always-opens (replay never refuses); index presence tied to the prefix and find({k:v}) correctness (rebuild after replay); countDocuments - unexpected server death (Zig panic, replay refusal) is a finding with the server log; --verify-exec read-backs updates to separate execution bugs from replay bugs; --no-kill for graceful-restart runs; --heavy passes a 1 MiB compact threshold to fuzz compaction/checkpoint windows Deterministic via seeded PRNG; failures dump a repro artifact with the seed. Mutation-checked: over-strict prefix check goes red on lost in-flight ops. Observation recorded: small churn-heavy DBs can exhaust the pager's 64 GB address-space reservation (data file grows in >=8 MiB compounding steps and never shrinks without a rebuild), surfacing as DatabaseTooLarge on writes. --- tests/fuzz/README.md | 83 +++++ tests/fuzz/crash-fuzz.js | 660 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 743 insertions(+) create mode 100644 tests/fuzz/README.md create mode 100644 tests/fuzz/crash-fuzz.js diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md new file mode 100644 index 0000000..e89a1ba --- /dev/null +++ b/tests/fuzz/README.md @@ -0,0 +1,83 @@ +# Fuzz / torture harnesses + +Black-box and in-process harnesses that hunt for engine bugs and bottlenecks +that the fixed e2e scenarios cannot reach. The e2e suites prove specific +behaviours; these generate their own workloads and check invariants. + +## crash-fuzz.js — crash-consistency fuzzer + +The P0 harness for the M0 (mmap + WAL) crash story. Drives the server through +the official driver with a random write workload (insert batches, single +inserts, `$set`/`$inc`/`$unset` updates, deletes, `createIndex`), kills it with +`SIGKILL` at a random point, reopens the same log file, and verifies the +recovered state against an in-memory model. + +**The invariant under test (the "prefix invariant").** With one sequential +client and fsync-before-ack commits, the state that survives a crash must be +`apply(history[0..m))` for some `m` with `acked <= m <= sent`: every +fully-acked write is durable, an in-flight command is either fully there or +fully gone (group commit = one fsync per command), and nothing after it may +survive. The verifier also checks, at the matched prefix: + +- the database always opens (replay never refuses — PLAN ground rule 4); +- the `k_1` index exists iff its create record is in the prefix, and + `find({k: v})` returns exactly the docs with `k == v` (index rebuild after + replay must be correct, whether the query used the index or a scan); +- `countDocuments({})` matches the model. + +Cycles share one log file, so checkpoints, log truncation, COW free-list +reuse and compaction (in `--heavy`) are exercised across cycles. The server +spawned by the harness may never self-crash: a Zig panic or a replay refusal +is reported as a finding with the server log. + +**Usage** + +```sh +zig build # fresh server binary first! +node tests/fuzz/crash-fuzz.js # 60 quick cycles, random seed +node tests/fuzz/crash-fuzz.js --seed 42 # reproducible scenario +node tests/fuzz/crash-fuzz.js --heavy # big batches + 1 MB compact + # threshold: hits checkpoint/ + # compaction windows +node tests/fuzz/crash-fuzz.js --no-kill --verify-exec + # graceful stop + reopen, and + # read-back every update (isolates + # execution bugs from replay bugs) +``` + +Options: `--seed N --rounds N --max-docs N --batch-max N --kill-delay-ms N +--verbose --keep-log --port N`. On failure a repro artifact is written to +`/tmp/crash-fuzz-fail-.json` and the rerun command is printed. + +Determinism: the op sequence and the kill decision come from a seeded PRNG, +so the same seed reproduces the same scenario. Kill timing inside the window +is OS-scheduled (as in any crash tester — RocksDB db_crashtest works the same +way); a failure's seed + artifact reproduce the *scenario*, and the invariant +check is timing-independent. + +**Mutation-checked** (repo ground rule 2): making the verifier require +`m == sent` (over-strict) turns seeds with lost in-flight ops red, proving the +harness actually observes both prefix states (`acked` and `sent`), not just +the full state. + +## Notes on what the crash fuzzer does *not* cover (by design) + +- **Torn log tails / torn watermark writes**: `kill -9` is process death, not + power loss — page cache survives, so a partially-written block is rare and + OS-scheduled. The replay fuzzer (log truncation at random offsets) is the + deterministic way to hit that surface; it is planned (P0 item 2) but not + yet built. +- **Concurrent clients**: the fuzzer uses one sequential client so the prefix + invariant is exactly checkable. Race/interleaving coverage stays with + `tests/e2e/e2e2.js concurrent`. +- **Semantic differential vs real MongoDB**: planned (P1 item 4). + +## Known engine observations (findings so far) + +- A small churn-heavy database can exhaust the pager's 64 GB address-space + reservation: the data file grows in ≥8 MiB steps, compounding, and never + shrinks until a compaction (data-file rebuild) or checkpoint reclaims it. + With `--compact-threshold` left at the 16 MB default and a small log, the + file can reach 64 GB and writes start failing with `DatabaseTooLarge` after + enough growth events. `--heavy` passes `--compact-threshold 1 MiB` to keep + the file bounded and to fuzz the rebuild+crash path. diff --git a/tests/fuzz/crash-fuzz.js b/tests/fuzz/crash-fuzz.js new file mode 100644 index 0000000..5b1884b --- /dev/null +++ b/tests/fuzz/crash-fuzz.js @@ -0,0 +1,660 @@ +//!/usr/bin/env node +// +// Crash-consistency fuzzer for multiforadb (tests/fuzz/crash-fuzz.js). +// +// Black-box: drives the server through the official driver, kills it with +// SIGKILL at a random point, reopens the same log file, and verifies the +// recovered state against an in-memory model. +// +// The invariant under test (the "prefix invariant"): +// with one sequential client and fsync-before-ack commits, the state that +// survives a crash is exactly `apply(history[0..m))` for some m with +// acked <= m <= sent, where the in-flight command (if any) contributes a +// prefix of its documents. Every fully-acked write must be durable; an +// in-flight write may or may not be; nothing after it may be. +// +// A cycle is: generate random ops -> SIGKILL at a random point (sometimes +// mid-write, sometimes right after an ack) -> reopen -> verify prefix +// invariant + index correctness + count. All cycles share one log file, so +// checkpoints, log truncation and free-list reuse are exercised across +// cycles when the log grows past the checkpoint threshold (--heavy). +// +// Determinism: the op sequence and the kill decision come from a seeded PRNG, +// so `--seed N` reproduces the same scenario (kill timing inside the window +// is OS-scheduled, exactly as in any crash tester, e.g. RocksDB db_crashtest). +// +// node tests/fuzz/crash-fuzz.js # 60 quick cycles +// node tests/fuzz/crash-fuzz.js --heavy # grow log past the +// # checkpoint threshold +// node tests/fuzz/crash-fuzz.js --seed 1234 --rounds 200 +// +// Env: CRASH_FUZZ_PORT listen port (default 27230) +// MFDB_BIN server binary (default ../../zig-out/bin/multiforadb) +// CRASH_FUZZ_DB log file (default ../../.zig-cache/crash-fuzz.log) +const { MongoClient } = require('../e2e/node_modules/mongodb'); +const { spawn } = require('child_process'); +const fs = require('fs'); +const path = require('path'); + +// -------------------------------------------------------------------------- +// argv / config +// -------------------------------------------------------------------------- +const argv = process.argv.slice(2); +const args = {}; +for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (!a.startsWith('--')) continue; + const v = argv[i + 1]; + args[a.slice(2)] = v !== undefined && !v.startsWith('--') ? v : true; +} +function intArg(k, dflt) { + if (args[k] === undefined || args[k] === true) return dflt; + const n = Number(args[k]); + if (!Number.isInteger(n) || n < 0) { + console.error(`bad --${k} value: ${args[k]}`); + process.exit(2); + } + return n; +} + +const HEAVY = !!args.heavy; +const PORT = Number(process.env.CRASH_FUZZ_PORT || args.port || 27230); +const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb'); +const DBFILE = process.env.CRASH_FUZZ_DB || path.resolve(__dirname, '../../.zig-cache/crash-fuzz.log'); +const VERBOSE = !!args.verbose; +const VERIFY_EXEC = !!args['verify-exec']; +const NO_KILL = !!args['no-kill']; +const KEEP_LOG = !!args['keep-log']; +const ROUNDS = intArg('rounds', HEAVY ? 8 : 60); +const SEED = args.seed !== undefined ? intArg('seed', 0) : ((Math.random() * 0xffffffff) >>> 0); +const KILL_PROB = HEAVY ? 0.3 : 0.22; +const KILL_INFLIGHT_PROB = 0.6; // of kill decisions, share that fire mid-write +const MAX_DOCS = intArg('max-docs', HEAVY ? 3000 : 400); +const BATCH_MAX = intArg('batch-max', HEAVY ? 400 : 120); +const MAX_CYCLE_OPS = intArg('max-cycle-ops', HEAVY ? 60 : 30); +const KILL_DELAY_MS = intArg('kill-delay-ms', 1); +const LONG_STR = HEAVY ? 4096 : 1024; + +// -------------------------------------------------------------------------- +// deterministic PRNG (mulberry32) +// -------------------------------------------------------------------------- +function mulberry32(seed) { + let a = seed >>> 0; + return function () { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +// -------------------------------------------------------------------------- +// model: the intended database state, plus the flat op history of a cycle +// -------------------------------------------------------------------------- +// Deep-clone a model doc: applyFlat mutates doc objects in place, and every +// candidate prefix state must be an independent copy (docs are JSON-safe: +// numbers, strings, bools, null, arrays, nested objects). +function cloneDoc(d) { + return JSON.parse(JSON.stringify(d)); +} + +class Model { + constructor(other) { + this.docs = new Map(); // _id -> doc (plain JS object) + this.indexCreated = false; + if (other) { + for (const [k, v] of other.docs) this.docs.set(k, cloneDoc(v)); + this.indexCreated = other.indexCreated; + } + } +} + +// A flat op: {kind:'insert', doc} | {kind:'update', id, patch} | +// {kind:'delete', id} | {kind:'createIndex'} +function applyFlat(docs, f) { + switch (f.kind) { + case 'insert': + // Clone: the same flat op is applied by applyOp (the live model) and by + // stateAfter (every candidate prefix). Without a clone, both mutate the + // shared genDoc object and an update would be applied twice. + docs.set(f.doc._id, cloneDoc(f.doc)); + break; + case 'update': { + const d = docs.get(f.id); + if (!d) throw new Fail('model bug: update of missing doc', { f }); + const p = f.patch; + if (p.$set) for (const [k, v] of Object.entries(p.$set)) d[k] = v; + if (p.$inc) for (const [k, v] of Object.entries(p.$inc)) d[k] = (d[k] || 0) + v; + if (p.$unset) for (const k of Object.keys(p.$unset)) delete d[k]; + break; + } + case 'delete': + docs.delete(f.id); + break; + case 'createIndex': + break; // no doc effect + } +} + +function stateAfter(base, flat, m) { + const docs = new Map(); + for (const [k, v] of base.docs) docs.set(k, cloneDoc(v)); + for (let i = 0; i < m; i++) applyFlat(docs, flat[i]); + return docs; +} + +function applyOp(intended, op) { + if (op.kind === 'batch') { + for (const d of op.docs) intended.docs.set(d._id, cloneDoc(d)); + } else { + applyFlat(intended.docs, op); + } + if (op.kind === 'createIndex') intended.indexCreated = true; +} + +// -------------------------------------------------------------------------- +// canonical comparison (driver-parsed BSON vs plain JS model docs) +// -------------------------------------------------------------------------- +function canon(v) { + if (v === null || v === undefined) return 'null'; + if (typeof v === 'number') return Number.isInteger(v) ? 'i' + v : 'd' + v; + if (typeof v === 'string') return 's' + v; + if (typeof v === 'boolean') return 'b' + v; + if (Array.isArray(v)) return '[' + v.map(canon).join(',') + ']'; + if (typeof v === 'object') { + const ks = Object.keys(v).sort(); + return '{' + ks.map((k) => canon(k) + ':' + canon(v[k])).join(',') + '}'; + } + return '?' + String(v); +} + +function sameDocs(a, b) { + if (a.size !== b.size) return false; + for (const [k, v] of a) { + if (!b.has(k)) return false; + if (canon(v) !== canon(b.get(k))) return false; + } + return true; +} + +// First few concrete differences between two doc maps (for diagnostics). +function diffState(a, b) { + const out = []; + for (const [k, v] of a) { + if (!b.has(k)) { + out.push(`db missing _id=${k}`); + if (out.length >= 5) return out; + continue; + } + if (canon(v) !== canon(b.get(k))) { + out.push(`_id=${k} differs: model=${canon(v)} db=${canon(b.get(k))}`); + if (out.length >= 5) return out; + } + } + for (const [k] of b) { + if (!a.has(k)) { + out.push(`db has extra _id=${k}`); + if (out.length >= 5) return out; + } + } + return out; +} + +// -------------------------------------------------------------------------- +// random document / op generation +// -------------------------------------------------------------------------- +function randString(rng) { + const lenRoll = rng(); + let len; + if (lenRoll < 0.4) len = Math.floor(rng() * 32); // short + else if (lenRoll < 0.8) len = 32 + Math.floor(rng() * 128); + else len = 128 + Math.floor(rng() * LONG_STR); // long: makes batches slow + let s = ''; + const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \t\n"\'{}[]'; + for (let i = 0; i < len; i++) s += chars[Math.floor(rng() * chars.length)]; + return s; +} + +function genDoc(rng, id, idx) { + const d = { _id: id, k: Math.floor(rng() * 10), n: Math.floor(rng() * 100000) }; + const r = rng(); + if (r < 0.35) d.s = randString(rng); + if (r < 0.5) d.nested = { a: Math.floor(rng() * 100), b: [Math.floor(rng() * 10), Math.floor(rng() * 10)], c: { d: rng() < 0.5 } }; + if (r < 0.65) d.arr = [Math.floor(rng() * 5), randString(rng), rng() < 0.5]; + if (r < 0.75) d.flag = rng() < 0.5; + if (r < 0.8) d.z = null; + if (idx !== undefined && rng() < 0.3) d.idx = idx; // occasionally tagged + return d; +} + +function pickKey(rng, docs) { + const keys = [...docs.keys()]; + return keys[Math.floor(rng() * keys.length)]; +} + +function genUpdate(rng, intended) { + const id = pickKey(rng, intended.docs); + const r = rng(); + if (r < 0.35) return { kind: 'update', id, patch: { $set: { k: Math.floor(rng() * 10) } } }; + if (r < 0.55) return { kind: 'update', id, patch: { $inc: { n: 1 } } }; + if (r < 0.75) return { kind: 'update', id, patch: { $set: { s: randString(rng) } } }; + if (r < 0.9) return { kind: 'update', id, patch: { $unset: { flag: '' } } }; + return { kind: 'update', id, patch: { $set: { nested: { a: 7, b: [1, 2], c: { d: false } }, arr: ['x', 3] } } }; +} + +// Generator state: rng, nextId, thresholds, per-run switches. +function makeGen(seed, cfg) { + const rng = mulberry32(seed); + return { + rng, + nextId: 1, + idxThreshold: null, // flat count at which createIndex becomes durable, once generated + cfg, + nextOp(intended) { + const { rng, cfg } = this; + const size = intended.docs.size; + const r = rng(); + if (!intended.indexCreated && r < 0.08) return { kind: 'createIndex' }; + const overCap = size > cfg.maxDocs * 0.85; + let kind; + if (size === 0) { + kind = 'batch'; + } else if (overCap) { + const x = rng(); + kind = x < 0.55 ? 'delete' : x < 0.8 ? 'update' : 'batch'; + } else { + const x = rng(); + kind = x < 0.3 ? 'batch' : x < 0.42 ? 'insert' : x < 0.68 ? 'update' : 'delete'; + } + switch (kind) { + case 'batch': { + const n = overCap ? 1 + Math.floor(rng() * 10) : 1 + Math.floor(rng() * cfg.batchMax); + const docs = []; + for (let i = 0; i < n; i++) docs.push(genDoc(rng, this.nextId++, i)); + return { kind: 'batch', docs }; + } + case 'insert': + return { kind: 'insert', doc: genDoc(rng, this.nextId++) }; + case 'update': + return genUpdate(rng, intended); + case 'delete': + return { kind: 'delete', id: pickKey(rng, intended.docs) }; + } + }, + }; +} + +// -------------------------------------------------------------------------- +// server control (mirrors tests/e2e/e2e6.js) +// -------------------------------------------------------------------------- +let server = null; +let serverLog = ''; +let exited = null; // {code, sig} once the child has exited +let killedByUs = false; + +function cleanup() { + if (server && !exited) { + try { server.kill('SIGKILL'); } catch {} + } +} +process.on('exit', cleanup); +process.on('SIGINT', () => { if (args.debug) console.error('[harness SIGINT]'); cleanup(); process.exit(130); }); +process.on('SIGTERM', () => { if (args.debug) console.error('[harness SIGTERM]'); cleanup(); process.exit(143); }); + +function checkUnexpectedDeath(where) { + // Our own SIGKILL is expected; anything else (Zig panic, replay refusing + // to open, SIGABRT...) is a finding. + if (exited && !(killedByUs && exited.sig === 'SIGKILL')) { + throw new Fail(`server died unexpectedly at ${where}`, { + code: exited.code, + sig: exited.sig, + logTail: serverLog.slice(-3000), + }); + } +} + +function startServer(fresh) { + return new Promise((resolve, reject) => { + if (fresh) { + fs.rmSync(DBFILE, { force: true }); + fs.rmSync(DBFILE + '.data', { force: true }); + } + serverLog = ''; + exited = null; + killedByUs = false; + const extra = HEAVY ? ['--compact-threshold', String(1024 * 1024)] : []; + server = spawn(BIN, ['--port', String(PORT), '--db', DBFILE].concat(extra), { + stdio: ['ignore', 'pipe', 'pipe'], + }); + server.stdout.on('data', (d) => (serverLog += d)); + server.stderr.on('data', (d) => (serverLog += d)); + server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`))); + server.on('exit', (code, sig) => { + exited = { code, sig }; + if (args.debug) console.error(`[${Date.now()} server ${server?.pid} exited code=${code} sig=${sig} killedByUs=${killedByUs} graceful=${gracefulStop}]`); + server = null; + if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`; + }); + const deadline = Date.now() + 20000; + (async () => { + while (Date.now() < deadline) { + if (exited) { + reject(new Error(`server child exited during start (port ${PORT} busy?)\n${serverLog}`)); + return; + } + const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 1000 }); + try { + await c.connect(); + await c.db('admin').command({ ping: 1 }); + await c.close(); + return resolve(); + } catch { + try { await c.close(); } catch {} + await sleep(100); + } + } + reject(new Error(`server did not come up on :${PORT}\n${serverLog}`)); + })(); + }); +} + +async function stopServerGracefully() { + gracefulStop = true; + if (server && !exited) { + const exitedP = new Promise((r) => server.once('exit', r)); + if (args.debug) console.error(`[${Date.now()} graceful-stop targeting pid ${server.pid}]\n${new Error().stack.split('\n').slice(2, 5).join('\n')}`); + try { server.kill('SIGTERM'); } catch {} + await Promise.race([exitedP, sleep(5000)]); + } + server = null; +} + +async function killServer() { + killedByUs = true; + if (!exited) { + const exitedP = new Promise((r) => { + if (exited) return r(); + const h = () => r(); + if (server) server.once('exit', h); + else r(); + }); + try { server.kill('SIGKILL'); } catch {} + await Promise.race([exitedP, sleep(5000)]); + } + checkUnexpectedDeath('kill'); + server = null; +} + +// -------------------------------------------------------------------------- +// client helpers +// -------------------------------------------------------------------------- +async function connectClient() { + const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { + serverSelectionTimeoutMS: 5000, + retryWrites: false, + }); + await c.connect(); + return c; +} + +async function closeClient(c) { + if (!c) return; + await Promise.race([c.close().catch(() => {}), sleep(2000)]).catch(() => {}); +} + +// -------------------------------------------------------------------------- +// one cycle: random ops then SIGKILL +// -------------------------------------------------------------------------- +async function execOp(coll, op) { + switch (op.kind) { + case 'insert': + return coll.insertOne(op.doc); + case 'batch': + return coll.insertMany(op.docs); + case 'update': + return coll.updateOne({ _id: op.id }, op.patch); + case 'delete': + return coll.deleteOne({ _id: op.id }); + case 'createIndex': + return coll.createIndex({ k: 1 }); + } +} + +async function runCycle(client, base, gen) { + const coll = client.db('fuzz').collection('c'); + const intended = new Model(base); + const flat = []; + let acked = 0; + let killMode = null; + let matched = null; + let idxPos = null; // flat count at which this cycle's createIndex becomes durable + + for (let i = 0; i < gen.cfg.maxCycleOps; i++) { + const kill = gen.rng() < gen.cfg.killProb || i === gen.cfg.maxCycleOps - 1; + const op = gen.nextOp(intended); + const inflight = !NO_KILL && kill && gen.rng() < KILL_INFLIGHT_PROB; + + if (op.kind === 'batch') { + for (const d of op.docs) flat.push({ kind: 'insert', doc: d }); + } else { + flat.push(op); + } + if (op.kind === 'createIndex') { + idxPos = flat.length; // flat count at which this cycle's record is durable + } + const sent = flat.length; + + if (inflight) { + const p = execOp(coll, op); + p.catch(() => {}); // the coming SIGKILL may kill the request: expected + await sleep(gen.cfg.killDelayMs); + await killServer(); + killMode = 'inflight'; + applyOp(intended, op); // optimistic; the prefix check decides what survived + } else { + await execOp(coll, op); + await sleep(5); // let a self-crash's exit event deliver before we check + checkUnexpectedDeath(`op ${flat.length} of cycle`); + acked = sent; + applyOp(intended, op); + // Diagnostic: read the doc back and compare with the model. Pins down + // whether a divergence is an execution bug (in-memory wrong) or a + // log/replay bug (in-memory right, lost after reopen). Must run while + // the server is still alive -- after the stop below the query would + // fail with ECONNREFUSED. + if (VERIFY_EXEC && op.kind === 'update') { + const back = await coll.findOne({ _id: op.id }); + const want = intended.docs.get(op.id); + if (!back || canon(back) !== canon(want)) { + throw new Fail(`cycle exec mismatch: update ${op.id} executed wrong`, { + back: back ? canon(back) : null, + want: canon(want), + }); + } + } + if (kill) { + if (NO_KILL) { + // --no-kill: verify durability of the run's writes via a graceful + // stop + reopen instead of SIGKILL. + await stopServerGracefully(); + } else { + await killServer(); + } + killMode = 'after'; + } + } + if (kill) break; + } + if (killMode === null) killMode = 'after'; // unreachable: last op forces a kill + return { flat, acked, sentCount: flat.length, killMode, matched, idxPos }; +} + +// -------------------------------------------------------------------------- +// verification after reopen: the prefix invariant +// -------------------------------------------------------------------------- +async function verify(client, base, r, cycleNo) { + const db = client.db('fuzz'); + const coll = db.collection('c'); + const dbDocs = await coll.find({}, { sort: { _id: 1 } }).toArray(); + const dbMap = new Map(dbDocs.map((d) => [d._id, d])); + const dbIndexes = await coll.indexes(); + const indexExists = dbIndexes.some((i) => i.name === 'k_1'); + if (VERBOSE) { + console.log(` [verify] indexes=${JSON.stringify(dbIndexes.map((i) => i.name))} baseIdx=${base.indexCreated} idxPos=${r.idxPos}`); + } + + let matched = null; + const diag = []; + for (let m = r.acked; m <= r.sentCount; m++) { + const st = stateAfter(base, r.flat, m); + const wantIdx = base.indexCreated || (r.idxPos !== null && m >= r.idxPos); + const stateOk = sameDocs(st, dbMap); + const indexOk = indexExists === wantIdx; + if (stateOk && indexOk) { matched = m; break; } + diag.push({ + m, + size: st.size, + stateOk, + indexOk, + diff: stateOk ? [] : diffState(st, dbMap), + }); + } + + if (matched === null) { + throw new Fail(`cycle ${cycleNo}: recovered state is not a valid prefix`, { + cycleNo, + acked: r.acked, + sent: r.sentCount, + killMode: r.killMode, + baseIdx: base.indexCreated, + idxPos: r.idxPos, + indexExists, + dbIndexes: dbIndexes.map((i) => i.name), + dbSize: dbMap.size, + candidates: diag, + serverLog: serverLog.slice(-4000), + ops: r.flat.map((f) => + f.kind === 'insert' ? `ins ${f.doc._id}` : + f.kind === 'update' ? `upd ${f.id} ${JSON.stringify(f.patch)}` : + f.kind === 'delete' ? `del ${f.id}` : 'idx'), + }); + } + + // Index / filter correctness at the matched prefix: find({k:v}) must return + // exactly the docs with k==v, whether the index exists (candidates+reapply) + // or the query was a full scan. + const st = stateAfter(base, r.flat, matched); + for (let v = 0; v < 10; v++) { + const got = (await coll.find({ k: v }, { sort: { _id: 1 } }).toArray()) + .map((d) => canon(d)) + .sort(); + const want = [...st.values()] + .filter((d) => d.k === v) + .map((d) => canon(d)) + .sort(); + if (got.length !== want.length || got.some((c, i) => c !== want[i])) { + throw new Fail(`cycle ${cycleNo}: find({k:${v}}) mismatch at prefix ${matched}`, { + cycleNo, v, matched, got, want, + }); + } + } + const cnt = await coll.countDocuments({}); + if (cnt !== st.size) { + throw new Fail(`cycle ${cycleNo}: countDocuments mismatch at prefix ${matched}`, { + cycleNo, matched, count: cnt, expected: st.size, + }); + } + r.matched = matched; + return { docs: st, indexCreated: indexExists }; +} + +// -------------------------------------------------------------------------- +// failure artifacts +// -------------------------------------------------------------------------- +class Fail extends Error { + constructor(msg, detail) { super(msg); this.detail = detail; } +} + +function dumpArtifact(err, gen) { + const art = { + seed: SEED, + mode: HEAVY ? 'heavy' : 'default', + rounds: ROUNDS, + error: err.message, + detail: err.detail, + nextId: gen.nextId, + }; + const file = `/tmp/crash-fuzz-fail-${SEED}.json`; + fs.writeFileSync(file, JSON.stringify(art, null, 2)); + console.error(`\n✗ ${err.message}`); + console.error(` detail: ${JSON.stringify(err.detail, null, 2).slice(0, 4000)}`); + console.error(` artifact: ${file}`); + console.error(` rerun: node tests/fuzz/crash-fuzz.js --seed ${SEED} --rounds ${ROUNDS}${HEAVY ? ' --heavy' : ''}`); +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// -------------------------------------------------------------------------- +// main +// -------------------------------------------------------------------------- +async function main() { + console.log(`crash-fuzz: seed=${SEED} rounds=${ROUNDS} mode=${HEAVY ? 'heavy' : 'default'} port=${PORT}`); + console.log(` bin=${BIN}`); + console.log(` db=${DBFILE}`); + + const gen = makeGen(SEED, { maxDocs: MAX_DOCS, batchMax: BATCH_MAX, maxCycleOps: MAX_CYCLE_OPS, killProb: KILL_PROB, killDelayMs: KILL_DELAY_MS }); + + if (!KEEP_LOG) { + fs.rmSync(DBFILE, { force: true }); + fs.rmSync(DBFILE + '.data', { force: true }); // stale data files hit the 64GB reservation ceiling + } + await startServer(true); + let base = new Model(); + + if (args.debug) { + const iv = setInterval(() => { + if (server) console.error(`[${Date.now()} watch server pid=${server.pid} exited=${JSON.stringify(exited)}]`); + else console.error(`[${Date.now()} watch server=null exited=${JSON.stringify(exited)}]`); + }, 100); + iv.unref(); + } + + try { + for (let c = 1; c <= ROUNDS; c++) { + let client = await connectClient(); + const r = await runCycle(client, base, gen); // ends with the server killed + await closeClient(client); + + await startServer(false); // reopen: replay must never refuse + client = await connectClient(); + const v = await verify(client, base, r, c); + await closeClient(client); + base = new Model(); + for (const [k, d] of v.docs) base.docs.set(k, d); + base.indexCreated = v.indexCreated; + + if (VERBOSE || c === 1 || c % 10 === 0 || c === ROUNDS) { + let size = 0, dsize = 0; + try { size = fs.statSync(DBFILE).size; } catch {} + try { dsize = fs.statSync(DBFILE + '.data').size; } catch {} + console.log(` cycle ${c}/${ROUNDS}: ops=${r.flat.length} acked=${r.acked} sent=${r.sentCount} kill=${r.killMode} prefix=${r.matched} docs=${base.docs.size} log=${(size / 1e6).toFixed(1)}MB data=${(dsize / 1e6).toFixed(0)}MB`); + } + } + } catch (err) { + if (err instanceof Fail) { + dumpArtifact(err, gen); + process.exit(1); + } + console.error(`\n✗ harness error: ${err.stack || err}`); + console.error(` server exited: ${JSON.stringify(exited)}`); + console.error(` serverLog: ${serverLog.slice(-2000)}`); + process.exit(1); + } finally { + cleanup(); + } + + console.log(`crash-fuzz: PASS — ${ROUNDS} crash/reopen cycles, prefix invariant held every time`); +} + +main().catch((e) => { console.error(e.stack || e); process.exit(1); });