//!/usr/bin/env node // // Crash-consistency fuzzer for multiforadb (tests/fuzz/crash-fuzz.js). // // Black-box: drives the server through the official driver, kills it with // SIGKILL at a random point, reopens the same log file, and verifies the // recovered state against an in-memory model. // // The invariant under test (the "prefix invariant"): // with one sequential client and fsync-before-ack commits, the state that // survives a crash is exactly `apply(history[0..m))` for some m with // acked <= m <= sent, where the in-flight command (if any) contributes a // prefix of its documents. Every fully-acked write must be durable; an // in-flight write may or may not be; nothing after it may be. // // A cycle is: generate random ops -> SIGKILL at a random point (sometimes // mid-write, sometimes right after an ack) -> reopen -> verify prefix // invariant + index correctness + count. All cycles share one log file, so // checkpoints, log truncation and free-list reuse are exercised across // cycles when the log grows past the checkpoint threshold (--heavy). // // Determinism: the op sequence and the kill decision come from a seeded PRNG, // so `--seed N` reproduces the same scenario (kill timing inside the window // is OS-scheduled, exactly as in any crash tester, e.g. RocksDB db_crashtest). // // node tests/fuzz/crash-fuzz.js # 60 quick cycles // node tests/fuzz/crash-fuzz.js --heavy # grow log past the // # checkpoint threshold // node tests/fuzz/crash-fuzz.js --seed 1234 --rounds 200 // // Env: CRASH_FUZZ_PORT listen port (default 27230) // MFDB_BIN server binary (default ../../zig-out/bin/multiforadb) // CRASH_FUZZ_DB log file (default ../../.zig-cache/crash-fuzz.log) const { MongoClient } = require('../e2e/node_modules/mongodb'); const { spawn } = require('child_process'); const fs = require('fs'); const path = require('path'); // -------------------------------------------------------------------------- // argv / config // -------------------------------------------------------------------------- const argv = process.argv.slice(2); const args = {}; for (let i = 0; i < argv.length; i++) { const a = argv[i]; if (!a.startsWith('--')) continue; const v = argv[i + 1]; args[a.slice(2)] = v !== undefined && !v.startsWith('--') ? v : true; } function intArg(k, dflt) { if (args[k] === undefined || args[k] === true) return dflt; const n = Number(args[k]); if (!Number.isInteger(n) || n < 0) { console.error(`bad --${k} value: ${args[k]}`); process.exit(2); } return n; } const HEAVY = !!args.heavy; const PORT = Number(process.env.CRASH_FUZZ_PORT || args.port || 27230); const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb'); const DBFILE = process.env.CRASH_FUZZ_DB || path.resolve(__dirname, '../../.zig-cache/crash-fuzz.log'); const VERBOSE = !!args.verbose; const VERIFY_EXEC = !!args['verify-exec']; const NO_KILL = !!args['no-kill']; const KEEP_LOG = !!args['keep-log']; const ROUNDS = intArg('rounds', HEAVY ? 8 : 60); const SEED = args.seed !== undefined ? intArg('seed', 0) : ((Math.random() * 0xffffffff) >>> 0); const KILL_PROB = HEAVY ? 0.3 : 0.22; const KILL_INFLIGHT_PROB = 0.6; // of kill decisions, share that fire mid-write const MAX_DOCS = intArg('max-docs', HEAVY ? 3000 : 400); const BATCH_MAX = intArg('batch-max', HEAVY ? 400 : 120); const MAX_CYCLE_OPS = intArg('max-cycle-ops', HEAVY ? 60 : 30); const KILL_DELAY_MS = intArg('kill-delay-ms', 1); const LONG_STR = HEAVY ? 4096 : 1024; // -------------------------------------------------------------------------- // deterministic PRNG (mulberry32) // -------------------------------------------------------------------------- function mulberry32(seed) { let a = seed >>> 0; return function () { a |= 0; a = (a + 0x6d2b79f5) | 0; let t = Math.imul(a ^ (a >>> 15), 1 | a); t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; return ((t ^ (t >>> 14)) >>> 0) / 4294967296; }; } // -------------------------------------------------------------------------- // model: the intended database state, plus the flat op history of a cycle // -------------------------------------------------------------------------- // Deep-clone a model doc: applyFlat mutates doc objects in place, and every // candidate prefix state must be an independent copy (docs are JSON-safe: // numbers, strings, bools, null, arrays, nested objects). function cloneDoc(d) { return JSON.parse(JSON.stringify(d)); } class Model { constructor(other) { this.docs = new Map(); // _id -> doc (plain JS object) this.indexCreated = false; if (other) { for (const [k, v] of other.docs) this.docs.set(k, cloneDoc(v)); this.indexCreated = other.indexCreated; } } } // A flat op: {kind:'insert', doc} | {kind:'update', id, patch} | // {kind:'delete', id} | {kind:'createIndex'} function applyFlat(docs, f) { switch (f.kind) { case 'insert': // Clone: the same flat op is applied by applyOp (the live model) and by // stateAfter (every candidate prefix). Without a clone, both mutate the // shared genDoc object and an update would be applied twice. docs.set(f.doc._id, cloneDoc(f.doc)); break; case 'update': { const d = docs.get(f.id); if (!d) throw new Fail('model bug: update of missing doc', { f }); const p = f.patch; if (p.$set) for (const [k, v] of Object.entries(p.$set)) d[k] = v; if (p.$inc) for (const [k, v] of Object.entries(p.$inc)) d[k] = (d[k] || 0) + v; if (p.$unset) for (const k of Object.keys(p.$unset)) delete d[k]; break; } case 'delete': docs.delete(f.id); break; case 'createIndex': break; // no doc effect } } function stateAfter(base, flat, m) { const docs = new Map(); for (const [k, v] of base.docs) docs.set(k, cloneDoc(v)); for (let i = 0; i < m; i++) applyFlat(docs, flat[i]); return docs; } function applyOp(intended, op) { if (op.kind === 'batch') { for (const d of op.docs) intended.docs.set(d._id, cloneDoc(d)); } else { applyFlat(intended.docs, op); } if (op.kind === 'createIndex') intended.indexCreated = true; } // -------------------------------------------------------------------------- // canonical comparison (driver-parsed BSON vs plain JS model docs) // -------------------------------------------------------------------------- function canon(v) { if (v === null || v === undefined) return 'null'; if (typeof v === 'number') return Number.isInteger(v) ? 'i' + v : 'd' + v; if (typeof v === 'string') return 's' + v; if (typeof v === 'boolean') return 'b' + v; if (Array.isArray(v)) return '[' + v.map(canon).join(',') + ']'; if (typeof v === 'object') { const ks = Object.keys(v).sort(); return '{' + ks.map((k) => canon(k) + ':' + canon(v[k])).join(',') + '}'; } return '?' + String(v); } function sameDocs(a, b) { if (a.size !== b.size) return false; for (const [k, v] of a) { if (!b.has(k)) return false; if (canon(v) !== canon(b.get(k))) return false; } return true; } // First few concrete differences between two doc maps (for diagnostics). function diffState(a, b) { const out = []; for (const [k, v] of a) { if (!b.has(k)) { out.push(`db missing _id=${k}`); if (out.length >= 5) return out; continue; } if (canon(v) !== canon(b.get(k))) { out.push(`_id=${k} differs: model=${canon(v)} db=${canon(b.get(k))}`); if (out.length >= 5) return out; } } for (const [k] of b) { if (!a.has(k)) { out.push(`db has extra _id=${k}`); if (out.length >= 5) return out; } } return out; } // -------------------------------------------------------------------------- // random document / op generation // -------------------------------------------------------------------------- function randString(rng) { const lenRoll = rng(); let len; if (lenRoll < 0.4) len = Math.floor(rng() * 32); // short else if (lenRoll < 0.8) len = 32 + Math.floor(rng() * 128); else len = 128 + Math.floor(rng() * LONG_STR); // long: makes batches slow let s = ''; const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \t\n"\'{}[]'; for (let i = 0; i < len; i++) s += chars[Math.floor(rng() * chars.length)]; return s; } function genDoc(rng, id, idx) { const d = { _id: id, k: Math.floor(rng() * 10), n: Math.floor(rng() * 100000) }; const r = rng(); if (r < 0.35) d.s = randString(rng); if (r < 0.5) d.nested = { a: Math.floor(rng() * 100), b: [Math.floor(rng() * 10), Math.floor(rng() * 10)], c: { d: rng() < 0.5 } }; if (r < 0.65) d.arr = [Math.floor(rng() * 5), randString(rng), rng() < 0.5]; if (r < 0.75) d.flag = rng() < 0.5; if (r < 0.8) d.z = null; if (idx !== undefined && rng() < 0.3) d.idx = idx; // occasionally tagged return d; } function pickKey(rng, docs) { const keys = [...docs.keys()]; return keys[Math.floor(rng() * keys.length)]; } function genUpdate(rng, intended) { const id = pickKey(rng, intended.docs); const r = rng(); if (r < 0.35) return { kind: 'update', id, patch: { $set: { k: Math.floor(rng() * 10) } } }; if (r < 0.55) return { kind: 'update', id, patch: { $inc: { n: 1 } } }; if (r < 0.75) return { kind: 'update', id, patch: { $set: { s: randString(rng) } } }; if (r < 0.9) return { kind: 'update', id, patch: { $unset: { flag: '' } } }; return { kind: 'update', id, patch: { $set: { nested: { a: 7, b: [1, 2], c: { d: false } }, arr: ['x', 3] } } }; } // Generator state: rng, nextId, thresholds, per-run switches. function makeGen(seed, cfg) { const rng = mulberry32(seed); return { rng, nextId: 1, idxThreshold: null, // flat count at which createIndex becomes durable, once generated cfg, nextOp(intended) { const { rng, cfg } = this; const size = intended.docs.size; const r = rng(); if (!intended.indexCreated && r < 0.08) return { kind: 'createIndex' }; const overCap = size > cfg.maxDocs * 0.85; let kind; if (size === 0) { kind = 'batch'; } else if (overCap) { const x = rng(); kind = x < 0.55 ? 'delete' : x < 0.8 ? 'update' : 'batch'; } else { const x = rng(); kind = x < 0.3 ? 'batch' : x < 0.42 ? 'insert' : x < 0.68 ? 'update' : 'delete'; } switch (kind) { case 'batch': { const n = overCap ? 1 + Math.floor(rng() * 10) : 1 + Math.floor(rng() * cfg.batchMax); const docs = []; for (let i = 0; i < n; i++) docs.push(genDoc(rng, this.nextId++, i)); return { kind: 'batch', docs }; } case 'insert': return { kind: 'insert', doc: genDoc(rng, this.nextId++) }; case 'update': return genUpdate(rng, intended); case 'delete': return { kind: 'delete', id: pickKey(rng, intended.docs) }; } }, }; } // -------------------------------------------------------------------------- // server control (mirrors tests/e2e/e2e6.js) // -------------------------------------------------------------------------- let server = null; let serverLog = ''; let exited = null; // {code, sig} once the child has exited let killedByUs = false; function cleanup() { if (server && !exited) { try { server.kill('SIGKILL'); } catch {} } } process.on('exit', cleanup); process.on('SIGINT', () => { if (args.debug) console.error('[harness SIGINT]'); cleanup(); process.exit(130); }); process.on('SIGTERM', () => { if (args.debug) console.error('[harness SIGTERM]'); cleanup(); process.exit(143); }); function checkUnexpectedDeath(where) { // Our own SIGKILL is expected; anything else (Zig panic, replay refusing // to open, SIGABRT...) is a finding. if (exited && !(killedByUs && exited.sig === 'SIGKILL')) { throw new Fail(`server died unexpectedly at ${where}`, { code: exited.code, sig: exited.sig, logTail: serverLog.slice(-3000), }); } } function startServer(fresh) { return new Promise((resolve, reject) => { if (fresh) { fs.rmSync(DBFILE, { force: true }); fs.rmSync(DBFILE + '.data', { force: true }); } serverLog = ''; exited = null; killedByUs = false; const extra = HEAVY ? ['--compact-threshold', String(1024 * 1024)] : []; server = spawn(BIN, ['--port', String(PORT), '--db', DBFILE].concat(extra), { stdio: ['ignore', 'pipe', 'pipe'], }); server.stdout.on('data', (d) => (serverLog += d)); server.stderr.on('data', (d) => (serverLog += d)); server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`))); server.on('exit', (code, sig) => { exited = { code, sig }; if (args.debug) console.error(`[${Date.now()} server ${server?.pid} exited code=${code} sig=${sig} killedByUs=${killedByUs} graceful=${gracefulStop}]`); server = null; if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`; }); const deadline = Date.now() + 20000; (async () => { while (Date.now() < deadline) { if (exited) { reject(new Error(`server child exited during start (port ${PORT} busy?)\n${serverLog}`)); return; } const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 1000 }); try { await c.connect(); await c.db('admin').command({ ping: 1 }); await c.close(); return resolve(); } catch { try { await c.close(); } catch {} await sleep(100); } } reject(new Error(`server did not come up on :${PORT}\n${serverLog}`)); })(); }); } async function stopServerGracefully() { gracefulStop = true; if (server && !exited) { const exitedP = new Promise((r) => server.once('exit', r)); if (args.debug) console.error(`[${Date.now()} graceful-stop targeting pid ${server.pid}]\n${new Error().stack.split('\n').slice(2, 5).join('\n')}`); try { server.kill('SIGTERM'); } catch {} await Promise.race([exitedP, sleep(5000)]); } server = null; } async function killServer() { killedByUs = true; if (!exited) { const exitedP = new Promise((r) => { if (exited) return r(); const h = () => r(); if (server) server.once('exit', h); else r(); }); try { server.kill('SIGKILL'); } catch {} await Promise.race([exitedP, sleep(5000)]); } checkUnexpectedDeath('kill'); server = null; } // -------------------------------------------------------------------------- // client helpers // -------------------------------------------------------------------------- async function connectClient() { const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 5000, retryWrites: false, }); await c.connect(); return c; } async function closeClient(c) { if (!c) return; await Promise.race([c.close().catch(() => {}), sleep(2000)]).catch(() => {}); } // -------------------------------------------------------------------------- // one cycle: random ops then SIGKILL // -------------------------------------------------------------------------- async function execOp(coll, op) { switch (op.kind) { case 'insert': return coll.insertOne(op.doc); case 'batch': return coll.insertMany(op.docs); case 'update': return coll.updateOne({ _id: op.id }, op.patch); case 'delete': return coll.deleteOne({ _id: op.id }); case 'createIndex': return coll.createIndex({ k: 1 }); } } async function runCycle(client, base, gen) { const coll = client.db('fuzz').collection('c'); const intended = new Model(base); const flat = []; let acked = 0; let killMode = null; let matched = null; let idxPos = null; // flat count at which this cycle's createIndex becomes durable for (let i = 0; i < gen.cfg.maxCycleOps; i++) { const kill = gen.rng() < gen.cfg.killProb || i === gen.cfg.maxCycleOps - 1; const op = gen.nextOp(intended); const inflight = !NO_KILL && kill && gen.rng() < KILL_INFLIGHT_PROB; if (op.kind === 'batch') { for (const d of op.docs) flat.push({ kind: 'insert', doc: d }); } else { flat.push(op); } if (op.kind === 'createIndex') { idxPos = flat.length; // flat count at which this cycle's record is durable } const sent = flat.length; if (inflight) { const p = execOp(coll, op); p.catch(() => {}); // the coming SIGKILL may kill the request: expected await sleep(gen.cfg.killDelayMs); await killServer(); killMode = 'inflight'; applyOp(intended, op); // optimistic; the prefix check decides what survived } else { await execOp(coll, op); await sleep(5); // let a self-crash's exit event deliver before we check checkUnexpectedDeath(`op ${flat.length} of cycle`); acked = sent; applyOp(intended, op); // Diagnostic: read the doc back and compare with the model. Pins down // whether a divergence is an execution bug (in-memory wrong) or a // log/replay bug (in-memory right, lost after reopen). Must run while // the server is still alive -- after the stop below the query would // fail with ECONNREFUSED. if (VERIFY_EXEC && op.kind === 'update') { const back = await coll.findOne({ _id: op.id }); const want = intended.docs.get(op.id); if (!back || canon(back) !== canon(want)) { throw new Fail(`cycle exec mismatch: update ${op.id} executed wrong`, { back: back ? canon(back) : null, want: canon(want), }); } } if (kill) { if (NO_KILL) { // --no-kill: verify durability of the run's writes via a graceful // stop + reopen instead of SIGKILL. await stopServerGracefully(); } else { await killServer(); } killMode = 'after'; } } if (kill) break; } if (killMode === null) killMode = 'after'; // unreachable: last op forces a kill return { flat, acked, sentCount: flat.length, killMode, matched, idxPos }; } // -------------------------------------------------------------------------- // verification after reopen: the prefix invariant // -------------------------------------------------------------------------- async function verify(client, base, r, cycleNo) { const db = client.db('fuzz'); const coll = db.collection('c'); const dbDocs = await coll.find({}, { sort: { _id: 1 } }).toArray(); const dbMap = new Map(dbDocs.map((d) => [d._id, d])); const dbIndexes = await coll.indexes(); const indexExists = dbIndexes.some((i) => i.name === 'k_1'); if (VERBOSE) { console.log(` [verify] indexes=${JSON.stringify(dbIndexes.map((i) => i.name))} baseIdx=${base.indexCreated} idxPos=${r.idxPos}`); } let matched = null; const diag = []; for (let m = r.acked; m <= r.sentCount; m++) { const st = stateAfter(base, r.flat, m); const wantIdx = base.indexCreated || (r.idxPos !== null && m >= r.idxPos); const stateOk = sameDocs(st, dbMap); const indexOk = indexExists === wantIdx; if (stateOk && indexOk) { matched = m; break; } diag.push({ m, size: st.size, stateOk, indexOk, diff: stateOk ? [] : diffState(st, dbMap), }); } if (matched === null) { throw new Fail(`cycle ${cycleNo}: recovered state is not a valid prefix`, { cycleNo, acked: r.acked, sent: r.sentCount, killMode: r.killMode, baseIdx: base.indexCreated, idxPos: r.idxPos, indexExists, dbIndexes: dbIndexes.map((i) => i.name), dbSize: dbMap.size, candidates: diag, serverLog: serverLog.slice(-4000), ops: r.flat.map((f) => f.kind === 'insert' ? `ins ${f.doc._id}` : f.kind === 'update' ? `upd ${f.id} ${JSON.stringify(f.patch)}` : f.kind === 'delete' ? `del ${f.id}` : 'idx'), }); } // Index / filter correctness at the matched prefix: find({k:v}) must return // exactly the docs with k==v, whether the index exists (candidates+reapply) // or the query was a full scan. const st = stateAfter(base, r.flat, matched); for (let v = 0; v < 10; v++) { const got = (await coll.find({ k: v }, { sort: { _id: 1 } }).toArray()) .map((d) => canon(d)) .sort(); const want = [...st.values()] .filter((d) => d.k === v) .map((d) => canon(d)) .sort(); if (got.length !== want.length || got.some((c, i) => c !== want[i])) { throw new Fail(`cycle ${cycleNo}: find({k:${v}}) mismatch at prefix ${matched}`, { cycleNo, v, matched, got, want, }); } } const cnt = await coll.countDocuments({}); if (cnt !== st.size) { throw new Fail(`cycle ${cycleNo}: countDocuments mismatch at prefix ${matched}`, { cycleNo, matched, count: cnt, expected: st.size, }); } r.matched = matched; return { docs: st, indexCreated: indexExists }; } // -------------------------------------------------------------------------- // failure artifacts // -------------------------------------------------------------------------- class Fail extends Error { constructor(msg, detail) { super(msg); this.detail = detail; } } function dumpArtifact(err, gen) { const art = { seed: SEED, mode: HEAVY ? 'heavy' : 'default', rounds: ROUNDS, error: err.message, detail: err.detail, nextId: gen.nextId, }; const file = `/tmp/crash-fuzz-fail-${SEED}.json`; fs.writeFileSync(file, JSON.stringify(art, null, 2)); console.error(`\nāœ— ${err.message}`); console.error(` detail: ${JSON.stringify(err.detail, null, 2).slice(0, 4000)}`); console.error(` artifact: ${file}`); console.error(` rerun: node tests/fuzz/crash-fuzz.js --seed ${SEED} --rounds ${ROUNDS}${HEAVY ? ' --heavy' : ''}`); } const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); // -------------------------------------------------------------------------- // main // -------------------------------------------------------------------------- async function main() { console.log(`crash-fuzz: seed=${SEED} rounds=${ROUNDS} mode=${HEAVY ? 'heavy' : 'default'} port=${PORT}`); console.log(` bin=${BIN}`); console.log(` db=${DBFILE}`); const gen = makeGen(SEED, { maxDocs: MAX_DOCS, batchMax: BATCH_MAX, maxCycleOps: MAX_CYCLE_OPS, killProb: KILL_PROB, killDelayMs: KILL_DELAY_MS }); if (!KEEP_LOG) { fs.rmSync(DBFILE, { force: true }); fs.rmSync(DBFILE + '.data', { force: true }); // stale data files hit the 64GB reservation ceiling } await startServer(true); let base = new Model(); if (args.debug) { const iv = setInterval(() => { if (server) console.error(`[${Date.now()} watch server pid=${server.pid} exited=${JSON.stringify(exited)}]`); else console.error(`[${Date.now()} watch server=null exited=${JSON.stringify(exited)}]`); }, 100); iv.unref(); } try { for (let c = 1; c <= ROUNDS; c++) { let client = await connectClient(); const r = await runCycle(client, base, gen); // ends with the server killed await closeClient(client); await startServer(false); // reopen: replay must never refuse client = await connectClient(); const v = await verify(client, base, r, c); await closeClient(client); base = new Model(); for (const [k, d] of v.docs) base.docs.set(k, d); base.indexCreated = v.indexCreated; if (VERBOSE || c === 1 || c % 10 === 0 || c === ROUNDS) { let size = 0, dsize = 0; try { size = fs.statSync(DBFILE).size; } catch {} try { dsize = fs.statSync(DBFILE + '.data').size; } catch {} console.log(` cycle ${c}/${ROUNDS}: ops=${r.flat.length} acked=${r.acked} sent=${r.sentCount} kill=${r.killMode} prefix=${r.matched} docs=${base.docs.size} log=${(size / 1e6).toFixed(1)}MB data=${(dsize / 1e6).toFixed(0)}MB`); } } } catch (err) { if (err instanceof Fail) { dumpArtifact(err, gen); process.exit(1); } console.error(`\nāœ— harness error: ${err.stack || err}`); console.error(` server exited: ${JSON.stringify(exited)}`); console.error(` serverLog: ${serverLog.slice(-2000)}`); process.exit(1); } finally { cleanup(); } console.log(`crash-fuzz: PASS — ${ROUNDS} crash/reopen cycles, prefix invariant held every time`); } main().catch((e) => { console.error(e.stack || e); process.exit(1); });