Aleksey Shakhmatov
6c7f1f2e77
Black-box SIGKILL fuzzer for the M0 mmap+WAL crash story. Random write
workload through the official driver, kill -9 at a random point, reopen the
same log, verify the recovered state against an in-memory model:
- prefix invariant: recovered state == history[0..m) for some m in
[acked, sent]; every acked write durable, in-flight commands all-or-nothing
(group commit), nothing after them may survive
- always-opens (replay never refuses); index presence tied to the prefix and
find({k:v}) correctness (rebuild after replay); countDocuments
- unexpected server death (Zig panic, replay refusal) is a finding with the
server log; --verify-exec read-backs updates to separate execution bugs
from replay bugs; --no-kill for graceful-restart runs; --heavy passes a
1 MiB compact threshold to fuzz compaction/checkpoint windows
Deterministic via seeded PRNG; failures dump a repro artifact with the seed.
Mutation-checked: over-strict prefix check goes red on lost in-flight ops.
Observation recorded: small churn-heavy DBs can exhaust the pager's 64 GB
address-space reservation (data file grows in >=8 MiB compounding steps and
never shrinks without a rebuild), surfacing as DatabaseTooLarge on writes.
661 lines
24 KiB
JavaScript
661 lines
24 KiB
JavaScript
//!/usr/bin/env node
|
|
//
|
|
// Crash-consistency fuzzer for multiforadb (tests/fuzz/crash-fuzz.js).
|
|
//
|
|
// Black-box: drives the server through the official driver, kills it with
|
|
// SIGKILL at a random point, reopens the same log file, and verifies the
|
|
// recovered state against an in-memory model.
|
|
//
|
|
// The invariant under test (the "prefix invariant"):
|
|
// with one sequential client and fsync-before-ack commits, the state that
|
|
// survives a crash is exactly `apply(history[0..m))` for some m with
|
|
// acked <= m <= sent, where the in-flight command (if any) contributes a
|
|
// prefix of its documents. Every fully-acked write must be durable; an
|
|
// in-flight write may or may not be; nothing after it may be.
|
|
//
|
|
// A cycle is: generate random ops -> SIGKILL at a random point (sometimes
|
|
// mid-write, sometimes right after an ack) -> reopen -> verify prefix
|
|
// invariant + index correctness + count. All cycles share one log file, so
|
|
// checkpoints, log truncation and free-list reuse are exercised across
|
|
// cycles when the log grows past the checkpoint threshold (--heavy).
|
|
//
|
|
// Determinism: the op sequence and the kill decision come from a seeded PRNG,
|
|
// so `--seed N` reproduces the same scenario (kill timing inside the window
|
|
// is OS-scheduled, exactly as in any crash tester, e.g. RocksDB db_crashtest).
|
|
//
|
|
// node tests/fuzz/crash-fuzz.js # 60 quick cycles
|
|
// node tests/fuzz/crash-fuzz.js --heavy # grow log past the
|
|
// # checkpoint threshold
|
|
// node tests/fuzz/crash-fuzz.js --seed 1234 --rounds 200
|
|
//
|
|
// Env: CRASH_FUZZ_PORT listen port (default 27230)
|
|
// MFDB_BIN server binary (default ../../zig-out/bin/multiforadb)
|
|
// CRASH_FUZZ_DB log file (default ../../.zig-cache/crash-fuzz.log)
|
|
const { MongoClient } = require('../e2e/node_modules/mongodb');
|
|
const { spawn } = require('child_process');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// --------------------------------------------------------------------------
|
|
// argv / config
|
|
// --------------------------------------------------------------------------
|
|
const argv = process.argv.slice(2);
|
|
const args = {};
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (!a.startsWith('--')) continue;
|
|
const v = argv[i + 1];
|
|
args[a.slice(2)] = v !== undefined && !v.startsWith('--') ? v : true;
|
|
}
|
|
function intArg(k, dflt) {
|
|
if (args[k] === undefined || args[k] === true) return dflt;
|
|
const n = Number(args[k]);
|
|
if (!Number.isInteger(n) || n < 0) {
|
|
console.error(`bad --${k} value: ${args[k]}`);
|
|
process.exit(2);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
const HEAVY = !!args.heavy;
|
|
const PORT = Number(process.env.CRASH_FUZZ_PORT || args.port || 27230);
|
|
const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb');
|
|
const DBFILE = process.env.CRASH_FUZZ_DB || path.resolve(__dirname, '../../.zig-cache/crash-fuzz.log');
|
|
const VERBOSE = !!args.verbose;
|
|
const VERIFY_EXEC = !!args['verify-exec'];
|
|
const NO_KILL = !!args['no-kill'];
|
|
const KEEP_LOG = !!args['keep-log'];
|
|
const ROUNDS = intArg('rounds', HEAVY ? 8 : 60);
|
|
const SEED = args.seed !== undefined ? intArg('seed', 0) : ((Math.random() * 0xffffffff) >>> 0);
|
|
const KILL_PROB = HEAVY ? 0.3 : 0.22;
|
|
const KILL_INFLIGHT_PROB = 0.6; // of kill decisions, share that fire mid-write
|
|
const MAX_DOCS = intArg('max-docs', HEAVY ? 3000 : 400);
|
|
const BATCH_MAX = intArg('batch-max', HEAVY ? 400 : 120);
|
|
const MAX_CYCLE_OPS = intArg('max-cycle-ops', HEAVY ? 60 : 30);
|
|
const KILL_DELAY_MS = intArg('kill-delay-ms', 1);
|
|
const LONG_STR = HEAVY ? 4096 : 1024;
|
|
|
|
// --------------------------------------------------------------------------
|
|
// deterministic PRNG (mulberry32)
|
|
// --------------------------------------------------------------------------
|
|
function mulberry32(seed) {
|
|
let a = seed >>> 0;
|
|
return function () {
|
|
a |= 0;
|
|
a = (a + 0x6d2b79f5) | 0;
|
|
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
};
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// model: the intended database state, plus the flat op history of a cycle
|
|
// --------------------------------------------------------------------------
|
|
// Deep-clone a model doc: applyFlat mutates doc objects in place, and every
|
|
// candidate prefix state must be an independent copy (docs are JSON-safe:
|
|
// numbers, strings, bools, null, arrays, nested objects).
|
|
function cloneDoc(d) {
|
|
return JSON.parse(JSON.stringify(d));
|
|
}
|
|
|
|
class Model {
|
|
constructor(other) {
|
|
this.docs = new Map(); // _id -> doc (plain JS object)
|
|
this.indexCreated = false;
|
|
if (other) {
|
|
for (const [k, v] of other.docs) this.docs.set(k, cloneDoc(v));
|
|
this.indexCreated = other.indexCreated;
|
|
}
|
|
}
|
|
}
|
|
|
|
// A flat op: {kind:'insert', doc} | {kind:'update', id, patch} |
|
|
// {kind:'delete', id} | {kind:'createIndex'}
|
|
function applyFlat(docs, f) {
|
|
switch (f.kind) {
|
|
case 'insert':
|
|
// Clone: the same flat op is applied by applyOp (the live model) and by
|
|
// stateAfter (every candidate prefix). Without a clone, both mutate the
|
|
// shared genDoc object and an update would be applied twice.
|
|
docs.set(f.doc._id, cloneDoc(f.doc));
|
|
break;
|
|
case 'update': {
|
|
const d = docs.get(f.id);
|
|
if (!d) throw new Fail('model bug: update of missing doc', { f });
|
|
const p = f.patch;
|
|
if (p.$set) for (const [k, v] of Object.entries(p.$set)) d[k] = v;
|
|
if (p.$inc) for (const [k, v] of Object.entries(p.$inc)) d[k] = (d[k] || 0) + v;
|
|
if (p.$unset) for (const k of Object.keys(p.$unset)) delete d[k];
|
|
break;
|
|
}
|
|
case 'delete':
|
|
docs.delete(f.id);
|
|
break;
|
|
case 'createIndex':
|
|
break; // no doc effect
|
|
}
|
|
}
|
|
|
|
function stateAfter(base, flat, m) {
|
|
const docs = new Map();
|
|
for (const [k, v] of base.docs) docs.set(k, cloneDoc(v));
|
|
for (let i = 0; i < m; i++) applyFlat(docs, flat[i]);
|
|
return docs;
|
|
}
|
|
|
|
function applyOp(intended, op) {
|
|
if (op.kind === 'batch') {
|
|
for (const d of op.docs) intended.docs.set(d._id, cloneDoc(d));
|
|
} else {
|
|
applyFlat(intended.docs, op);
|
|
}
|
|
if (op.kind === 'createIndex') intended.indexCreated = true;
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// canonical comparison (driver-parsed BSON vs plain JS model docs)
|
|
// --------------------------------------------------------------------------
|
|
function canon(v) {
|
|
if (v === null || v === undefined) return 'null';
|
|
if (typeof v === 'number') return Number.isInteger(v) ? 'i' + v : 'd' + v;
|
|
if (typeof v === 'string') return 's' + v;
|
|
if (typeof v === 'boolean') return 'b' + v;
|
|
if (Array.isArray(v)) return '[' + v.map(canon).join(',') + ']';
|
|
if (typeof v === 'object') {
|
|
const ks = Object.keys(v).sort();
|
|
return '{' + ks.map((k) => canon(k) + ':' + canon(v[k])).join(',') + '}';
|
|
}
|
|
return '?' + String(v);
|
|
}
|
|
|
|
function sameDocs(a, b) {
|
|
if (a.size !== b.size) return false;
|
|
for (const [k, v] of a) {
|
|
if (!b.has(k)) return false;
|
|
if (canon(v) !== canon(b.get(k))) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// First few concrete differences between two doc maps (for diagnostics).
|
|
function diffState(a, b) {
|
|
const out = [];
|
|
for (const [k, v] of a) {
|
|
if (!b.has(k)) {
|
|
out.push(`db missing _id=${k}`);
|
|
if (out.length >= 5) return out;
|
|
continue;
|
|
}
|
|
if (canon(v) !== canon(b.get(k))) {
|
|
out.push(`_id=${k} differs: model=${canon(v)} db=${canon(b.get(k))}`);
|
|
if (out.length >= 5) return out;
|
|
}
|
|
}
|
|
for (const [k] of b) {
|
|
if (!a.has(k)) {
|
|
out.push(`db has extra _id=${k}`);
|
|
if (out.length >= 5) return out;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// random document / op generation
|
|
// --------------------------------------------------------------------------
|
|
function randString(rng) {
|
|
const lenRoll = rng();
|
|
let len;
|
|
if (lenRoll < 0.4) len = Math.floor(rng() * 32); // short
|
|
else if (lenRoll < 0.8) len = 32 + Math.floor(rng() * 128);
|
|
else len = 128 + Math.floor(rng() * LONG_STR); // long: makes batches slow
|
|
let s = '';
|
|
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \t\n"\'{}[]';
|
|
for (let i = 0; i < len; i++) s += chars[Math.floor(rng() * chars.length)];
|
|
return s;
|
|
}
|
|
|
|
function genDoc(rng, id, idx) {
|
|
const d = { _id: id, k: Math.floor(rng() * 10), n: Math.floor(rng() * 100000) };
|
|
const r = rng();
|
|
if (r < 0.35) d.s = randString(rng);
|
|
if (r < 0.5) d.nested = { a: Math.floor(rng() * 100), b: [Math.floor(rng() * 10), Math.floor(rng() * 10)], c: { d: rng() < 0.5 } };
|
|
if (r < 0.65) d.arr = [Math.floor(rng() * 5), randString(rng), rng() < 0.5];
|
|
if (r < 0.75) d.flag = rng() < 0.5;
|
|
if (r < 0.8) d.z = null;
|
|
if (idx !== undefined && rng() < 0.3) d.idx = idx; // occasionally tagged
|
|
return d;
|
|
}
|
|
|
|
function pickKey(rng, docs) {
|
|
const keys = [...docs.keys()];
|
|
return keys[Math.floor(rng() * keys.length)];
|
|
}
|
|
|
|
function genUpdate(rng, intended) {
|
|
const id = pickKey(rng, intended.docs);
|
|
const r = rng();
|
|
if (r < 0.35) return { kind: 'update', id, patch: { $set: { k: Math.floor(rng() * 10) } } };
|
|
if (r < 0.55) return { kind: 'update', id, patch: { $inc: { n: 1 } } };
|
|
if (r < 0.75) return { kind: 'update', id, patch: { $set: { s: randString(rng) } } };
|
|
if (r < 0.9) return { kind: 'update', id, patch: { $unset: { flag: '' } } };
|
|
return { kind: 'update', id, patch: { $set: { nested: { a: 7, b: [1, 2], c: { d: false } }, arr: ['x', 3] } } };
|
|
}
|
|
|
|
// Generator state: rng, nextId, thresholds, per-run switches.
|
|
function makeGen(seed, cfg) {
|
|
const rng = mulberry32(seed);
|
|
return {
|
|
rng,
|
|
nextId: 1,
|
|
idxThreshold: null, // flat count at which createIndex becomes durable, once generated
|
|
cfg,
|
|
nextOp(intended) {
|
|
const { rng, cfg } = this;
|
|
const size = intended.docs.size;
|
|
const r = rng();
|
|
if (!intended.indexCreated && r < 0.08) return { kind: 'createIndex' };
|
|
const overCap = size > cfg.maxDocs * 0.85;
|
|
let kind;
|
|
if (size === 0) {
|
|
kind = 'batch';
|
|
} else if (overCap) {
|
|
const x = rng();
|
|
kind = x < 0.55 ? 'delete' : x < 0.8 ? 'update' : 'batch';
|
|
} else {
|
|
const x = rng();
|
|
kind = x < 0.3 ? 'batch' : x < 0.42 ? 'insert' : x < 0.68 ? 'update' : 'delete';
|
|
}
|
|
switch (kind) {
|
|
case 'batch': {
|
|
const n = overCap ? 1 + Math.floor(rng() * 10) : 1 + Math.floor(rng() * cfg.batchMax);
|
|
const docs = [];
|
|
for (let i = 0; i < n; i++) docs.push(genDoc(rng, this.nextId++, i));
|
|
return { kind: 'batch', docs };
|
|
}
|
|
case 'insert':
|
|
return { kind: 'insert', doc: genDoc(rng, this.nextId++) };
|
|
case 'update':
|
|
return genUpdate(rng, intended);
|
|
case 'delete':
|
|
return { kind: 'delete', id: pickKey(rng, intended.docs) };
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// server control (mirrors tests/e2e/e2e6.js)
|
|
// --------------------------------------------------------------------------
|
|
let server = null;
|
|
let serverLog = '';
|
|
let exited = null; // {code, sig} once the child has exited
|
|
let killedByUs = false;
|
|
|
|
function cleanup() {
|
|
if (server && !exited) {
|
|
try { server.kill('SIGKILL'); } catch {}
|
|
}
|
|
}
|
|
process.on('exit', cleanup);
|
|
process.on('SIGINT', () => { if (args.debug) console.error('[harness SIGINT]'); cleanup(); process.exit(130); });
|
|
process.on('SIGTERM', () => { if (args.debug) console.error('[harness SIGTERM]'); cleanup(); process.exit(143); });
|
|
|
|
function checkUnexpectedDeath(where) {
|
|
// Our own SIGKILL is expected; anything else (Zig panic, replay refusing
|
|
// to open, SIGABRT...) is a finding.
|
|
if (exited && !(killedByUs && exited.sig === 'SIGKILL')) {
|
|
throw new Fail(`server died unexpectedly at ${where}`, {
|
|
code: exited.code,
|
|
sig: exited.sig,
|
|
logTail: serverLog.slice(-3000),
|
|
});
|
|
}
|
|
}
|
|
|
|
function startServer(fresh) {
|
|
return new Promise((resolve, reject) => {
|
|
if (fresh) {
|
|
fs.rmSync(DBFILE, { force: true });
|
|
fs.rmSync(DBFILE + '.data', { force: true });
|
|
}
|
|
serverLog = '';
|
|
exited = null;
|
|
killedByUs = false;
|
|
const extra = HEAVY ? ['--compact-threshold', String(1024 * 1024)] : [];
|
|
server = spawn(BIN, ['--port', String(PORT), '--db', DBFILE].concat(extra), {
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
server.stdout.on('data', (d) => (serverLog += d));
|
|
server.stderr.on('data', (d) => (serverLog += d));
|
|
server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`)));
|
|
server.on('exit', (code, sig) => {
|
|
exited = { code, sig };
|
|
if (args.debug) console.error(`[${Date.now()} server ${server?.pid} exited code=${code} sig=${sig} killedByUs=${killedByUs} graceful=${gracefulStop}]`);
|
|
server = null;
|
|
if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`;
|
|
});
|
|
const deadline = Date.now() + 20000;
|
|
(async () => {
|
|
while (Date.now() < deadline) {
|
|
if (exited) {
|
|
reject(new Error(`server child exited during start (port ${PORT} busy?)\n${serverLog}`));
|
|
return;
|
|
}
|
|
const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 1000 });
|
|
try {
|
|
await c.connect();
|
|
await c.db('admin').command({ ping: 1 });
|
|
await c.close();
|
|
return resolve();
|
|
} catch {
|
|
try { await c.close(); } catch {}
|
|
await sleep(100);
|
|
}
|
|
}
|
|
reject(new Error(`server did not come up on :${PORT}\n${serverLog}`));
|
|
})();
|
|
});
|
|
}
|
|
|
|
async function stopServerGracefully() {
|
|
gracefulStop = true;
|
|
if (server && !exited) {
|
|
const exitedP = new Promise((r) => server.once('exit', r));
|
|
if (args.debug) console.error(`[${Date.now()} graceful-stop targeting pid ${server.pid}]\n${new Error().stack.split('\n').slice(2, 5).join('\n')}`);
|
|
try { server.kill('SIGTERM'); } catch {}
|
|
await Promise.race([exitedP, sleep(5000)]);
|
|
}
|
|
server = null;
|
|
}
|
|
|
|
async function killServer() {
|
|
killedByUs = true;
|
|
if (!exited) {
|
|
const exitedP = new Promise((r) => {
|
|
if (exited) return r();
|
|
const h = () => r();
|
|
if (server) server.once('exit', h);
|
|
else r();
|
|
});
|
|
try { server.kill('SIGKILL'); } catch {}
|
|
await Promise.race([exitedP, sleep(5000)]);
|
|
}
|
|
checkUnexpectedDeath('kill');
|
|
server = null;
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// client helpers
|
|
// --------------------------------------------------------------------------
|
|
async function connectClient() {
|
|
const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, {
|
|
serverSelectionTimeoutMS: 5000,
|
|
retryWrites: false,
|
|
});
|
|
await c.connect();
|
|
return c;
|
|
}
|
|
|
|
async function closeClient(c) {
|
|
if (!c) return;
|
|
await Promise.race([c.close().catch(() => {}), sleep(2000)]).catch(() => {});
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// one cycle: random ops then SIGKILL
|
|
// --------------------------------------------------------------------------
|
|
async function execOp(coll, op) {
|
|
switch (op.kind) {
|
|
case 'insert':
|
|
return coll.insertOne(op.doc);
|
|
case 'batch':
|
|
return coll.insertMany(op.docs);
|
|
case 'update':
|
|
return coll.updateOne({ _id: op.id }, op.patch);
|
|
case 'delete':
|
|
return coll.deleteOne({ _id: op.id });
|
|
case 'createIndex':
|
|
return coll.createIndex({ k: 1 });
|
|
}
|
|
}
|
|
|
|
async function runCycle(client, base, gen) {
|
|
const coll = client.db('fuzz').collection('c');
|
|
const intended = new Model(base);
|
|
const flat = [];
|
|
let acked = 0;
|
|
let killMode = null;
|
|
let matched = null;
|
|
let idxPos = null; // flat count at which this cycle's createIndex becomes durable
|
|
|
|
for (let i = 0; i < gen.cfg.maxCycleOps; i++) {
|
|
const kill = gen.rng() < gen.cfg.killProb || i === gen.cfg.maxCycleOps - 1;
|
|
const op = gen.nextOp(intended);
|
|
const inflight = !NO_KILL && kill && gen.rng() < KILL_INFLIGHT_PROB;
|
|
|
|
if (op.kind === 'batch') {
|
|
for (const d of op.docs) flat.push({ kind: 'insert', doc: d });
|
|
} else {
|
|
flat.push(op);
|
|
}
|
|
if (op.kind === 'createIndex') {
|
|
idxPos = flat.length; // flat count at which this cycle's record is durable
|
|
}
|
|
const sent = flat.length;
|
|
|
|
if (inflight) {
|
|
const p = execOp(coll, op);
|
|
p.catch(() => {}); // the coming SIGKILL may kill the request: expected
|
|
await sleep(gen.cfg.killDelayMs);
|
|
await killServer();
|
|
killMode = 'inflight';
|
|
applyOp(intended, op); // optimistic; the prefix check decides what survived
|
|
} else {
|
|
await execOp(coll, op);
|
|
await sleep(5); // let a self-crash's exit event deliver before we check
|
|
checkUnexpectedDeath(`op ${flat.length} of cycle`);
|
|
acked = sent;
|
|
applyOp(intended, op);
|
|
// Diagnostic: read the doc back and compare with the model. Pins down
|
|
// whether a divergence is an execution bug (in-memory wrong) or a
|
|
// log/replay bug (in-memory right, lost after reopen). Must run while
|
|
// the server is still alive -- after the stop below the query would
|
|
// fail with ECONNREFUSED.
|
|
if (VERIFY_EXEC && op.kind === 'update') {
|
|
const back = await coll.findOne({ _id: op.id });
|
|
const want = intended.docs.get(op.id);
|
|
if (!back || canon(back) !== canon(want)) {
|
|
throw new Fail(`cycle exec mismatch: update ${op.id} executed wrong`, {
|
|
back: back ? canon(back) : null,
|
|
want: canon(want),
|
|
});
|
|
}
|
|
}
|
|
if (kill) {
|
|
if (NO_KILL) {
|
|
// --no-kill: verify durability of the run's writes via a graceful
|
|
// stop + reopen instead of SIGKILL.
|
|
await stopServerGracefully();
|
|
} else {
|
|
await killServer();
|
|
}
|
|
killMode = 'after';
|
|
}
|
|
}
|
|
if (kill) break;
|
|
}
|
|
if (killMode === null) killMode = 'after'; // unreachable: last op forces a kill
|
|
return { flat, acked, sentCount: flat.length, killMode, matched, idxPos };
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// verification after reopen: the prefix invariant
|
|
// --------------------------------------------------------------------------
|
|
async function verify(client, base, r, cycleNo) {
|
|
const db = client.db('fuzz');
|
|
const coll = db.collection('c');
|
|
const dbDocs = await coll.find({}, { sort: { _id: 1 } }).toArray();
|
|
const dbMap = new Map(dbDocs.map((d) => [d._id, d]));
|
|
const dbIndexes = await coll.indexes();
|
|
const indexExists = dbIndexes.some((i) => i.name === 'k_1');
|
|
if (VERBOSE) {
|
|
console.log(` [verify] indexes=${JSON.stringify(dbIndexes.map((i) => i.name))} baseIdx=${base.indexCreated} idxPos=${r.idxPos}`);
|
|
}
|
|
|
|
let matched = null;
|
|
const diag = [];
|
|
for (let m = r.acked; m <= r.sentCount; m++) {
|
|
const st = stateAfter(base, r.flat, m);
|
|
const wantIdx = base.indexCreated || (r.idxPos !== null && m >= r.idxPos);
|
|
const stateOk = sameDocs(st, dbMap);
|
|
const indexOk = indexExists === wantIdx;
|
|
if (stateOk && indexOk) { matched = m; break; }
|
|
diag.push({
|
|
m,
|
|
size: st.size,
|
|
stateOk,
|
|
indexOk,
|
|
diff: stateOk ? [] : diffState(st, dbMap),
|
|
});
|
|
}
|
|
|
|
if (matched === null) {
|
|
throw new Fail(`cycle ${cycleNo}: recovered state is not a valid prefix`, {
|
|
cycleNo,
|
|
acked: r.acked,
|
|
sent: r.sentCount,
|
|
killMode: r.killMode,
|
|
baseIdx: base.indexCreated,
|
|
idxPos: r.idxPos,
|
|
indexExists,
|
|
dbIndexes: dbIndexes.map((i) => i.name),
|
|
dbSize: dbMap.size,
|
|
candidates: diag,
|
|
serverLog: serverLog.slice(-4000),
|
|
ops: r.flat.map((f) =>
|
|
f.kind === 'insert' ? `ins ${f.doc._id}` :
|
|
f.kind === 'update' ? `upd ${f.id} ${JSON.stringify(f.patch)}` :
|
|
f.kind === 'delete' ? `del ${f.id}` : 'idx'),
|
|
});
|
|
}
|
|
|
|
// Index / filter correctness at the matched prefix: find({k:v}) must return
|
|
// exactly the docs with k==v, whether the index exists (candidates+reapply)
|
|
// or the query was a full scan.
|
|
const st = stateAfter(base, r.flat, matched);
|
|
for (let v = 0; v < 10; v++) {
|
|
const got = (await coll.find({ k: v }, { sort: { _id: 1 } }).toArray())
|
|
.map((d) => canon(d))
|
|
.sort();
|
|
const want = [...st.values()]
|
|
.filter((d) => d.k === v)
|
|
.map((d) => canon(d))
|
|
.sort();
|
|
if (got.length !== want.length || got.some((c, i) => c !== want[i])) {
|
|
throw new Fail(`cycle ${cycleNo}: find({k:${v}}) mismatch at prefix ${matched}`, {
|
|
cycleNo, v, matched, got, want,
|
|
});
|
|
}
|
|
}
|
|
const cnt = await coll.countDocuments({});
|
|
if (cnt !== st.size) {
|
|
throw new Fail(`cycle ${cycleNo}: countDocuments mismatch at prefix ${matched}`, {
|
|
cycleNo, matched, count: cnt, expected: st.size,
|
|
});
|
|
}
|
|
r.matched = matched;
|
|
return { docs: st, indexCreated: indexExists };
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// failure artifacts
|
|
// --------------------------------------------------------------------------
|
|
class Fail extends Error {
|
|
constructor(msg, detail) { super(msg); this.detail = detail; }
|
|
}
|
|
|
|
function dumpArtifact(err, gen) {
|
|
const art = {
|
|
seed: SEED,
|
|
mode: HEAVY ? 'heavy' : 'default',
|
|
rounds: ROUNDS,
|
|
error: err.message,
|
|
detail: err.detail,
|
|
nextId: gen.nextId,
|
|
};
|
|
const file = `/tmp/crash-fuzz-fail-${SEED}.json`;
|
|
fs.writeFileSync(file, JSON.stringify(art, null, 2));
|
|
console.error(`\n✗ ${err.message}`);
|
|
console.error(` detail: ${JSON.stringify(err.detail, null, 2).slice(0, 4000)}`);
|
|
console.error(` artifact: ${file}`);
|
|
console.error(` rerun: node tests/fuzz/crash-fuzz.js --seed ${SEED} --rounds ${ROUNDS}${HEAVY ? ' --heavy' : ''}`);
|
|
}
|
|
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
// --------------------------------------------------------------------------
|
|
// main
|
|
// --------------------------------------------------------------------------
|
|
async function main() {
|
|
console.log(`crash-fuzz: seed=${SEED} rounds=${ROUNDS} mode=${HEAVY ? 'heavy' : 'default'} port=${PORT}`);
|
|
console.log(` bin=${BIN}`);
|
|
console.log(` db=${DBFILE}`);
|
|
|
|
const gen = makeGen(SEED, { maxDocs: MAX_DOCS, batchMax: BATCH_MAX, maxCycleOps: MAX_CYCLE_OPS, killProb: KILL_PROB, killDelayMs: KILL_DELAY_MS });
|
|
|
|
if (!KEEP_LOG) {
|
|
fs.rmSync(DBFILE, { force: true });
|
|
fs.rmSync(DBFILE + '.data', { force: true }); // stale data files hit the 64GB reservation ceiling
|
|
}
|
|
await startServer(true);
|
|
let base = new Model();
|
|
|
|
if (args.debug) {
|
|
const iv = setInterval(() => {
|
|
if (server) console.error(`[${Date.now()} watch server pid=${server.pid} exited=${JSON.stringify(exited)}]`);
|
|
else console.error(`[${Date.now()} watch server=null exited=${JSON.stringify(exited)}]`);
|
|
}, 100);
|
|
iv.unref();
|
|
}
|
|
|
|
try {
|
|
for (let c = 1; c <= ROUNDS; c++) {
|
|
let client = await connectClient();
|
|
const r = await runCycle(client, base, gen); // ends with the server killed
|
|
await closeClient(client);
|
|
|
|
await startServer(false); // reopen: replay must never refuse
|
|
client = await connectClient();
|
|
const v = await verify(client, base, r, c);
|
|
await closeClient(client);
|
|
base = new Model();
|
|
for (const [k, d] of v.docs) base.docs.set(k, d);
|
|
base.indexCreated = v.indexCreated;
|
|
|
|
if (VERBOSE || c === 1 || c % 10 === 0 || c === ROUNDS) {
|
|
let size = 0, dsize = 0;
|
|
try { size = fs.statSync(DBFILE).size; } catch {}
|
|
try { dsize = fs.statSync(DBFILE + '.data').size; } catch {}
|
|
console.log(` cycle ${c}/${ROUNDS}: ops=${r.flat.length} acked=${r.acked} sent=${r.sentCount} kill=${r.killMode} prefix=${r.matched} docs=${base.docs.size} log=${(size / 1e6).toFixed(1)}MB data=${(dsize / 1e6).toFixed(0)}MB`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
if (err instanceof Fail) {
|
|
dumpArtifact(err, gen);
|
|
process.exit(1);
|
|
}
|
|
console.error(`\n✗ harness error: ${err.stack || err}`);
|
|
console.error(` server exited: ${JSON.stringify(exited)}`);
|
|
console.error(` serverLog: ${serverLog.slice(-2000)}`);
|
|
process.exit(1);
|
|
} finally {
|
|
cleanup();
|
|
}
|
|
|
|
console.log(`crash-fuzz: PASS — ${ROUNDS} crash/reopen cycles, prefix invariant held every time`);
|
|
}
|
|
|
|
main().catch((e) => { console.error(e.stack || e); process.exit(1); });
|