Aleksey Shakhmatov
a9f625f82a
`crash-fuzz.js` aborted with "harness error: MongoServerError: ns not found" whenever the surviving prefix contained no write that created the collection -- a kill during the first in-flight command on a fresh log. `listIndexes` on a missing namespace is NamespaceNotFound, which is what real MongoDB answers too, so the server was right and the harness treated a legitimate state as its own failure. Worse, it aborted the run instead of verifying that state, which is exactly the state worth verifying. Reproduces with `--seed 1234 --rounds 60` and is why seeded runs were unusable; `--heavy` happened to miss it. Confirmed against the previous commit before changing anything, so it is the harness and not the engine. Both seeds now pass 60 cycles.
670 lines
25 KiB
JavaScript
670 lines
25 KiB
JavaScript
//!/usr/bin/env node
|
|
//
|
|
// Crash-consistency fuzzer for multiforadb (tests/fuzz/crash-fuzz.js).
|
|
//
|
|
// Black-box: drives the server through the official driver, kills it with
|
|
// SIGKILL at a random point, reopens the same log file, and verifies the
|
|
// recovered state against an in-memory model.
|
|
//
|
|
// The invariant under test (the "prefix invariant"):
|
|
// with one sequential client and fsync-before-ack commits, the state that
|
|
// survives a crash is exactly `apply(history[0..m))` for some m with
|
|
// acked <= m <= sent, where the in-flight command (if any) contributes a
|
|
// prefix of its documents. Every fully-acked write must be durable; an
|
|
// in-flight write may or may not be; nothing after it may be.
|
|
//
|
|
// A cycle is: generate random ops -> SIGKILL at a random point (sometimes
|
|
// mid-write, sometimes right after an ack) -> reopen -> verify prefix
|
|
// invariant + index correctness + count. All cycles share one log file, so
|
|
// checkpoints, log truncation and free-list reuse are exercised across
|
|
// cycles when the log grows past the checkpoint threshold (--heavy).
|
|
//
|
|
// Determinism: the op sequence and the kill decision come from a seeded PRNG,
|
|
// so `--seed N` reproduces the same scenario (kill timing inside the window
|
|
// is OS-scheduled, exactly as in any crash tester, e.g. RocksDB db_crashtest).
|
|
//
|
|
// node tests/fuzz/crash-fuzz.js # 60 quick cycles
|
|
// node tests/fuzz/crash-fuzz.js --heavy # grow log past the
|
|
// # checkpoint threshold
|
|
// node tests/fuzz/crash-fuzz.js --seed 1234 --rounds 200
|
|
//
|
|
// Env: CRASH_FUZZ_PORT listen port (default 27230)
|
|
// MFDB_BIN server binary (default ../../zig-out/bin/multiforadb)
|
|
// CRASH_FUZZ_DB log file (default ../../.zig-cache/crash-fuzz.log)
|
|
const { MongoClient } = require('../e2e/node_modules/mongodb');
|
|
const { spawn } = require('child_process');
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
|
|
// --------------------------------------------------------------------------
|
|
// argv / config
|
|
// --------------------------------------------------------------------------
|
|
const argv = process.argv.slice(2);
|
|
const args = {};
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const a = argv[i];
|
|
if (!a.startsWith('--')) continue;
|
|
const v = argv[i + 1];
|
|
args[a.slice(2)] = v !== undefined && !v.startsWith('--') ? v : true;
|
|
}
|
|
function intArg(k, dflt) {
|
|
if (args[k] === undefined || args[k] === true) return dflt;
|
|
const n = Number(args[k]);
|
|
if (!Number.isInteger(n) || n < 0) {
|
|
console.error(`bad --${k} value: ${args[k]}`);
|
|
process.exit(2);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
const HEAVY = !!args.heavy;
|
|
const PORT = Number(process.env.CRASH_FUZZ_PORT || args.port || 27230);
|
|
const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb');
|
|
const DBFILE = process.env.CRASH_FUZZ_DB || path.resolve(__dirname, '../../.zig-cache/crash-fuzz.log');
|
|
const VERBOSE = !!args.verbose;
|
|
const VERIFY_EXEC = !!args['verify-exec'];
|
|
const NO_KILL = !!args['no-kill'];
|
|
const KEEP_LOG = !!args['keep-log'];
|
|
const ROUNDS = intArg('rounds', HEAVY ? 8 : 60);
|
|
const SEED = args.seed !== undefined ? intArg('seed', 0) : ((Math.random() * 0xffffffff) >>> 0);
|
|
const KILL_PROB = HEAVY ? 0.3 : 0.22;
|
|
const KILL_INFLIGHT_PROB = 0.6; // of kill decisions, share that fire mid-write
|
|
const MAX_DOCS = intArg('max-docs', HEAVY ? 3000 : 400);
|
|
const BATCH_MAX = intArg('batch-max', HEAVY ? 400 : 120);
|
|
const MAX_CYCLE_OPS = intArg('max-cycle-ops', HEAVY ? 60 : 30);
|
|
const KILL_DELAY_MS = intArg('kill-delay-ms', 1);
|
|
const LONG_STR = HEAVY ? 4096 : 1024;
|
|
|
|
// --------------------------------------------------------------------------
|
|
// deterministic PRNG (mulberry32)
|
|
// --------------------------------------------------------------------------
|
|
function mulberry32(seed) {
|
|
let a = seed >>> 0;
|
|
return function () {
|
|
a |= 0;
|
|
a = (a + 0x6d2b79f5) | 0;
|
|
let t = Math.imul(a ^ (a >>> 15), 1 | a);
|
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
};
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// model: the intended database state, plus the flat op history of a cycle
|
|
// --------------------------------------------------------------------------
|
|
// Deep-clone a model doc: applyFlat mutates doc objects in place, and every
|
|
// candidate prefix state must be an independent copy (docs are JSON-safe:
|
|
// numbers, strings, bools, null, arrays, nested objects).
|
|
function cloneDoc(d) {
|
|
return JSON.parse(JSON.stringify(d));
|
|
}
|
|
|
|
class Model {
|
|
constructor(other) {
|
|
this.docs = new Map(); // _id -> doc (plain JS object)
|
|
this.indexCreated = false;
|
|
if (other) {
|
|
for (const [k, v] of other.docs) this.docs.set(k, cloneDoc(v));
|
|
this.indexCreated = other.indexCreated;
|
|
}
|
|
}
|
|
}
|
|
|
|
// A flat op: {kind:'insert', doc} | {kind:'update', id, patch} |
|
|
// {kind:'delete', id} | {kind:'createIndex'}
|
|
function applyFlat(docs, f) {
|
|
switch (f.kind) {
|
|
case 'insert':
|
|
// Clone: the same flat op is applied by applyOp (the live model) and by
|
|
// stateAfter (every candidate prefix). Without a clone, both mutate the
|
|
// shared genDoc object and an update would be applied twice.
|
|
docs.set(f.doc._id, cloneDoc(f.doc));
|
|
break;
|
|
case 'update': {
|
|
const d = docs.get(f.id);
|
|
if (!d) throw new Fail('model bug: update of missing doc', { f });
|
|
const p = f.patch;
|
|
if (p.$set) for (const [k, v] of Object.entries(p.$set)) d[k] = v;
|
|
if (p.$inc) for (const [k, v] of Object.entries(p.$inc)) d[k] = (d[k] || 0) + v;
|
|
if (p.$unset) for (const k of Object.keys(p.$unset)) delete d[k];
|
|
break;
|
|
}
|
|
case 'delete':
|
|
docs.delete(f.id);
|
|
break;
|
|
case 'createIndex':
|
|
break; // no doc effect
|
|
}
|
|
}
|
|
|
|
function stateAfter(base, flat, m) {
|
|
const docs = new Map();
|
|
for (const [k, v] of base.docs) docs.set(k, cloneDoc(v));
|
|
for (let i = 0; i < m; i++) applyFlat(docs, flat[i]);
|
|
return docs;
|
|
}
|
|
|
|
function applyOp(intended, op) {
|
|
if (op.kind === 'batch') {
|
|
for (const d of op.docs) intended.docs.set(d._id, cloneDoc(d));
|
|
} else {
|
|
applyFlat(intended.docs, op);
|
|
}
|
|
if (op.kind === 'createIndex') intended.indexCreated = true;
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// canonical comparison (driver-parsed BSON vs plain JS model docs)
|
|
// --------------------------------------------------------------------------
|
|
function canon(v) {
|
|
if (v === null || v === undefined) return 'null';
|
|
if (typeof v === 'number') return Number.isInteger(v) ? 'i' + v : 'd' + v;
|
|
if (typeof v === 'string') return 's' + v;
|
|
if (typeof v === 'boolean') return 'b' + v;
|
|
if (Array.isArray(v)) return '[' + v.map(canon).join(',') + ']';
|
|
if (typeof v === 'object') {
|
|
const ks = Object.keys(v).sort();
|
|
return '{' + ks.map((k) => canon(k) + ':' + canon(v[k])).join(',') + '}';
|
|
}
|
|
return '?' + String(v);
|
|
}
|
|
|
|
function sameDocs(a, b) {
|
|
if (a.size !== b.size) return false;
|
|
for (const [k, v] of a) {
|
|
if (!b.has(k)) return false;
|
|
if (canon(v) !== canon(b.get(k))) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// First few concrete differences between two doc maps (for diagnostics).
|
|
function diffState(a, b) {
|
|
const out = [];
|
|
for (const [k, v] of a) {
|
|
if (!b.has(k)) {
|
|
out.push(`db missing _id=${k}`);
|
|
if (out.length >= 5) return out;
|
|
continue;
|
|
}
|
|
if (canon(v) !== canon(b.get(k))) {
|
|
out.push(`_id=${k} differs: model=${canon(v)} db=${canon(b.get(k))}`);
|
|
if (out.length >= 5) return out;
|
|
}
|
|
}
|
|
for (const [k] of b) {
|
|
if (!a.has(k)) {
|
|
out.push(`db has extra _id=${k}`);
|
|
if (out.length >= 5) return out;
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// random document / op generation
|
|
// --------------------------------------------------------------------------
|
|
function randString(rng) {
|
|
const lenRoll = rng();
|
|
let len;
|
|
if (lenRoll < 0.4) len = Math.floor(rng() * 32); // short
|
|
else if (lenRoll < 0.8) len = 32 + Math.floor(rng() * 128);
|
|
else len = 128 + Math.floor(rng() * LONG_STR); // long: makes batches slow
|
|
let s = '';
|
|
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \t\n"\'{}[]';
|
|
for (let i = 0; i < len; i++) s += chars[Math.floor(rng() * chars.length)];
|
|
return s;
|
|
}
|
|
|
|
function genDoc(rng, id, idx) {
|
|
const d = { _id: id, k: Math.floor(rng() * 10), n: Math.floor(rng() * 100000) };
|
|
const r = rng();
|
|
if (r < 0.35) d.s = randString(rng);
|
|
if (r < 0.5) d.nested = { a: Math.floor(rng() * 100), b: [Math.floor(rng() * 10), Math.floor(rng() * 10)], c: { d: rng() < 0.5 } };
|
|
if (r < 0.65) d.arr = [Math.floor(rng() * 5), randString(rng), rng() < 0.5];
|
|
if (r < 0.75) d.flag = rng() < 0.5;
|
|
if (r < 0.8) d.z = null;
|
|
if (idx !== undefined && rng() < 0.3) d.idx = idx; // occasionally tagged
|
|
return d;
|
|
}
|
|
|
|
function pickKey(rng, docs) {
|
|
const keys = [...docs.keys()];
|
|
return keys[Math.floor(rng() * keys.length)];
|
|
}
|
|
|
|
function genUpdate(rng, intended) {
|
|
const id = pickKey(rng, intended.docs);
|
|
const r = rng();
|
|
if (r < 0.35) return { kind: 'update', id, patch: { $set: { k: Math.floor(rng() * 10) } } };
|
|
if (r < 0.55) return { kind: 'update', id, patch: { $inc: { n: 1 } } };
|
|
if (r < 0.75) return { kind: 'update', id, patch: { $set: { s: randString(rng) } } };
|
|
if (r < 0.9) return { kind: 'update', id, patch: { $unset: { flag: '' } } };
|
|
return { kind: 'update', id, patch: { $set: { nested: { a: 7, b: [1, 2], c: { d: false } }, arr: ['x', 3] } } };
|
|
}
|
|
|
|
// Generator state: rng, nextId, thresholds, per-run switches.
|
|
function makeGen(seed, cfg) {
|
|
const rng = mulberry32(seed);
|
|
return {
|
|
rng,
|
|
nextId: 1,
|
|
idxThreshold: null, // flat count at which createIndex becomes durable, once generated
|
|
cfg,
|
|
nextOp(intended) {
|
|
const { rng, cfg } = this;
|
|
const size = intended.docs.size;
|
|
const r = rng();
|
|
if (!intended.indexCreated && r < 0.08) return { kind: 'createIndex' };
|
|
const overCap = size > cfg.maxDocs * 0.85;
|
|
let kind;
|
|
if (size === 0) {
|
|
kind = 'batch';
|
|
} else if (overCap) {
|
|
const x = rng();
|
|
kind = x < 0.55 ? 'delete' : x < 0.8 ? 'update' : 'batch';
|
|
} else {
|
|
const x = rng();
|
|
kind = x < 0.3 ? 'batch' : x < 0.42 ? 'insert' : x < 0.68 ? 'update' : 'delete';
|
|
}
|
|
switch (kind) {
|
|
case 'batch': {
|
|
const n = overCap ? 1 + Math.floor(rng() * 10) : 1 + Math.floor(rng() * cfg.batchMax);
|
|
const docs = [];
|
|
for (let i = 0; i < n; i++) docs.push(genDoc(rng, this.nextId++, i));
|
|
return { kind: 'batch', docs };
|
|
}
|
|
case 'insert':
|
|
return { kind: 'insert', doc: genDoc(rng, this.nextId++) };
|
|
case 'update':
|
|
return genUpdate(rng, intended);
|
|
case 'delete':
|
|
return { kind: 'delete', id: pickKey(rng, intended.docs) };
|
|
}
|
|
},
|
|
};
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// server control (mirrors tests/e2e/e2e6.js)
|
|
// --------------------------------------------------------------------------
|
|
let server = null;
|
|
let serverLog = '';
|
|
let exited = null; // {code, sig} once the child has exited
|
|
let killedByUs = false;
|
|
|
|
function cleanup() {
|
|
if (server && !exited) {
|
|
try { server.kill('SIGKILL'); } catch {}
|
|
}
|
|
}
|
|
process.on('exit', cleanup);
|
|
process.on('SIGINT', () => { if (args.debug) console.error('[harness SIGINT]'); cleanup(); process.exit(130); });
|
|
process.on('SIGTERM', () => { if (args.debug) console.error('[harness SIGTERM]'); cleanup(); process.exit(143); });
|
|
|
|
function checkUnexpectedDeath(where) {
|
|
// Our own SIGKILL is expected; anything else (Zig panic, replay refusing
|
|
// to open, SIGABRT...) is a finding.
|
|
if (exited && !(killedByUs && exited.sig === 'SIGKILL')) {
|
|
throw new Fail(`server died unexpectedly at ${where}`, {
|
|
code: exited.code,
|
|
sig: exited.sig,
|
|
logTail: serverLog.slice(-3000),
|
|
});
|
|
}
|
|
}
|
|
|
|
function startServer(fresh) {
|
|
return new Promise((resolve, reject) => {
|
|
if (fresh) {
|
|
fs.rmSync(DBFILE, { force: true });
|
|
fs.rmSync(DBFILE + '.data', { force: true });
|
|
}
|
|
serverLog = '';
|
|
exited = null;
|
|
killedByUs = false;
|
|
const extra = HEAVY ? ['--compact-threshold', String(1024 * 1024)] : [];
|
|
server = spawn(BIN, ['--port', String(PORT), '--db', DBFILE].concat(extra), {
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
});
|
|
server.stdout.on('data', (d) => (serverLog += d));
|
|
server.stderr.on('data', (d) => (serverLog += d));
|
|
server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`)));
|
|
server.on('exit', (code, sig) => {
|
|
exited = { code, sig };
|
|
if (args.debug) console.error(`[${Date.now()} server ${server?.pid} exited code=${code} sig=${sig} killedByUs=${killedByUs} graceful=${gracefulStop}]`);
|
|
server = null;
|
|
if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`;
|
|
});
|
|
const deadline = Date.now() + 20000;
|
|
(async () => {
|
|
while (Date.now() < deadline) {
|
|
if (exited) {
|
|
reject(new Error(`server child exited during start (port ${PORT} busy?)\n${serverLog}`));
|
|
return;
|
|
}
|
|
const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 1000 });
|
|
try {
|
|
await c.connect();
|
|
await c.db('admin').command({ ping: 1 });
|
|
await c.close();
|
|
return resolve();
|
|
} catch {
|
|
try { await c.close(); } catch {}
|
|
await sleep(100);
|
|
}
|
|
}
|
|
reject(new Error(`server did not come up on :${PORT}\n${serverLog}`));
|
|
})();
|
|
});
|
|
}
|
|
|
|
async function stopServerGracefully() {
|
|
gracefulStop = true;
|
|
if (server && !exited) {
|
|
const exitedP = new Promise((r) => server.once('exit', r));
|
|
if (args.debug) console.error(`[${Date.now()} graceful-stop targeting pid ${server.pid}]\n${new Error().stack.split('\n').slice(2, 5).join('\n')}`);
|
|
try { server.kill('SIGTERM'); } catch {}
|
|
await Promise.race([exitedP, sleep(5000)]);
|
|
}
|
|
server = null;
|
|
}
|
|
|
|
async function killServer() {
|
|
killedByUs = true;
|
|
if (!exited) {
|
|
const exitedP = new Promise((r) => {
|
|
if (exited) return r();
|
|
const h = () => r();
|
|
if (server) server.once('exit', h);
|
|
else r();
|
|
});
|
|
try { server.kill('SIGKILL'); } catch {}
|
|
await Promise.race([exitedP, sleep(5000)]);
|
|
}
|
|
checkUnexpectedDeath('kill');
|
|
server = null;
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// client helpers
|
|
// --------------------------------------------------------------------------
|
|
async function connectClient() {
|
|
const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, {
|
|
serverSelectionTimeoutMS: 5000,
|
|
retryWrites: false,
|
|
});
|
|
await c.connect();
|
|
return c;
|
|
}
|
|
|
|
async function closeClient(c) {
|
|
if (!c) return;
|
|
await Promise.race([c.close().catch(() => {}), sleep(2000)]).catch(() => {});
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// one cycle: random ops then SIGKILL
|
|
// --------------------------------------------------------------------------
|
|
async function execOp(coll, op) {
|
|
switch (op.kind) {
|
|
case 'insert':
|
|
return coll.insertOne(op.doc);
|
|
case 'batch':
|
|
return coll.insertMany(op.docs);
|
|
case 'update':
|
|
return coll.updateOne({ _id: op.id }, op.patch);
|
|
case 'delete':
|
|
return coll.deleteOne({ _id: op.id });
|
|
case 'createIndex':
|
|
return coll.createIndex({ k: 1 });
|
|
}
|
|
}
|
|
|
|
async function runCycle(client, base, gen) {
|
|
const coll = client.db('fuzz').collection('c');
|
|
const intended = new Model(base);
|
|
const flat = [];
|
|
let acked = 0;
|
|
let killMode = null;
|
|
let matched = null;
|
|
let idxPos = null; // flat count at which this cycle's createIndex becomes durable
|
|
|
|
for (let i = 0; i < gen.cfg.maxCycleOps; i++) {
|
|
const kill = gen.rng() < gen.cfg.killProb || i === gen.cfg.maxCycleOps - 1;
|
|
const op = gen.nextOp(intended);
|
|
const inflight = !NO_KILL && kill && gen.rng() < KILL_INFLIGHT_PROB;
|
|
|
|
if (op.kind === 'batch') {
|
|
for (const d of op.docs) flat.push({ kind: 'insert', doc: d });
|
|
} else {
|
|
flat.push(op);
|
|
}
|
|
if (op.kind === 'createIndex') {
|
|
idxPos = flat.length; // flat count at which this cycle's record is durable
|
|
}
|
|
const sent = flat.length;
|
|
|
|
if (inflight) {
|
|
const p = execOp(coll, op);
|
|
p.catch(() => {}); // the coming SIGKILL may kill the request: expected
|
|
await sleep(gen.cfg.killDelayMs);
|
|
await killServer();
|
|
killMode = 'inflight';
|
|
applyOp(intended, op); // optimistic; the prefix check decides what survived
|
|
} else {
|
|
await execOp(coll, op);
|
|
await sleep(5); // let a self-crash's exit event deliver before we check
|
|
checkUnexpectedDeath(`op ${flat.length} of cycle`);
|
|
acked = sent;
|
|
applyOp(intended, op);
|
|
// Diagnostic: read the doc back and compare with the model. Pins down
|
|
// whether a divergence is an execution bug (in-memory wrong) or a
|
|
// log/replay bug (in-memory right, lost after reopen). Must run while
|
|
// the server is still alive -- after the stop below the query would
|
|
// fail with ECONNREFUSED.
|
|
if (VERIFY_EXEC && op.kind === 'update') {
|
|
const back = await coll.findOne({ _id: op.id });
|
|
const want = intended.docs.get(op.id);
|
|
if (!back || canon(back) !== canon(want)) {
|
|
throw new Fail(`cycle exec mismatch: update ${op.id} executed wrong`, {
|
|
back: back ? canon(back) : null,
|
|
want: canon(want),
|
|
});
|
|
}
|
|
}
|
|
if (kill) {
|
|
if (NO_KILL) {
|
|
// --no-kill: verify durability of the run's writes via a graceful
|
|
// stop + reopen instead of SIGKILL.
|
|
await stopServerGracefully();
|
|
} else {
|
|
await killServer();
|
|
}
|
|
killMode = 'after';
|
|
}
|
|
}
|
|
if (kill) break;
|
|
}
|
|
if (killMode === null) killMode = 'after'; // unreachable: last op forces a kill
|
|
return { flat, acked, sentCount: flat.length, killMode, matched, idxPos };
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// verification after reopen: the prefix invariant
|
|
// --------------------------------------------------------------------------
|
|
async function verify(client, base, r, cycleNo) {
|
|
const db = client.db('fuzz');
|
|
const coll = db.collection('c');
|
|
const dbDocs = await coll.find({}, { sort: { _id: 1 } }).toArray();
|
|
const dbMap = new Map(dbDocs.map((d) => [d._id, d]));
|
|
// `listIndexes` on a namespace that does not exist is NamespaceNotFound, which
|
|
// is what real MongoDB answers too -- and it is the *expected* state whenever
|
|
// the surviving prefix contains no write that created the collection (a kill
|
|
// during the first in-flight command on a fresh log). Treating it as a harness
|
|
// error made any seed whose first cycle crashed at prefix 0 abort the run
|
|
// instead of verifying it, which is exactly the case worth verifying.
|
|
const dbIndexes = await coll.indexes().catch((e) => {
|
|
if (e.code === 26 || /ns not found|NamespaceNotFound/i.test(e.message)) return [];
|
|
throw e;
|
|
});
|
|
const indexExists = dbIndexes.some((i) => i.name === 'k_1');
|
|
if (VERBOSE) {
|
|
console.log(` [verify] indexes=${JSON.stringify(dbIndexes.map((i) => i.name))} baseIdx=${base.indexCreated} idxPos=${r.idxPos}`);
|
|
}
|
|
|
|
let matched = null;
|
|
const diag = [];
|
|
for (let m = r.acked; m <= r.sentCount; m++) {
|
|
const st = stateAfter(base, r.flat, m);
|
|
const wantIdx = base.indexCreated || (r.idxPos !== null && m >= r.idxPos);
|
|
const stateOk = sameDocs(st, dbMap);
|
|
const indexOk = indexExists === wantIdx;
|
|
if (stateOk && indexOk) { matched = m; break; }
|
|
diag.push({
|
|
m,
|
|
size: st.size,
|
|
stateOk,
|
|
indexOk,
|
|
diff: stateOk ? [] : diffState(st, dbMap),
|
|
});
|
|
}
|
|
|
|
if (matched === null) {
|
|
throw new Fail(`cycle ${cycleNo}: recovered state is not a valid prefix`, {
|
|
cycleNo,
|
|
acked: r.acked,
|
|
sent: r.sentCount,
|
|
killMode: r.killMode,
|
|
baseIdx: base.indexCreated,
|
|
idxPos: r.idxPos,
|
|
indexExists,
|
|
dbIndexes: dbIndexes.map((i) => i.name),
|
|
dbSize: dbMap.size,
|
|
candidates: diag,
|
|
serverLog: serverLog.slice(-4000),
|
|
ops: r.flat.map((f) =>
|
|
f.kind === 'insert' ? `ins ${f.doc._id}` :
|
|
f.kind === 'update' ? `upd ${f.id} ${JSON.stringify(f.patch)}` :
|
|
f.kind === 'delete' ? `del ${f.id}` : 'idx'),
|
|
});
|
|
}
|
|
|
|
// Index / filter correctness at the matched prefix: find({k:v}) must return
|
|
// exactly the docs with k==v, whether the index exists (candidates+reapply)
|
|
// or the query was a full scan.
|
|
const st = stateAfter(base, r.flat, matched);
|
|
for (let v = 0; v < 10; v++) {
|
|
const got = (await coll.find({ k: v }, { sort: { _id: 1 } }).toArray())
|
|
.map((d) => canon(d))
|
|
.sort();
|
|
const want = [...st.values()]
|
|
.filter((d) => d.k === v)
|
|
.map((d) => canon(d))
|
|
.sort();
|
|
if (got.length !== want.length || got.some((c, i) => c !== want[i])) {
|
|
throw new Fail(`cycle ${cycleNo}: find({k:${v}}) mismatch at prefix ${matched}`, {
|
|
cycleNo, v, matched, got, want,
|
|
});
|
|
}
|
|
}
|
|
const cnt = await coll.countDocuments({});
|
|
if (cnt !== st.size) {
|
|
throw new Fail(`cycle ${cycleNo}: countDocuments mismatch at prefix ${matched}`, {
|
|
cycleNo, matched, count: cnt, expected: st.size,
|
|
});
|
|
}
|
|
r.matched = matched;
|
|
return { docs: st, indexCreated: indexExists };
|
|
}
|
|
|
|
// --------------------------------------------------------------------------
|
|
// failure artifacts
|
|
// --------------------------------------------------------------------------
|
|
class Fail extends Error {
|
|
constructor(msg, detail) { super(msg); this.detail = detail; }
|
|
}
|
|
|
|
function dumpArtifact(err, gen) {
|
|
const art = {
|
|
seed: SEED,
|
|
mode: HEAVY ? 'heavy' : 'default',
|
|
rounds: ROUNDS,
|
|
error: err.message,
|
|
detail: err.detail,
|
|
nextId: gen.nextId,
|
|
};
|
|
const file = `/tmp/crash-fuzz-fail-${SEED}.json`;
|
|
fs.writeFileSync(file, JSON.stringify(art, null, 2));
|
|
console.error(`\n✗ ${err.message}`);
|
|
console.error(` detail: ${JSON.stringify(err.detail, null, 2).slice(0, 4000)}`);
|
|
console.error(` artifact: ${file}`);
|
|
console.error(` rerun: node tests/fuzz/crash-fuzz.js --seed ${SEED} --rounds ${ROUNDS}${HEAVY ? ' --heavy' : ''}`);
|
|
}
|
|
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
|
|
// --------------------------------------------------------------------------
|
|
// main
|
|
// --------------------------------------------------------------------------
|
|
async function main() {
|
|
console.log(`crash-fuzz: seed=${SEED} rounds=${ROUNDS} mode=${HEAVY ? 'heavy' : 'default'} port=${PORT}`);
|
|
console.log(` bin=${BIN}`);
|
|
console.log(` db=${DBFILE}`);
|
|
|
|
const gen = makeGen(SEED, { maxDocs: MAX_DOCS, batchMax: BATCH_MAX, maxCycleOps: MAX_CYCLE_OPS, killProb: KILL_PROB, killDelayMs: KILL_DELAY_MS });
|
|
|
|
if (!KEEP_LOG) {
|
|
fs.rmSync(DBFILE, { force: true });
|
|
fs.rmSync(DBFILE + '.data', { force: true }); // stale data files hit the 64GB reservation ceiling
|
|
}
|
|
await startServer(true);
|
|
let base = new Model();
|
|
|
|
if (args.debug) {
|
|
const iv = setInterval(() => {
|
|
if (server) console.error(`[${Date.now()} watch server pid=${server.pid} exited=${JSON.stringify(exited)}]`);
|
|
else console.error(`[${Date.now()} watch server=null exited=${JSON.stringify(exited)}]`);
|
|
}, 100);
|
|
iv.unref();
|
|
}
|
|
|
|
try {
|
|
for (let c = 1; c <= ROUNDS; c++) {
|
|
let client = await connectClient();
|
|
const r = await runCycle(client, base, gen); // ends with the server killed
|
|
await closeClient(client);
|
|
|
|
await startServer(false); // reopen: replay must never refuse
|
|
client = await connectClient();
|
|
const v = await verify(client, base, r, c);
|
|
await closeClient(client);
|
|
base = new Model();
|
|
for (const [k, d] of v.docs) base.docs.set(k, d);
|
|
base.indexCreated = v.indexCreated;
|
|
|
|
if (VERBOSE || c === 1 || c % 10 === 0 || c === ROUNDS) {
|
|
let size = 0, dsize = 0;
|
|
try { size = fs.statSync(DBFILE).size; } catch {}
|
|
try { dsize = fs.statSync(DBFILE + '.data').size; } catch {}
|
|
console.log(` cycle ${c}/${ROUNDS}: ops=${r.flat.length} acked=${r.acked} sent=${r.sentCount} kill=${r.killMode} prefix=${r.matched} docs=${base.docs.size} log=${(size / 1e6).toFixed(1)}MB data=${(dsize / 1e6).toFixed(0)}MB`);
|
|
}
|
|
}
|
|
} catch (err) {
|
|
if (err instanceof Fail) {
|
|
dumpArtifact(err, gen);
|
|
process.exit(1);
|
|
}
|
|
console.error(`\n✗ harness error: ${err.stack || err}`);
|
|
console.error(` server exited: ${JSON.stringify(exited)}`);
|
|
console.error(` serverLog: ${serverLog.slice(-2000)}`);
|
|
process.exit(1);
|
|
} finally {
|
|
cleanup();
|
|
}
|
|
|
|
console.log(`crash-fuzz: PASS — ${ROUNDS} crash/reopen cycles, prefix invariant held every time`);
|
|
}
|
|
|
|
main().catch((e) => { console.error(e.stack || e); process.exit(1); });
|