MultiforaDB/tests/fuzz/crash-fuzz.js

//!/usr/bin/env node
//
// Crash-consistency fuzzer for multiforadb (tests/fuzz/crash-fuzz.js).
//
// Black-box: drives the server through the official driver, kills it with
// SIGKILL at a random point, reopens the same log file, and verifies the
// recovered state against an in-memory model.
//
// The invariant under test (the "prefix invariant"):
//   with one sequential client and fsync-before-ack commits, the state that
//   survives a crash is exactly `apply(history[0..m))` for some m with
//   acked <= m <= sent, where the in-flight command (if any) contributes a
//   prefix of its documents.  Every fully-acked write must be durable; an
//   in-flight write may or may not be; nothing after it may be.
//
// A cycle is: generate random ops -> SIGKILL at a random point (sometimes
// mid-write, sometimes right after an ack) -> reopen -> verify prefix
// invariant + index correctness + count.  All cycles share one log file, so
// checkpoints, log truncation and free-list reuse are exercised across
// cycles when the log grows past the checkpoint threshold (--heavy).
//
// Determinism: the op sequence and the kill decision come from a seeded PRNG,
// so `--seed N` reproduces the same scenario (kill timing inside the window
// is OS-scheduled, exactly as in any crash tester, e.g. RocksDB db_crashtest).
//
//   node tests/fuzz/crash-fuzz.js                       # 60 quick cycles
//   node tests/fuzz/crash-fuzz.js --heavy               # grow log past the
//                                                       # checkpoint threshold
//   node tests/fuzz/crash-fuzz.js --seed 1234 --rounds 200
//
// Env:  CRASH_FUZZ_PORT  listen port (default 27230)
//       MFDB_BIN         server binary (default ../../zig-out/bin/multiforadb)
//       CRASH_FUZZ_DB    log file (default ../../.zig-cache/crash-fuzz.log)
const { MongoClient } = require('../e2e/node_modules/mongodb');
const { spawn } = require('child_process');
const fs = require('fs');
const path = require('path');

// --------------------------------------------------------------------------
// argv / config
// --------------------------------------------------------------------------
const argv = process.argv.slice(2);
const args = {};
for (let i = 0; i < argv.length; i++) {
  const a = argv[i];
  if (!a.startsWith('--')) continue;
  const v = argv[i + 1];
  args[a.slice(2)] = v !== undefined && !v.startsWith('--') ? v : true;
}
function intArg(k, dflt) {
  if (args[k] === undefined || args[k] === true) return dflt;
  const n = Number(args[k]);
  if (!Number.isInteger(n) || n < 0) {
    console.error(`bad --${k} value: ${args[k]}`);
    process.exit(2);
  }
  return n;
}

const HEAVY = !!args.heavy;
const PORT = Number(process.env.CRASH_FUZZ_PORT || args.port || 27230);
const BIN = process.env.MFDB_BIN || path.resolve(__dirname, '../../zig-out/bin/multiforadb');
const DBFILE = process.env.CRASH_FUZZ_DB || path.resolve(__dirname, '../../.zig-cache/crash-fuzz.log');
const VERBOSE = !!args.verbose;
const VERIFY_EXEC = !!args['verify-exec'];
const NO_KILL = !!args['no-kill'];
const KEEP_LOG = !!args['keep-log'];
const ROUNDS = intArg('rounds', HEAVY ? 8 : 60);
const SEED = args.seed !== undefined ? intArg('seed', 0) : ((Math.random() * 0xffffffff) >>> 0);
const KILL_PROB = HEAVY ? 0.3 : 0.22;
const KILL_INFLIGHT_PROB = 0.6; // of kill decisions, share that fire mid-write
const MAX_DOCS = intArg('max-docs', HEAVY ? 3000 : 400);
const BATCH_MAX = intArg('batch-max', HEAVY ? 400 : 120);
const MAX_CYCLE_OPS = intArg('max-cycle-ops', HEAVY ? 60 : 30);
const KILL_DELAY_MS = intArg('kill-delay-ms', 1);
const LONG_STR = HEAVY ? 4096 : 1024;

// --------------------------------------------------------------------------
// deterministic PRNG (mulberry32)
// --------------------------------------------------------------------------
function mulberry32(seed) {
  let a = seed >>> 0;
  return function () {
    a |= 0;
    a = (a + 0x6d2b79f5) | 0;
    let t = Math.imul(a ^ (a >>> 15), 1 | a);
    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
  };
}

// --------------------------------------------------------------------------
// model: the intended database state, plus the flat op history of a cycle
// --------------------------------------------------------------------------
// Deep-clone a model doc: applyFlat mutates doc objects in place, and every
// candidate prefix state must be an independent copy (docs are JSON-safe:
// numbers, strings, bools, null, arrays, nested objects).
function cloneDoc(d) {
  return JSON.parse(JSON.stringify(d));
}

class Model {
  constructor(other) {
    this.docs = new Map(); // _id -> doc (plain JS object)
    this.indexCreated = false;
    if (other) {
      for (const [k, v] of other.docs) this.docs.set(k, cloneDoc(v));
      this.indexCreated = other.indexCreated;
    }
  }
}

// A flat op: {kind:'insert', doc} | {kind:'update', id, patch} |
//            {kind:'delete', id} | {kind:'createIndex'}
function applyFlat(docs, f) {
  switch (f.kind) {
    case 'insert':
      // Clone: the same flat op is applied by applyOp (the live model) and by
      // stateAfter (every candidate prefix). Without a clone, both mutate the
      // shared genDoc object and an update would be applied twice.
      docs.set(f.doc._id, cloneDoc(f.doc));
      break;
    case 'update': {
      const d = docs.get(f.id);
      if (!d) throw new Fail('model bug: update of missing doc', { f });
      const p = f.patch;
      if (p.$set) for (const [k, v] of Object.entries(p.$set)) d[k] = v;
      if (p.$inc) for (const [k, v] of Object.entries(p.$inc)) d[k] = (d[k] || 0) + v;
      if (p.$unset) for (const k of Object.keys(p.$unset)) delete d[k];
      break;
    }
    case 'delete':
      docs.delete(f.id);
      break;
    case 'createIndex':
      break; // no doc effect
  }
}

function stateAfter(base, flat, m) {
  const docs = new Map();
  for (const [k, v] of base.docs) docs.set(k, cloneDoc(v));
  for (let i = 0; i < m; i++) applyFlat(docs, flat[i]);
  return docs;
}

function applyOp(intended, op) {
  if (op.kind === 'batch') {
    for (const d of op.docs) intended.docs.set(d._id, cloneDoc(d));
  } else {
    applyFlat(intended.docs, op);
  }
  if (op.kind === 'createIndex') intended.indexCreated = true;
}

// --------------------------------------------------------------------------
// canonical comparison (driver-parsed BSON vs plain JS model docs)
// --------------------------------------------------------------------------
function canon(v) {
  if (v === null || v === undefined) return 'null';
  if (typeof v === 'number') return Number.isInteger(v) ? 'i' + v : 'd' + v;
  if (typeof v === 'string') return 's' + v;
  if (typeof v === 'boolean') return 'b' + v;
  if (Array.isArray(v)) return '[' + v.map(canon).join(',') + ']';
  if (typeof v === 'object') {
    const ks = Object.keys(v).sort();
    return '{' + ks.map((k) => canon(k) + ':' + canon(v[k])).join(',') + '}';
  }
  return '?' + String(v);
}

function sameDocs(a, b) {
  if (a.size !== b.size) return false;
  for (const [k, v] of a) {
    if (!b.has(k)) return false;
    if (canon(v) !== canon(b.get(k))) return false;
  }
  return true;
}

// First few concrete differences between two doc maps (for diagnostics).
function diffState(a, b) {
  const out = [];
  for (const [k, v] of a) {
    if (!b.has(k)) {
      out.push(`db missing _id=${k}`);
      if (out.length >= 5) return out;
      continue;
    }
    if (canon(v) !== canon(b.get(k))) {
      out.push(`_id=${k} differs: model=${canon(v)} db=${canon(b.get(k))}`);
      if (out.length >= 5) return out;
    }
  }
  for (const [k] of b) {
    if (!a.has(k)) {
      out.push(`db has extra _id=${k}`);
      if (out.length >= 5) return out;
    }
  }
  return out;
}

// --------------------------------------------------------------------------
// random document / op generation
// --------------------------------------------------------------------------
function randString(rng) {
  const lenRoll = rng();
  let len;
  if (lenRoll < 0.4) len = Math.floor(rng() * 32); // short
  else if (lenRoll < 0.8) len = 32 + Math.floor(rng() * 128);
  else len = 128 + Math.floor(rng() * LONG_STR); // long: makes batches slow
  let s = '';
  const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 \t\n"\'{}[]';
  for (let i = 0; i < len; i++) s += chars[Math.floor(rng() * chars.length)];
  return s;
}

function genDoc(rng, id, idx) {
  const d = { _id: id, k: Math.floor(rng() * 10), n: Math.floor(rng() * 100000) };
  const r = rng();
  if (r < 0.35) d.s = randString(rng);
  if (r < 0.5) d.nested = { a: Math.floor(rng() * 100), b: [Math.floor(rng() * 10), Math.floor(rng() * 10)], c: { d: rng() < 0.5 } };
  if (r < 0.65) d.arr = [Math.floor(rng() * 5), randString(rng), rng() < 0.5];
  if (r < 0.75) d.flag = rng() < 0.5;
  if (r < 0.8) d.z = null;
  if (idx !== undefined && rng() < 0.3) d.idx = idx; // occasionally tagged
  return d;
}

function pickKey(rng, docs) {
  const keys = [...docs.keys()];
  return keys[Math.floor(rng() * keys.length)];
}

function genUpdate(rng, intended) {
  const id = pickKey(rng, intended.docs);
  const r = rng();
  if (r < 0.35) return { kind: 'update', id, patch: { $set: { k: Math.floor(rng() * 10) } } };
  if (r < 0.55) return { kind: 'update', id, patch: { $inc: { n: 1 } } };
  if (r < 0.75) return { kind: 'update', id, patch: { $set: { s: randString(rng) } } };
  if (r < 0.9) return { kind: 'update', id, patch: { $unset: { flag: '' } } };
  return { kind: 'update', id, patch: { $set: { nested: { a: 7, b: [1, 2], c: { d: false } }, arr: ['x', 3] } } };
}

// Generator state: rng, nextId, thresholds, per-run switches.
function makeGen(seed, cfg) {
  const rng = mulberry32(seed);
  return {
    rng,
    nextId: 1,
    idxThreshold: null, // flat count at which createIndex becomes durable, once generated
    cfg,
    nextOp(intended) {
      const { rng, cfg } = this;
      const size = intended.docs.size;
      const r = rng();
      if (!intended.indexCreated && r < 0.08) return { kind: 'createIndex' };
      const overCap = size > cfg.maxDocs * 0.85;
      let kind;
      if (size === 0) {
        kind = 'batch';
      } else if (overCap) {
        const x = rng();
        kind = x < 0.55 ? 'delete' : x < 0.8 ? 'update' : 'batch';
      } else {
        const x = rng();
        kind = x < 0.3 ? 'batch' : x < 0.42 ? 'insert' : x < 0.68 ? 'update' : 'delete';
      }
      switch (kind) {
        case 'batch': {
          const n = overCap ? 1 + Math.floor(rng() * 10) : 1 + Math.floor(rng() * cfg.batchMax);
          const docs = [];
          for (let i = 0; i < n; i++) docs.push(genDoc(rng, this.nextId++, i));
          return { kind: 'batch', docs };
        }
        case 'insert':
          return { kind: 'insert', doc: genDoc(rng, this.nextId++) };
        case 'update':
          return genUpdate(rng, intended);
        case 'delete':
          return { kind: 'delete', id: pickKey(rng, intended.docs) };
      }
    },
  };
}

// --------------------------------------------------------------------------
// server control (mirrors tests/e2e/e2e6.js)
// --------------------------------------------------------------------------
let server = null;
let serverLog = '';
let exited = null; // {code, sig} once the child has exited
let killedByUs = false;

function cleanup() {
  if (server && !exited) {
    try { server.kill('SIGKILL'); } catch {}
  }
}
process.on('exit', cleanup);
process.on('SIGINT', () => { if (args.debug) console.error('[harness SIGINT]'); cleanup(); process.exit(130); });
process.on('SIGTERM', () => { if (args.debug) console.error('[harness SIGTERM]'); cleanup(); process.exit(143); });

function checkUnexpectedDeath(where) {
  // Our own SIGKILL is expected; anything else (Zig panic, replay refusing
  // to open, SIGABRT...) is a finding.
  if (exited && !(killedByUs && exited.sig === 'SIGKILL')) {
    throw new Fail(`server died unexpectedly at ${where}`, {
      code: exited.code,
      sig: exited.sig,
      logTail: serverLog.slice(-3000),
    });
  }
}

function startServer(fresh) {
  return new Promise((resolve, reject) => {
    if (fresh) {
      fs.rmSync(DBFILE, { force: true });
      fs.rmSync(DBFILE + '.data', { force: true });
    }
    serverLog = '';
    exited = null;
    killedByUs = false;
    const extra = HEAVY ? ['--compact-threshold', String(1024 * 1024)] : [];
    server = spawn(BIN, ['--port', String(PORT), '--db', DBFILE].concat(extra), {
      stdio: ['ignore', 'pipe', 'pipe'],
    });
    server.stdout.on('data', (d) => (serverLog += d));
    server.stderr.on('data', (d) => (serverLog += d));
    server.on('error', (e) => reject(new Error(`cannot start ${BIN}: ${e.message}`)));
    server.on('exit', (code, sig) => {
      exited = { code, sig };
      if (args.debug) console.error(`[${Date.now()} server ${server?.pid} exited code=${code} sig=${sig} killedByUs=${killedByUs} graceful=${gracefulStop}]`);
      server = null;
      if (code !== null && sig === null) serverLog += `\n[child exited rc=${code}]`;
    });
    const deadline = Date.now() + 20000;
    (async () => {
      while (Date.now() < deadline) {
        if (exited) {
          reject(new Error(`server child exited during start (port ${PORT} busy?)\n${serverLog}`));
          return;
        }
        const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 1000 });
        try {
          await c.connect();
          await c.db('admin').command({ ping: 1 });
          await c.close();
          return resolve();
        } catch {
          try { await c.close(); } catch {}
          await sleep(100);
        }
      }
      reject(new Error(`server did not come up on :${PORT}\n${serverLog}`));
    })();
  });
}

async function stopServerGracefully() {
  gracefulStop = true;
  if (server && !exited) {
    const exitedP = new Promise((r) => server.once('exit', r));
    if (args.debug) console.error(`[${Date.now()} graceful-stop targeting pid ${server.pid}]\n${new Error().stack.split('\n').slice(2, 5).join('\n')}`);
    try { server.kill('SIGTERM'); } catch {}
    await Promise.race([exitedP, sleep(5000)]);
  }
  server = null;
}

async function killServer() {
  killedByUs = true;
  if (!exited) {
    const exitedP = new Promise((r) => {
      if (exited) return r();
      const h = () => r();
      if (server) server.once('exit', h);
      else r();
    });
    try { server.kill('SIGKILL'); } catch {}
    await Promise.race([exitedP, sleep(5000)]);
  }
  checkUnexpectedDeath('kill');
  server = null;
}

// --------------------------------------------------------------------------
// client helpers
// --------------------------------------------------------------------------
async function connectClient() {
  const c = new MongoClient(`mongodb://127.0.0.1:${PORT}`, {
    serverSelectionTimeoutMS: 5000,
    retryWrites: false,
  });
  await c.connect();
  return c;
}

async function closeClient(c) {
  if (!c) return;
  await Promise.race([c.close().catch(() => {}), sleep(2000)]).catch(() => {});
}

// --------------------------------------------------------------------------
// one cycle: random ops then SIGKILL
// --------------------------------------------------------------------------
async function execOp(coll, op) {
  switch (op.kind) {
    case 'insert':
      return coll.insertOne(op.doc);
    case 'batch':
      return coll.insertMany(op.docs);
    case 'update':
      return coll.updateOne({ _id: op.id }, op.patch);
    case 'delete':
      return coll.deleteOne({ _id: op.id });
    case 'createIndex':
      return coll.createIndex({ k: 1 });
  }
}

async function runCycle(client, base, gen) {
  const coll = client.db('fuzz').collection('c');
  const intended = new Model(base);
  const flat = [];
  let acked = 0;
  let killMode = null;
  let matched = null;
  let idxPos = null; // flat count at which this cycle's createIndex becomes durable

  for (let i = 0; i < gen.cfg.maxCycleOps; i++) {
    const kill = gen.rng() < gen.cfg.killProb || i === gen.cfg.maxCycleOps - 1;
    const op = gen.nextOp(intended);
    const inflight = !NO_KILL && kill && gen.rng() < KILL_INFLIGHT_PROB;

    if (op.kind === 'batch') {
      for (const d of op.docs) flat.push({ kind: 'insert', doc: d });
    } else {
      flat.push(op);
    }
    if (op.kind === 'createIndex') {
      idxPos = flat.length; // flat count at which this cycle's record is durable
    }
    const sent = flat.length;

    if (inflight) {
      const p = execOp(coll, op);
      p.catch(() => {}); // the coming SIGKILL may kill the request: expected
      await sleep(gen.cfg.killDelayMs);
      await killServer();
      killMode = 'inflight';
      applyOp(intended, op); // optimistic; the prefix check decides what survived
    } else {
      await execOp(coll, op);
      await sleep(5); // let a self-crash's exit event deliver before we check
      checkUnexpectedDeath(`op ${flat.length} of cycle`);
      acked = sent;
      applyOp(intended, op);
      // Diagnostic: read the doc back and compare with the model. Pins down
      // whether a divergence is an execution bug (in-memory wrong) or a
      // log/replay bug (in-memory right, lost after reopen). Must run while
      // the server is still alive -- after the stop below the query would
      // fail with ECONNREFUSED.
      if (VERIFY_EXEC && op.kind === 'update') {
        const back = await coll.findOne({ _id: op.id });
        const want = intended.docs.get(op.id);
        if (!back || canon(back) !== canon(want)) {
          throw new Fail(`cycle exec mismatch: update ${op.id} executed wrong`, {
            back: back ? canon(back) : null,
            want: canon(want),
          });
        }
      }
      if (kill) {
        if (NO_KILL) {
          // --no-kill: verify durability of the run's writes via a graceful
          // stop + reopen instead of SIGKILL.
          await stopServerGracefully();
        } else {
          await killServer();
        }
        killMode = 'after';
      }
    }
    if (kill) break;
  }
  if (killMode === null) killMode = 'after'; // unreachable: last op forces a kill
  return { flat, acked, sentCount: flat.length, killMode, matched, idxPos };
}

// --------------------------------------------------------------------------
// verification after reopen: the prefix invariant
// --------------------------------------------------------------------------
async function verify(client, base, r, cycleNo) {
  const db = client.db('fuzz');
  const coll = db.collection('c');
  const dbDocs = await coll.find({}, { sort: { _id: 1 } }).toArray();
  const dbMap = new Map(dbDocs.map((d) => [d._id, d]));
  // listIndexes on a nonexistent namespace answers NamespaceNotFound, as real
  // MongoDB does too -- the expected state when a kill lands during the first
  // in-flight command on a fresh log. Treating it as a harness error made seeds
  // that crashed at prefix 0 abort instead of verifying.
  const dbIndexes = await coll.indexes().catch((e) => {
    if (e.code === 26 || /ns not found|NamespaceNotFound/i.test(e.message)) return [];
    throw e;
  });
  const indexExists = dbIndexes.some((i) => i.name === 'k_1');
  if (VERBOSE) {
    console.log(`    [verify] indexes=${JSON.stringify(dbIndexes.map((i) => i.name))} baseIdx=${base.indexCreated} idxPos=${r.idxPos}`);
  }

  let matched = null;
  const diag = [];
  for (let m = r.acked; m <= r.sentCount; m++) {
    const st = stateAfter(base, r.flat, m);
    const wantIdx = base.indexCreated || (r.idxPos !== null && m >= r.idxPos);
    const stateOk = sameDocs(st, dbMap);
    const indexOk = indexExists === wantIdx;
    if (stateOk && indexOk) { matched = m; break; }
    diag.push({
      m,
      size: st.size,
      stateOk,
      indexOk,
      diff: stateOk ? [] : diffState(st, dbMap),
    });
  }

  if (matched === null) {
    throw new Fail(`cycle ${cycleNo}: recovered state is not a valid prefix`, {
      cycleNo,
      acked: r.acked,
      sent: r.sentCount,
      killMode: r.killMode,
      baseIdx: base.indexCreated,
      idxPos: r.idxPos,
      indexExists,
      dbIndexes: dbIndexes.map((i) => i.name),
      dbSize: dbMap.size,
      candidates: diag,
      serverLog: serverLog.slice(-4000),
      ops: r.flat.map((f) =>
        f.kind === 'insert' ? `ins ${f.doc._id}` :
        f.kind === 'update' ? `upd ${f.id} ${JSON.stringify(f.patch)}` :
        f.kind === 'delete' ? `del ${f.id}` : 'idx'),
    });
  }

  // Index / filter correctness at the matched prefix: find({k:v}) must return
  // exactly the docs with k==v, whether the index exists (candidates+reapply)
  // or the query was a full scan.
  const st = stateAfter(base, r.flat, matched);
  for (let v = 0; v < 10; v++) {
    const got = (await coll.find({ k: v }, { sort: { _id: 1 } }).toArray())
      .map((d) => canon(d))
      .sort();
    const want = [...st.values()]
      .filter((d) => d.k === v)
      .map((d) => canon(d))
      .sort();
    if (got.length !== want.length || got.some((c, i) => c !== want[i])) {
      // Decisive diagnostic: the same question asked without the index. `dbDocs`
      // came from find({}) on this same reopened server, so filtering it here
      // says whether the *documents* are wrong or only the index's answer about
      // them. An index that returns fewer documents than a scan is the canonical
      // under-approximation -- candidates are generated from the index and the
      // full filter is only re-applied to those, so a missing entry is a
      // silently missing result.
      const scanGot = dbDocs.filter((d) => d.k === v).map((d) => canon(d)).sort();
      throw new Fail(`cycle ${cycleNo}: find({k:${v}}) mismatch at prefix ${matched}`, {
        cycleNo,
        v,
        matched,
        verdict:
          scanGot.length === want.length && scanGot.every((c, i) => c === want[i])
            ? 'INDEX under-approximates: a scan of the same server returns the expected documents'
            : 'DOCUMENTS differ too: the scan does not match the model either',
        indexReturned: got.length,
        scanReturned: scanGot.length,
        modelExpected: want.length,
        // Per-key totals, so a single lost leaf is distinguishable from an
        // index that came back empty.
        perKeyIndexVsScan: await (async () => {
          const rows = [];
          for (let u = 0; u < 10; u++) {
            const idx = (await coll.find({ k: u }).toArray()).length;
            const scan = dbDocs.filter((d) => d.k === u).length;
            rows.push({ k: u, index: idx, scan });
          }
          return rows;
        })(),
        indexes: dbIndexes.map((i) => i.name),
        totalDocs: dbDocs.length,
        serverLog: serverLog.slice(-2000),
        got,
        want,
      });
    }
  }
  const cnt = await coll.countDocuments({});
  if (cnt !== st.size) {
    throw new Fail(`cycle ${cycleNo}: countDocuments mismatch at prefix ${matched}`, {
      cycleNo, matched, count: cnt, expected: st.size,
    });
  }
  r.matched = matched;
  return { docs: st, indexCreated: indexExists };
}

// --------------------------------------------------------------------------
// failure artifacts
// --------------------------------------------------------------------------
class Fail extends Error {
  constructor(msg, detail) { super(msg); this.detail = detail; }
}

function dumpArtifact(err, gen) {
  const art = {
    seed: SEED,
    mode: HEAVY ? 'heavy' : 'default',
    rounds: ROUNDS,
    error: err.message,
    detail: err.detail,
    nextId: gen.nextId,
  };
  const file = `/tmp/crash-fuzz-fail-${SEED}.json`;
  fs.writeFileSync(file, JSON.stringify(art, null, 2));
  console.error(`\n✗ ${err.message}`);
  console.error(`  detail: ${JSON.stringify(err.detail, null, 2).slice(0, 4000)}`);
  console.error(`  artifact: ${file}`);
  console.error(`  rerun:    node tests/fuzz/crash-fuzz.js --seed ${SEED} --rounds ${ROUNDS}${HEAVY ? ' --heavy' : ''}`);
}

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// --------------------------------------------------------------------------
// main
// --------------------------------------------------------------------------
async function main() {
  console.log(`crash-fuzz: seed=${SEED} rounds=${ROUNDS} mode=${HEAVY ? 'heavy' : 'default'} port=${PORT}`);
  console.log(`  bin=${BIN}`);
  console.log(`  db=${DBFILE}`);

  const gen = makeGen(SEED, { maxDocs: MAX_DOCS, batchMax: BATCH_MAX, maxCycleOps: MAX_CYCLE_OPS, killProb: KILL_PROB, killDelayMs: KILL_DELAY_MS });

  if (!KEEP_LOG) {
    fs.rmSync(DBFILE, { force: true });
    fs.rmSync(DBFILE + '.data', { force: true }); // stale data files hit the 64GB reservation ceiling
  }
  await startServer(true);
  let base = new Model();

  if (args.debug) {
    const iv = setInterval(() => {
      if (server) console.error(`[${Date.now()} watch server pid=${server.pid} exited=${JSON.stringify(exited)}]`);
      else console.error(`[${Date.now()} watch server=null exited=${JSON.stringify(exited)}]`);
    }, 100);
    iv.unref();
  }

  try {
    for (let c = 1; c <= ROUNDS; c++) {
      let client = await connectClient();
      const r = await runCycle(client, base, gen); // ends with the server killed
      await closeClient(client);

      await startServer(false); // reopen: replay must never refuse
      client = await connectClient();
      const v = await verify(client, base, r, c);
      await closeClient(client);
      base = new Model();
      for (const [k, d] of v.docs) base.docs.set(k, d);
      base.indexCreated = v.indexCreated;

      if (VERBOSE || c === 1 || c % 10 === 0 || c === ROUNDS) {
        let size = 0, dsize = 0;
        try { size = fs.statSync(DBFILE).size; } catch {}
        try { dsize = fs.statSync(DBFILE + '.data').size; } catch {}
        console.log(`  cycle ${c}/${ROUNDS}: ops=${r.flat.length} acked=${r.acked} sent=${r.sentCount} kill=${r.killMode} prefix=${r.matched} docs=${base.docs.size} log=${(size / 1e6).toFixed(1)}MB data=${(dsize / 1e6).toFixed(0)}MB`);
      }
    }
  } catch (err) {
    if (err instanceof Fail) {
      dumpArtifact(err, gen);
      process.exit(1);
    }
    console.error(`\n✗ harness error: ${err.stack || err}`);
    console.error(`  server exited: ${JSON.stringify(exited)}`);
    console.error(`  serverLog: ${serverLog.slice(-2000)}`);
    process.exit(1);
  } finally {
    cleanup();
  }

  console.log(`crash-fuzz: PASS — ${ROUNDS} crash/reopen cycles, prefix invariant held every time`);
}

main().catch((e) => { console.error(e.stack || e); process.exit(1); });