MultiforaDB/tests/e2e/compare.js

// Benchmark: the same workload through the official driver against multiforadb
// and a real MongoDB. Run once per server URL, then diff the reports.
//
//   node tests/e2e/compare.js --url mongodb://127.0.0.1:27018 --label mongodb
//   node tests/e2e/compare.js --url mongodb://127.0.0.1:27019 --label multiforadb
//
// Options:
//   --url <u>        server URL (required)
//   --label <s>      report label (default: url host:port)
//   --size <n>       dataset size, k/m/g suffixes (default 1g)
//   --doc-size <n>   bytes per document (default 16k)
//   --batch <n>      docs per insertMany (default 500)
//   --index <field>  field to index before the op benchmarks (default k)
//   --wc <j|none>    write concern: 'j' = {w:1, j:true} durable ack on every
//                    write (fair vs multiforadb's fsync-per-write); 'none' =
//                    driver default (default j)
//
// Every benchmark is awaited (no fire-and-forget), which is exactly how the
// big.js harness measures multiforadb, so the numbers are directly comparable.
const { MongoClient, ObjectId } = require('mongodb');
const fs = require('fs');
const os = require('os');

function parseSize(s) {
  const m = /^(\d+(?:\.\d+)?)([kmgt]?)$/i.exec(String(s).trim());
  if (!m) throw new Error(`bad size '${s}'`);
  const mult = { '': 1, k: 1 << 10, m: 1 << 20, g: 1 << 30, t: 1 << 40 }[m[2].toLowerCase()];
  return Math.round(parseFloat(m[1]) * mult);
}
function fmt(n) { return n >= 1e9 ? (n / 1e9).toFixed(2) + ' GB' : n >= 1e6 ? (n / 1e6).toFixed(1) + ' MB' : n >= 1e3 ? (n / 1e3).toFixed(1) + ' KB' : n + ' B'; }

let opt = { url: null, label: null, size: '1g', docSize: '16k', batch: 500, index: 'k', wc: 'j' };
for (let i = 2; i < process.argv.length; i++) {
  const a = process.argv[i];
  if (a === '--url') opt.url = process.argv[++i];
  else if (a === '--label') opt.label = process.argv[++i];
  else if (a === '--size') opt.size = process.argv[++i];
  else if (a === '--doc-size') opt.docSize = process.argv[++i];
  else if (a === '--batch') opt.batch = Number(process.argv[++i]);
  else if (a === '--index') opt.index = process.argv[++i];
  else if (a === '--wc') opt.wc = process.argv[++i];
  else { console.error(`unknown option ${a}`); process.exit(2); }
}
if (!opt.url) { console.error('--url required'); process.exit(2); }
opt.label = opt.label || opt.url.replace(/^mongodb:\/\//, '');
const SIZE = parseSize(opt.size);
const DOC_SIZE = parseSize(opt.docSize);
const WC = opt.wc === 'j' ? { writeConcern: { w: 1, j: true } } : {};

const report = [];
const row = (k, v, note = '') => { report.push([k, v, note]); console.log(`${k}\t${v}${note ? '\t' + note : ''}`); };
const fmtMs = (ms) => (ms < 1 ? ms.toFixed(2) + ' ms' : ms < 1000 ? ms.toFixed(1) + ' ms' : (ms / 1000).toFixed(2) + ' s');
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

async function bench(label, fn, min = 0) {
  const a = process.hrtime.bigint();
  const out = await fn();
  const ms = Number(process.hrtime.bigint() - a) / 1e6;
  row(label, fmtMs(ms), out ?? '');
  return ms;
}
async function benchMany(label, n, fn) {
  const times = [];
  const a = process.hrtime.bigint();
  for (let i = 0; i < n; i++) await fn();
  const total = Number(process.hrtime.bigint() - a) / 1e6;
  const sorted = times.sort((x, y) => x - y);
  row(`${label} ×${n}`, fmtMs(total / n), `total ${fmtMs(total)}`);
  return total / n;
}
function rssMB() {
  try {
    return Math.round(process.memoryUsage().rss / 1048576);
  } catch { return 0; }
}

async function main() {
  const client = new MongoClient(opt.url, { serverSelectionTimeoutMS: 15000, maxPoolSize: 4 });
  await client.connect();
  const dbName = `cmp${Date.now() % 100000}`;
  const db = client.db(dbName);
  const coll = db.collection('items');
  const payload = 'y'.repeat(Math.max(1, DOC_SIZE - 140));
  const nDocs = Math.max(1, Math.ceil(SIZE / DOC_SIZE));

  console.log(`== ${opt.label} ==  size ${fmt(SIZE)} · doc ~${fmt(DOC_SIZE)} · wc ${opt.wc}`);
  await db.command({ ping: 1 });
  row('driver handshake + ping', '');

  // ---- single-doc insert latency (sequential, durable ack) ----------------
  console.log('\n-- single-doc writes --');
  await coll.deleteMany({});
  await benchMany('insertOne (sequential)', 200, () => coll.insertOne({ _id: new ObjectId(), k: 1, p: 2, ts: new Date(), payload: 'z'.repeat(256) }, WC));
  const one = await coll.findOne({});
  row('insertOne round-trip sanity', one ? 'ok' : 'MISSING');

  // ---- bulk load ------------------------------------------------------------
  console.log('\n-- bulk load --');
  await coll.drop().catch(() => {});
  await coll.createIndex({ _id: 1 }, { unique: true }).catch(() => {});
  let t0 = Date.now();
  let inserted = 0;
  while (inserted < nDocs) {
    const n = Math.min(opt.batch, nDocs - inserted);
    const docs = new Array(n);
    for (let i = 0; i < n; i++) {
      const id = inserted + i + 1;
      docs[i] = { _id: new ObjectId(), k: id % 1000, p: id % 5000, ts: new Date(Date.UTC(2024, 0, 1) + id * 1000), payload };
    }
    await coll.insertMany(docs, Object.assign({ ordered: false }, WC));
    inserted += n;
  }
  const bulkMs = Date.now() - t0;
  row('docs loaded', inserted.toLocaleString());
  row('bytes loaded', fmt(inserted * DOC_SIZE));
  row('bulk insert throughput', `${(inserted * DOC_SIZE / 1e6 / (bulkMs / 1000)).toFixed(1)} MB/s`, `(${(inserted / (bulkMs / 1000)).toFixed(0)} docs/s, ${(bulkMs / 1000).toFixed(1)} s)`);
  let collInfo = null;
  try { collInfo = await db.command({ collStats: 'items' }); } catch {}
  if (collInfo) row('server-side data size', fmt(collInfo.size ?? 0), `storage ${fmt(collInfo.storageSize ?? 0)}`);

  // ---- secondary index (multiforadb: planner uses it; mongodb: normal) ------
  if (opt.index) {
    await bench(`createIndex({${opt.index}: 1})`, async () => { await coll.createIndex({ [opt.index]: 1 }); });
  }

  // ---- read benchmarks --------------------------------------------------------
  console.log('\n-- reads on full dataset --');
  await bench('countDocuments({})', async () => `${(await coll.countDocuments({})).toLocaleString()} docs`);
  await bench('findOne({_id: <ObjectId>})', async () => {
    const d = await coll.findOne({ _id: new ObjectId(hexOf(nDocs / 2)) }, { projection: { _id: 1 } });
    return d ? 'hit' : 'MISSING';
  });
  if (opt.index) {
    await bench(`findOne({${opt.index}: 500}) (indexed)`, async () => {
      const d = await coll.findOne({ k: 500 }, { projection: { _id: 1 } });
      return d ? 'hit' : 'MISSING';
    });
  }
  await bench('find({p: {$gte,$lt}}).count() (scan)', async () => `${(await coll.countDocuments({ p: { $gte: 1000, $lt: 3000 } })).toLocaleString()} hits`);
  await bench('find({}).sort({_id:-1}).limit(20)', async () => (await coll.find({}).sort({ _id: -1 }).limit(20).toArray()).length + ' docs');
  await bench('find({}, {proj}).limit(1000)', async () => (await coll.find({}, { projection: { payload: 0 } }).limit(1000).toArray()).length + ' docs');
  await bench('aggregate $group by k', async () => {
    const g = await coll.aggregate([{ $group: { _id: '$k', n: { $sum: 1 } } }]).toArray();
    return g.length + ' groups';
  });

  // ---- write benchmarks on the full dataset ---------------------------------
  console.log('\n-- writes on full dataset --');
  const midOid = new ObjectId(hexOf(Math.floor(nDocs / 2)));
  await benchMany('updateOne({_id})', 50, () => coll.updateOne({ _id: midOid }, { $set: { touch: 1 } }, WC));
  await bench('updateMany({k: 7}, {$inc})', async () => {
    const r = await coll.updateMany({ k: 7 }, { $inc: { hits: 1 } }, WC);
    return `${r.modifiedCount} modified`;
  });
  await bench('deleteOne({_id}) + insertOne', async () => {
    await coll.deleteOne({ _id: midOid }, WC);
    await coll.insertOne({ _id: new ObjectId(), k: 1, p: 2, payload }, WC);
  });

  row('node client RSS', `${rssMB()} MB`);

  await client.close();
  // Drop the db so a second run against the same server starts clean.
  await new MongoClient(opt.url, { serverSelectionTimeoutMS: 5000 }).connect().then(async (c) => {
    await c.db(dbName).dropDatabase();
    await c.close();
  }).catch(() => {});

  console.log('\nCOMPARE_OK');
}

// ObjectId from a deterministic 12-byte hex (so both servers see identical keys).
function hexOf(n) {
  return n.toString(16).padStart(24, '0');
}

main().catch((e) => { console.error('COMPARE_FAIL', e); process.exit(1); });