#!/bin/bash # Full benchmark run, saved for iteration-to-iteration comparison. # # bash tests/e2e/bench-run.sh [size] [doc-size] [clients...] # (defaults: 1g, 16k, "1 4 8 16 32"; clients apply to the concurrent phase) # # Runs the compare-run.sh main suite (multiforadb vs mongod, same driver, # durable writes) plus the concurrent durable-write comparison, then writes # a versioned machine-readable report to tests/e2e/results/ and prints a # diff of the multiforadb numbers against the previous run. set -u cd "$(dirname "$0")/../.." SIZE="${1:-1g}"; DOC="${2:-16k}"; CLIENTS="${3:-1 4 8 16 32}" PER_CLIENT=2000 RESDIR=tests/e2e/results mkdir -p "$RESDIR" NOW=$(date +%Y%m%d-%H%M%S) REV=$(git rev-parse --short HEAD 2>/dev/null || echo unknown) DIRTY=$(git status --porcelain | grep -q . && echo "+dirty" || echo "") REPORT="$RESDIR/bench-$NOW.txt" LATEST="$RESDIR/bench-latest.txt" TMP=$(mktemp -d /tmp/bench-run.XXXXXX) trap 'rm -rf "$TMP"' EXIT # ---- main suite ---------------------------------------------------------- bash tests/e2e/compare-run.sh "$SIZE" "$DOC" > "$TMP/compare.out" 2>&1 if [ $? -ne 0 ]; then echo "main suite failed — see $TMP/compare.out" >&2 tail -20 "$TMP/compare.out" >&2 exit 1 fi # ---- concurrent durable writes -------------------------------------------- echo "### concurrent durable insertOne (clients: $CLIENTS)" >&2 rm -rf "$TMP/mongod" && mkdir -p "$TMP/mongod" mongod --dbpath "$TMP/mongod" --port 27018 --bind_ip 127.0.0.1 --quiet >"$TMP/md.out" 2>&1 & MD_PID=$! ./zig-out/bin/multiforadb --port 27019 --db "$TMP/mfdb.log" --compact-threshold 1g >"$TMP/mfdb.out" 2>&1 & MFDB_PID=$! # Poll both servers with the real driver until they answer (fresh mongod # dbpaths can take several seconds; a fixed sleep is flaky). wait_ready() { # $1 = url for _ in $(seq 1 90); do NODE_PATH="tests/e2e/node_modules" node -e "require('mongodb').MongoClient.connect(process.argv[1],{serverSelectionTimeoutMS:800}).then(c=>c.close().then(()=>process.exit(0))).catch(()=>process.exit(1))" "$1" 2>/dev/null \ && return 0 sleep 1 done return 1 } wait_ready mongodb://127.0.0.1:27018 || { echo "mongod never became ready" >&2; kill -9 $MD_PID $MFDB_PID 2>/dev/null; exit 1; } wait_ready mongodb://127.0.0.1:27019 || { echo "multiforadb never became ready" >&2; kill -9 $MD_PID $MFDB_PID 2>/dev/null; exit 1; } CC="$TMP/cc.txt" for C in $CLIENTS; do MD=$(node tests/e2e/concurrent.js --url mongodb://127.0.0.1:27018 --label mongodb --clients "$C" --per-client "$PER_CLIENT" 2>/dev/null | tail -1) MFDB=$(node tests/e2e/concurrent.js --url mongodb://127.0.0.1:27019 --label multiforadb --clients "$C" --per-client "$PER_CLIENT" 2>/dev/null | tail -1) MFDB_D=$(echo "$MFDB" | sed -E 's/.*\t([0-9.]+) docs\/s.*/\1/') MD_D=$(echo "$MD" | sed -E 's/.*\t([0-9.]+) docs\/s.*/\1/') if [ -z "$MFDB_D" ] || [ -z "$MD_D" ]; then echo "WARNING: concurrent run at $C clients produced no result (mfdb='$MFDB' md='$MD')" >&2 fi RATIO=$(node -e "const m=Number('$MFDB_D'),d=Number('$MD_D');console.log(d>0?(m/d).toFixed(1)+'x':'—')") echo "clients $C $MFDB_D $MD_D $RATIO" | tee -a "$CC" done kill -9 $MD_PID $MFDB_PID 2>/dev/null; wait 2>/dev/null # ---- assemble the report -------------------------------------------------- { echo "# multiforadb vs MongoDB benchmark" echo "# date: $(date -u +%Y-%m-%dT%H:%M:%SZ) git: $REV$DIRTY" echo "# args: size=$SIZE doc-size=$DOC wc=j clients=$CLIENTS per-client=$PER_CLIENT" echo "# reproduce: bash tests/e2e/bench-run.sh $SIZE $DOC \"$CLIENTS\"" echo echo "[main]" node -e ' const fs = require("fs"); const read = (p) => { const m = {}; for (const line of fs.readFileSync(p, "utf8").split("\n")) { const i = line.indexOf("\t"); if (i > 0) m[line.slice(0, i)] = line.slice(i + 1).replace(/\t.*$/, ""); } return m; }; const a = read("/tmp/mongo-cmp/mfdb-report.txt"); const b = read("/tmp/mongo-cmp/mongo-report.txt"); const keys = ["insertOne (sequential) ×200","bulk insert throughput","docs loaded","createIndex({k: 1})", "countDocuments({})","findOne({_id: })","findOne({k: 500}) (indexed)","find({p: {$gte,$lt}}).count() (scan)", "find({}).sort({_id:-1}).limit(20)","find({}, {proj}).limit(1000)","aggregate $group by k", "updateOne({_id}) ×50","updateMany({k: 7}, {$inc})","deleteOne({_id}) + insertOne","node client RSS"]; for (const k of keys) console.log(`${k}\t${a[k] || "—"}\t${b[k] || "—"}`); ' echo echo "[concurrency]" cat "$CC" echo echo "[meta]" node -e ' const fs = require("fs"); const m = JSON.parse(fs.readFileSync("/tmp/mongo-cmp/meta.json", "utf8")); for (const [k, v] of Object.entries(m)) console.log(`${k}\t${v}`); ' } > "$REPORT" # ---- diff against the previous run --------------------------------------- echo "report: $REPORT" if [ -f "$LATEST" ]; then echo; echo "### multiforadb numbers vs previous run ($(head -2 "$LATEST" | tail -1 | sed 's/# //'))" node -e ' const fs = require("fs"); const old = fs.readFileSync(process.argv[1], "utf8"); const neu = fs.readFileSync(process.argv[2], "utf8"); const section = (txt, name) => { const i = txt.indexOf("[" + name + "]"); if (i < 0) return new Map(); const body = txt.slice(i).split(/\n\[/)[0]; const m = new Map(); for (const line of body.split("\n")) { const p = line.split("\t"); if (p.length >= 3 && !line.startsWith("#")) { // concurrency rows are "clients\t\t..." — key on the // client count, not the literal "clients". const key = p[0] === "clients" ? p[0] + p[1] : p[0]; const val = p[0] === "clients" ? p[2] : p[1]; m.set(key, val); } } return m; }; const o = section(old, "main"), n = section(neu, "main"); const fmt = (v) => v.padEnd(20); console.log(`benchmark`.padEnd(42) + fmt("old") + fmt("new") + "delta"); for (const [k, v] of n) { if (!o.has(k)) continue; const ov = o.get(k); const num = (s) => parseFloat(s); const d = isFinite(num(ov)) && isFinite(num(v)) && num(ov) > 0 ? ((num(v) - num(ov)) / num(ov) * 100).toFixed(0) + "%" : ""; console.log(k.padEnd(42) + fmt(ov) + fmt(v) + d); } // concurrency (multiforadb docs/s per client count) const oc = section(old, "concurrency"), nc = section(neu, "concurrency"); console.log("\nconcurrency — multiforadb docs/s:"); for (const [k, v] of nc) { const ov = oc.get(k); const d = ov && Number(ov) > 0 ? ((Number(v) - Number(ov)) / Number(ov) * 100).toFixed(0) + "%" : ""; console.log(`clients ${k.slice(7)}:`.padEnd(12) + `${ov ?? "—"}`.padEnd(14) + `${v}`.padEnd(14) + d); } ' "$LATEST" "$REPORT" else echo "(no previous run to diff — this is the baseline)" fi cp "$REPORT" "$LATEST" echo; echo "latest: $LATEST"