MultiforaDB/tests/e2e/bench-run.sh

#!/bin/bash
# Full benchmark run, saved for iteration-to-iteration comparison.
#
#   bash tests/e2e/bench-run.sh [size] [doc-size] [clients...]
#   (defaults: 1g, 16k, "1 4 8 16 32"; clients apply to the concurrent phase)
#
# Runs the compare-run.sh main suite (multiforadb vs mongod, same driver,
# durable writes) plus the concurrent durable-write comparison, then writes
# a versioned machine-readable report to tests/e2e/results/ and prints a
# diff of the multiforadb numbers against the previous run.
set -u
cd "$(dirname "$0")/../.."
SIZE="${1:-1g}"; DOC="${2:-16k}"; CLIENTS="${3:-1 4 8 16 32}"
PER_CLIENT=2000
RESDIR=tests/e2e/results
mkdir -p "$RESDIR"
NOW=$(date +%Y%m%d-%H%M%S)
REV=$(git rev-parse --short HEAD 2>/dev/null || echo unknown)
DIRTY=$(git status --porcelain | grep -q . && echo "+dirty" || echo "")
REPORT="$RESDIR/bench-$NOW.txt"
LATEST="$RESDIR/bench-latest.txt"
TMP=$(mktemp -d /tmp/bench-run.XXXXXX)
trap 'rm -rf "$TMP"' EXIT

# ---- main suite ----------------------------------------------------------
bash tests/e2e/compare-run.sh "$SIZE" "$DOC" > "$TMP/compare.out" 2>&1
if [ $? -ne 0 ]; then
  echo "main suite failed — see $TMP/compare.out" >&2
  tail -20 "$TMP/compare.out" >&2
  exit 1
fi

# ---- concurrent durable writes --------------------------------------------
echo "### concurrent durable insertOne (clients: $CLIENTS)" >&2
rm -rf "$TMP/mongod" && mkdir -p "$TMP/mongod"
mongod --dbpath "$TMP/mongod" --port 27018 --bind_ip 127.0.0.1 --quiet >"$TMP/md.out" 2>&1 &
MD_PID=$!
./zig-out/bin/multiforadb --port 27019 --db "$TMP/mfdb.log" --compact-threshold 1g >"$TMP/mfdb.out" 2>&1 &
MFDB_PID=$!
# Poll both servers with the real driver until they answer (fresh mongod
# dbpaths can take several seconds; a fixed sleep is flaky).
wait_ready() { # $1 = url
  for _ in $(seq 1 90); do
    NODE_PATH="tests/e2e/node_modules" node -e "require('mongodb').MongoClient.connect(process.argv[1],{serverSelectionTimeoutMS:800}).then(c=>c.close().then(()=>process.exit(0))).catch(()=>process.exit(1))" "$1" 2>/dev/null \
      && return 0
    sleep 1
  done
  return 1
}
wait_ready mongodb://127.0.0.1:27018 || { echo "mongod never became ready" >&2; kill -9 $MD_PID $MFDB_PID 2>/dev/null; exit 1; }
wait_ready mongodb://127.0.0.1:27019 || { echo "multiforadb never became ready" >&2; kill -9 $MD_PID $MFDB_PID 2>/dev/null; exit 1; }
CC="$TMP/cc.txt"
for C in $CLIENTS; do
  MD=$(node tests/e2e/concurrent.js --url mongodb://127.0.0.1:27018 --label mongodb --clients "$C" --per-client "$PER_CLIENT" 2>/dev/null | tail -1)
  MFDB=$(node tests/e2e/concurrent.js --url mongodb://127.0.0.1:27019 --label multiforadb --clients "$C" --per-client "$PER_CLIENT" 2>/dev/null | tail -1)
  MFDB_D=$(echo "$MFDB" | sed -E 's/.*\t([0-9.]+) docs\/s.*/\1/')
  MD_D=$(echo "$MD" | sed -E 's/.*\t([0-9.]+) docs\/s.*/\1/')
  if [ -z "$MFDB_D" ] || [ -z "$MD_D" ]; then
    echo "WARNING: concurrent run at $C clients produced no result (mfdb='$MFDB' md='$MD')" >&2
    DEGRADED=1
  fi
  RATIO=$(node -e "const m=Number('$MFDB_D'),d=Number('$MD_D');console.log(d>0?(m/d).toFixed(1)+'x':'—')")
  echo "clients	$C	$MFDB_D	$MD_D	$RATIO" | tee -a "$CC"
done
kill -9 $MD_PID $MFDB_PID 2>/dev/null; wait 2>/dev/null

# ---- assemble the report --------------------------------------------------
{
  echo "# multiforadb vs MongoDB benchmark"
  echo "# date: $(date -u +%Y-%m-%dT%H:%M:%SZ)  git: $REV$DIRTY"
  echo "# args: size=$SIZE doc-size=$DOC wc=j clients=$CLIENTS per-client=$PER_CLIENT"
  echo "# reproduce: bash tests/e2e/bench-run.sh $SIZE $DOC \"$CLIENTS\""
  echo
  echo "[main]"
  node -e '
    const fs = require("fs");
    const read = (p) => {
      const m = {};
      for (const line of fs.readFileSync(p, "utf8").split("\n")) {
        const i = line.indexOf("\t");
        if (i > 0) m[line.slice(0, i)] = line.slice(i + 1).replace(/\t.*$/, "");
      }
      return m;
    };
    const a = read("/tmp/mongo-cmp/mfdb-report.txt");
    const b = read("/tmp/mongo-cmp/mongo-report.txt");
    const keys = ["insertOne (sequential) ×200","bulk insert throughput","docs loaded","createIndex({k: 1})",
      "countDocuments({})","findOne({_id: <ObjectId>})","findOne({k: 500}) (indexed)","find({p: {$gte,$lt}}).count() (scan)",
      "find({}).sort({_id:-1}).limit(20)","find({}, {proj}).limit(1000)","aggregate $group by k",
      "updateOne({_id}) ×50","updateMany({k: 7}, {$inc})","deleteOne({_id}) + insertOne","node client RSS"];
    for (const k of keys) console.log(`${k}\t${a[k] || "—"}\t${b[k] || "—"}`);
  '
  echo
  echo "[concurrency]"
  cat "$CC"
  echo
  echo "[meta]"
  node -e '
    const fs = require("fs");
    const m = JSON.parse(fs.readFileSync("/tmp/mongo-cmp/meta.json", "utf8"));
    for (const [k, v] of Object.entries(m)) console.log(`${k}\t${v}`);
  '
} > "$REPORT"

# ---- diff against the previous run ---------------------------------------
echo "report: $REPORT"
if [ -f "$LATEST" ]; then
  echo; echo "### multiforadb numbers vs previous run ($(head -2 "$LATEST" | tail -1 | sed 's/# //'))"
  node -e '
    const fs = require("fs");
    const old = fs.readFileSync(process.argv[1], "utf8");
    const neu = fs.readFileSync(process.argv[2], "utf8");
    const section = (txt, name) => {
      const i = txt.indexOf("[" + name + "]");
      if (i < 0) return new Map();
      const body = txt.slice(i).split(/\n\[/)[0];
      const m = new Map();
      for (const line of body.split("\n")) {
        const p = line.split("\t");
        if (p.length >= 3 && !line.startsWith("#")) {
          // concurrency rows are "clients\t<N>\t<docs/s>..." — key on the
          // client count, not the literal "clients".
          const key = p[0] === "clients" ? p[0] + p[1] : p[0];
          const val = p[0] === "clients" ? p[2] : p[1];
          m.set(key, val);
        }
      }
      return m;
    };
    const o = section(old, "main"), n = section(neu, "main");
    const fmt = (v) => v.padEnd(20);
    console.log(`benchmark`.padEnd(42) + fmt("old") + fmt("new") + "delta");
    for (const [k, v] of n) {
      if (!o.has(k)) continue;
      const ov = o.get(k);
      const num = (s) => parseFloat(s);
      const d = isFinite(num(ov)) && isFinite(num(v)) && num(ov) > 0 ? ((num(v) - num(ov)) / num(ov) * 100).toFixed(0) + "%" : "";
      console.log(k.padEnd(42) + fmt(ov) + fmt(v) + d);
    }
    // concurrency (multiforadb docs/s per client count)
    const oc = section(old, "concurrency"), nc = section(neu, "concurrency");
    console.log("\nconcurrency — multiforadb docs/s:");
    for (const [k, v] of nc) {
      const ov = oc.get(k);
      const d = ov && Number(ov) > 0 ? ((Number(v) - Number(ov)) / Number(ov) * 100).toFixed(0) + "%" : "";
      console.log(`clients ${k.slice(7)}:`.padEnd(12) + `${ov ?? "—"}`.padEnd(14) + `${v}`.padEnd(14) + d);
    }
  ' "$LATEST" "$REPORT"
else
  echo "(no previous run to diff — this is the baseline)"
fi
# bench-latest.txt is the baseline the next run diffs against, and PLAN D7.5
# makes "no phase8 row regresses" a milestone gate -- so a run that only
# half-produced its numbers must not become the thing we compare to. This used
# to be an unconditional copy, which meant a degraded run silently replaced the
# baseline it was supposed to be measured against, and the missing rows then
# read as "no change" forever.
if [ "${DEGRADED:-0}" = "1" ]; then
  echo >&2
  echo "NOT updating $LATEST: this run was degraded (see WARNING above)." >&2
  echo "The report is still at $REPORT if you want it." >&2
  exit 1
fi
cp "$REPORT" "$LATEST"
echo; echo "latest: $LATEST"