Aleksey Shakhmatov
ac464f2b92
compare-run.sh answers "how do we compare to MongoDB"; it says nothing
about whether a change made things better or worse than last week. Add a
harness that records each run and diffs it against the previous one.
bench-run.sh wraps compare-run.sh, adds a concurrent durable-write
comparison (concurrent.js: N clients each doing sequential insertOne with
{w:1, j:true}, exercising the group-commit path under real contention),
writes a versioned name<TAB>value report to results/bench-<timestamp>.txt,
and prints a diff of our numbers against results/bench-latest.txt.
Also:
- compare-run.sh polled with fixed sleeps, which are flaky once earlier
benchmark phases have warmed the machine; both servers now wait on a
real driver connection instead.
- a dispatch error only reached the client as a generic InternalError,
with nothing on the server side naming the failing command; log the
connection, command and error name before replacing the reply.
153 lines
6.8 KiB
Bash
153 lines
6.8 KiB
Bash
#!/bin/bash
|
||
# Full benchmark run, saved for iteration-to-iteration comparison.
|
||
#
|
||
# bash tests/e2e/bench-run.sh [size] [doc-size] [clients...]
|
||
# (defaults: 1g, 16k, "1 4 8 16 32"; clients apply to the concurrent phase)
|
||
#
|
||
# Runs the compare-run.sh main suite (mongo-lite vs mongod, same driver,
|
||
# durable writes) plus the concurrent durable-write comparison, then writes
|
||
# a versioned machine-readable report to tests/e2e/results/ and prints a
|
||
# diff of the mongo-lite numbers against the previous run.
|
||
set -u
|
||
cd "$(dirname "$0")/../.."
|
||
SIZE="${1:-1g}"; DOC="${2:-16k}"; CLIENTS="${3:-1 4 8 16 32}"
|
||
PER_CLIENT=2000
|
||
RESDIR=tests/e2e/results
|
||
mkdir -p "$RESDIR"
|
||
NOW=$(date +%Y%m%d-%H%M%S)
|
||
REV=$(git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
||
DIRTY=$(git status --porcelain | grep -q . && echo "+dirty" || echo "")
|
||
REPORT="$RESDIR/bench-$NOW.txt"
|
||
LATEST="$RESDIR/bench-latest.txt"
|
||
TMP=$(mktemp -d /tmp/bench-run.XXXXXX)
|
||
trap 'rm -rf "$TMP"' EXIT
|
||
|
||
# ---- main suite ----------------------------------------------------------
|
||
bash tests/e2e/compare-run.sh "$SIZE" "$DOC" > "$TMP/compare.out" 2>&1
|
||
if [ $? -ne 0 ]; then
|
||
echo "main suite failed — see $TMP/compare.out" >&2
|
||
tail -20 "$TMP/compare.out" >&2
|
||
exit 1
|
||
fi
|
||
|
||
# ---- concurrent durable writes --------------------------------------------
|
||
echo "### concurrent durable insertOne (clients: $CLIENTS)" >&2
|
||
rm -rf "$TMP/mongod" && mkdir -p "$TMP/mongod"
|
||
mongod --dbpath "$TMP/mongod" --port 27018 --bind_ip 127.0.0.1 --quiet >"$TMP/md.out" 2>&1 &
|
||
MD_PID=$!
|
||
./zig-out/bin/mongo-lite --port 27019 --db "$TMP/ml.log" --compact-threshold 1g >"$TMP/ml.out" 2>&1 &
|
||
ML_PID=$!
|
||
# Poll both servers with the real driver until they answer (fresh mongod
|
||
# dbpaths can take several seconds; a fixed sleep is flaky).
|
||
wait_ready() { # $1 = url
|
||
for _ in $(seq 1 90); do
|
||
NODE_PATH="tests/e2e/node_modules" node -e "require('mongodb').MongoClient.connect(process.argv[1],{serverSelectionTimeoutMS:800}).then(c=>c.close().then(()=>process.exit(0))).catch(()=>process.exit(1))" "$1" 2>/dev/null \
|
||
&& return 0
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
wait_ready mongodb://127.0.0.1:27018 || { echo "mongod never became ready" >&2; kill -9 $MD_PID $ML_PID 2>/dev/null; exit 1; }
|
||
wait_ready mongodb://127.0.0.1:27019 || { echo "mongo-lite never became ready" >&2; kill -9 $MD_PID $ML_PID 2>/dev/null; exit 1; }
|
||
CC="$TMP/cc.txt"
|
||
for C in $CLIENTS; do
|
||
MD=$(node tests/e2e/concurrent.js --url mongodb://127.0.0.1:27018 --label mongodb --clients "$C" --per-client "$PER_CLIENT" 2>/dev/null | tail -1)
|
||
ML=$(node tests/e2e/concurrent.js --url mongodb://127.0.0.1:27019 --label mongo-lite --clients "$C" --per-client "$PER_CLIENT" 2>/dev/null | tail -1)
|
||
ML_D=$(echo "$ML" | sed -E 's/.*\t([0-9.]+) docs\/s.*/\1/')
|
||
MD_D=$(echo "$MD" | sed -E 's/.*\t([0-9.]+) docs\/s.*/\1/')
|
||
if [ -z "$ML_D" ] || [ -z "$MD_D" ]; then
|
||
echo "WARNING: concurrent run at $C clients produced no result (ml='$ML' md='$MD')" >&2
|
||
fi
|
||
RATIO=$(node -e "const m=Number('$ML_D'),d=Number('$MD_D');console.log(d>0?(m/d).toFixed(1)+'x':'—')")
|
||
echo "clients $C $ML_D $MD_D $RATIO" | tee -a "$CC"
|
||
done
|
||
kill -9 $MD_PID $ML_PID 2>/dev/null; wait 2>/dev/null
|
||
|
||
# ---- assemble the report --------------------------------------------------
|
||
{
|
||
echo "# mongo-lite vs MongoDB benchmark"
|
||
echo "# date: $(date -u +%Y-%m-%dT%H:%M:%SZ) git: $REV$DIRTY"
|
||
echo "# args: size=$SIZE doc-size=$DOC wc=j clients=$CLIENTS per-client=$PER_CLIENT"
|
||
echo "# reproduce: bash tests/e2e/bench-run.sh $SIZE $DOC \"$CLIENTS\""
|
||
echo
|
||
echo "[main]"
|
||
node -e '
|
||
const fs = require("fs");
|
||
const read = (p) => {
|
||
const m = {};
|
||
for (const line of fs.readFileSync(p, "utf8").split("\n")) {
|
||
const i = line.indexOf("\t");
|
||
if (i > 0) m[line.slice(0, i)] = line.slice(i + 1).replace(/\t.*$/, "");
|
||
}
|
||
return m;
|
||
};
|
||
const a = read("/tmp/mongo-cmp/ml-report.txt");
|
||
const b = read("/tmp/mongo-cmp/mongo-report.txt");
|
||
const keys = ["insertOne (sequential) ×200","bulk insert throughput","docs loaded","createIndex({k: 1})",
|
||
"countDocuments({})","findOne({_id: <ObjectId>})","findOne({k: 500}) (indexed)","find({p: {$gte,$lt}}).count() (scan)",
|
||
"find({}).sort({_id:-1}).limit(20)","find({}, {proj}).limit(1000)","aggregate $group by k",
|
||
"updateOne({_id}) ×50","updateMany({k: 7}, {$inc})","deleteOne({_id}) + insertOne","node client RSS"];
|
||
for (const k of keys) console.log(`${k}\t${a[k] || "—"}\t${b[k] || "—"}`);
|
||
'
|
||
echo
|
||
echo "[concurrency]"
|
||
cat "$CC"
|
||
echo
|
||
echo "[meta]"
|
||
node -e '
|
||
const fs = require("fs");
|
||
const m = JSON.parse(fs.readFileSync("/tmp/mongo-cmp/meta.json", "utf8"));
|
||
for (const [k, v] of Object.entries(m)) console.log(`${k}\t${v}`);
|
||
'
|
||
} > "$REPORT"
|
||
|
||
# ---- diff against the previous run ---------------------------------------
|
||
echo "report: $REPORT"
|
||
if [ -f "$LATEST" ]; then
|
||
echo; echo "### mongo-lite numbers vs previous run ($(head -2 "$LATEST" | tail -1 | sed 's/# //'))"
|
||
node -e '
|
||
const fs = require("fs");
|
||
const old = fs.readFileSync(process.argv[1], "utf8");
|
||
const neu = fs.readFileSync(process.argv[2], "utf8");
|
||
const section = (txt, name) => {
|
||
const i = txt.indexOf("[" + name + "]");
|
||
if (i < 0) return new Map();
|
||
const body = txt.slice(i).split(/\n\[/)[0];
|
||
const m = new Map();
|
||
for (const line of body.split("\n")) {
|
||
const p = line.split("\t");
|
||
if (p.length >= 3 && !line.startsWith("#")) {
|
||
// concurrency rows are "clients\t<N>\t<docs/s>..." — key on the
|
||
// client count, not the literal "clients".
|
||
const key = p[0] === "clients" ? p[0] + p[1] : p[0];
|
||
const val = p[0] === "clients" ? p[2] : p[1];
|
||
m.set(key, val);
|
||
}
|
||
}
|
||
return m;
|
||
};
|
||
const o = section(old, "main"), n = section(neu, "main");
|
||
const fmt = (v) => v.padEnd(20);
|
||
console.log(`benchmark`.padEnd(42) + fmt("old") + fmt("new") + "delta");
|
||
for (const [k, v] of n) {
|
||
if (!o.has(k)) continue;
|
||
const ov = o.get(k);
|
||
const num = (s) => parseFloat(s);
|
||
const d = isFinite(num(ov)) && isFinite(num(v)) && num(ov) > 0 ? ((num(v) - num(ov)) / num(ov) * 100).toFixed(0) + "%" : "";
|
||
console.log(k.padEnd(42) + fmt(ov) + fmt(v) + d);
|
||
}
|
||
// concurrency (mongo-lite docs/s per client count)
|
||
const oc = section(old, "concurrency"), nc = section(neu, "concurrency");
|
||
console.log("\nconcurrency — mongo-lite docs/s:");
|
||
for (const [k, v] of nc) {
|
||
const ov = oc.get(k);
|
||
const d = ov && Number(ov) > 0 ? ((Number(v) - Number(ov)) / Number(ov) * 100).toFixed(0) + "%" : "";
|
||
console.log(`clients ${k.slice(7)}:`.padEnd(12) + `${ov ?? "—"}`.padEnd(14) + `${v}`.padEnd(14) + d);
|
||
}
|
||
' "$LATEST" "$REPORT"
|
||
else
|
||
echo "(no previous run to diff — this is the baseline)"
|
||
fi
|
||
cp "$REPORT" "$LATEST"
|
||
echo; echo "latest: $LATEST"
|