Aleksey Shakhmatov
ac464f2b92
compare-run.sh answers "how do we compare to MongoDB"; it says nothing
about whether a change made things better or worse than last week. Add a
harness that records each run and diffs it against the previous one.
bench-run.sh wraps compare-run.sh, adds a concurrent durable-write
comparison (concurrent.js: N clients each doing sequential insertOne with
{w:1, j:true}, exercising the group-commit path under real contention),
writes a versioned name<TAB>value report to results/bench-<timestamp>.txt,
and prints a diff of our numbers against results/bench-latest.txt.
Also:
- compare-run.sh polled with fixed sleeps, which are flaky once earlier
benchmark phases have warmed the machine; both servers now wait on a
real driver connection instead.
- a dispatch error only reached the client as a generic InternalError,
with nothing on the server side naming the failing command; log the
connection, command and error name before replacing the reply.
130 lines
6.4 KiB
Bash
130 lines
6.4 KiB
Bash
#!/bin/bash
|
||
# Compare mongo-lite against a real MongoDB with the same workload, same driver.
|
||
#
|
||
# bash tests/e2e/compare-run.sh [size] [doc-size]
|
||
# (defaults: 1g, 16k)
|
||
#
|
||
# Starts mongod on :27018 and mongo-lite on :27019, runs compare.js against
|
||
# each (durable writes: mongo-lite fsyncs per doc, mongod ack'd with j:true),
|
||
# measures kill -9 reopen time for both, and prints a side-by-side table.
|
||
set -u
|
||
cd "$(dirname "$0")/../.."
|
||
SIZE="${1:-1g}"
|
||
DOC="${2:-16k}"
|
||
echo "comparing mongo-lite vs mongodb — dataset ${SIZE}, docs ~${DOC}"
|
||
|
||
CMPDIR=/tmp/mongo-cmp
|
||
mkdir -p "$CMPDIR/mongod"
|
||
ML_LOG="$CMPDIR/ml.log"
|
||
ML_OUT="$CMPDIR/ml-srv.out"
|
||
MD_OUT="$CMPDIR/md-srv.out"
|
||
ML_PORT=27019
|
||
MD_PORT=27018
|
||
rm -f "$ML_LOG"
|
||
rm -rf "$CMPDIR/mongod" && mkdir -p "$CMPDIR/mongod"
|
||
|
||
# ---- MongoDB -------------------------------------------------------------
|
||
echo; echo "### mongod (MongoDB $(mongod --version | grep -oE 'v[0-9.]+' | head -1))"
|
||
mongod --dbpath "$CMPDIR/mongod" --port $MD_PORT --bind_ip 127.0.0.1 \
|
||
--quiet >"$MD_OUT" 2>&1 &
|
||
MD_PID=$!
|
||
# Poll until mongod answers; a fixed sleep is flaky right after other
|
||
# benchmark phases have warmed the machine.
|
||
wait_ready() { # $1 = url
|
||
for _ in $(seq 1 90); do
|
||
NODE_PATH="tests/e2e/node_modules" node -e "require('mongodb').MongoClient.connect(process.argv[1],{serverSelectionTimeoutMS:800}).then(c=>c.close().then(()=>process.exit(0))).catch(()=>process.exit(1))" "$1" 2>/dev/null \
|
||
&& return 0
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
wait_ready "mongodb://127.0.0.1:$MD_PORT" || { echo "mongod never became ready" >&2; exit 1; }
|
||
node tests/e2e/compare.js --url "mongodb://127.0.0.1:$MD_PORT" --label mongodb --size "$SIZE" --doc-size "$DOC" \
|
||
> "$CMPDIR/mongo-report.txt" 2>&1 || { echo "mongodb bench failed:"; tail -5 "$CMPDIR/mongo-report.txt"; }
|
||
MD_RSS=$(ps -o rss= -p $MD_PID | awk '{printf "%.0f", $1/1024}')
|
||
MD_DISK=$(du -sm "$CMPDIR/mongod" | awk '{print $1}')
|
||
|
||
echo; echo "### mongod kill -9 + reopen"
|
||
kill -9 $MD_PID; wait $MD_PID 2>/dev/null
|
||
MD_REOPEN=$(cd tests/e2e && node -e '
|
||
const { spawn } = require("child_process");
|
||
const { MongoClient } = require("mongodb");
|
||
const t0 = Date.now();
|
||
const p = spawn("mongod", ["--dbpath","/tmp/mongo-cmp/mongod","--port","27018","--bind_ip","127.0.0.1","--quiet"], {stdio:"ignore"});
|
||
const poll = async () => {
|
||
const c = new MongoClient("mongodb://127.0.0.1:27018", {serverSelectionTimeoutMS: 800});
|
||
try { await c.connect(); await c.db("admin").command({ping:1}); await c.close(); p.kill("SIGKILL"); console.log(((Date.now()-t0)/1000).toFixed(1)); }
|
||
catch { try { await c.close(); } catch {}; setTimeout(poll, 200); }
|
||
};
|
||
setTimeout(poll, 300);
|
||
')
|
||
kill -9 $MD_PID 2>/dev/null
|
||
|
||
# ---- mongo-lite ----------------------------------------------------------
|
||
echo; echo "### mongo-lite (recommended config: --compact-threshold 1g)"
|
||
# Debug is ~10-200x slower (see the README's perf section) — the comparison
|
||
# must use the optimized build.
|
||
zig build -Doptimize=ReleaseFast 2>&1 | grep -c "^error" | grep -q "^0" || { echo "build failed"; exit 1; }
|
||
./zig-out/bin/mongo-lite --port $ML_PORT --db "$ML_LOG" --compact-threshold 1g >"$ML_OUT" 2>&1 &
|
||
ML_PID=$!
|
||
wait_ready "mongodb://127.0.0.1:$ML_PORT" || { echo "mongo-lite never became ready" >&2; exit 1; }
|
||
node tests/e2e/compare.js --url "mongodb://127.0.0.1:$ML_PORT" --label mongo-lite --size "$SIZE" --doc-size "$DOC" \
|
||
> "$CMPDIR/ml-report.txt" 2>&1 || { echo "mongo-lite bench failed:"; tail -5 "$CMPDIR/ml-report.txt"; }
|
||
ML_RSS=$(ps -o rss= -p $ML_PID | awk '{printf "%.0f", $1/1024}')
|
||
ML_DISK=$(du -sm "$ML_LOG" | awk '{print $1}')
|
||
|
||
echo; echo "### mongo-lite kill -9 + reopen (replay)"
|
||
kill -9 $ML_PID; wait $ML_PID 2>/dev/null
|
||
ML_REOPEN=$(cd tests/e2e && node -e '
|
||
const { spawn } = require("child_process");
|
||
const { MongoClient } = require("mongodb");
|
||
const t0 = Date.now();
|
||
const p = spawn("/Users/shkmv/workspace/sandbox/mongo-lite/zig-out/bin/mongo-lite", ["--port","27019","--db","/tmp/mongo-cmp/ml.log","--compact-threshold","1g"], {stdio:"ignore"});
|
||
const poll = async () => {
|
||
const c = new MongoClient("mongodb://127.0.0.1:27019", {serverSelectionTimeoutMS: 800});
|
||
try { await c.connect(); await c.db("admin").command({ping:1}); await c.close(); p.kill("SIGKILL"); console.log(((Date.now()-t0)/1000).toFixed(1)); }
|
||
catch { try { await c.close(); } catch {}; setTimeout(poll, 200); }
|
||
};
|
||
setTimeout(poll, 300);
|
||
')
|
||
kill -9 $ML_PID 2>/dev/null
|
||
|
||
# ---- side by side ----------------------------------------------------------
|
||
echo; echo "### side by side — ${SIZE} dataset, ~${DOC} docs"
|
||
cat > "$CMPDIR/meta.json" <<EOF
|
||
{"ml_rss_mb": "$ML_RSS", "md_rss_mb": "$MD_RSS", "ml_reopen": "${ML_REOPEN}s", "md_reopen": "${MD_REOPEN}s", "ml_disk_mb": "${ML_DISK}MB", "md_disk_mb": "${MD_DISK}MB"}
|
||
EOF
|
||
node -e '
|
||
const fs = require("fs");
|
||
const read = (p) => {
|
||
const m = {};
|
||
if (!fs.existsSync(p)) return m;
|
||
for (const line of fs.readFileSync(p, "utf8").split("\n")) {
|
||
const i = line.indexOf("\t");
|
||
if (i > 0) m[line.slice(0, i)] = line.slice(i + 1).replace(/\t.*$/, "");
|
||
}
|
||
return m;
|
||
};
|
||
const a = read("/tmp/mongo-cmp/ml-report.txt");
|
||
const b = read("/tmp/mongo-cmp/mongo-report.txt");
|
||
const meta = JSON.parse(fs.readFileSync("/tmp/mongo-cmp/meta.json", "utf8"));
|
||
const keys = ["insertOne (sequential) ×200","bulk insert throughput","docs loaded","createIndex({k: 1})",
|
||
"countDocuments({})","findOne({_id: <ObjectId>})","findOne({k: 500}) (indexed)","find({p: {$gte,$lt}}).count() (scan)",
|
||
"find({}).sort({_id:-1}).limit(20)","find({}, {proj}).limit(1000)","aggregate $group by k",
|
||
"updateOne({_id}) ×50","updateMany({k: 7}, {$inc})","deleteOne({_id}) + insertOne","node client RSS"];
|
||
const col = (v) => String(v).padEnd(22);
|
||
console.log(`${"benchmark".padEnd(42)} ${col("mongo-lite")} ${col("mongodb")} ratio`);
|
||
for (const k of keys) {
|
||
const av = a[k] || "—", bv = b[k] || "—";
|
||
const ar = parseFloat(av), br = parseFloat(bv);
|
||
const ratio = isFinite(ar) && isFinite(br) && ar > 0 && br > 0 ? (ar / br).toFixed(1) + "x" : "";
|
||
console.log(`${k.padEnd(42)} ${col(av)} ${col(bv)} ${ratio}`);
|
||
}
|
||
console.log(`${`server RSS`.padEnd(42)} ${col(meta.ml_rss_mb + " MB")} ${col(meta.md_rss_mb + " MB")}`);
|
||
console.log(`${`kill -9 reopen`.padEnd(42)} ${col(meta.ml_reopen)} ${col(meta.md_reopen)}`);
|
||
console.log(`${`db on disk`.padEnd(42)} ${col(meta.ml_disk_mb)} ${col(meta.md_disk_mb)}`);
|
||
'
|
||
|
||
pkill -9 -f "mongo-lite --port $ML_PORT" 2>/dev/null
|
||
echo; echo "done — reports: $CMPDIR/ml-report.txt, $CMPDIR/mongo-report.txt"
|