tests/spec: MongoDB spec-test runner and the M0 scorecard
PLAN D2 makes the official specification suites the gate for command semantics; D7.6 asks for the harness to exist at M0 with a recorded baseline. This is that harness, pinned on both sides -- mongodb/specifications @ 615e0f9 and mongodb@7.5.0 -- because a scorecard is only comparable across milestones if a delta cannot be an upstream test change. It implements the unified format's Evaluating Matches algorithm as written, including the two rules that decide whether a pass is earned: extra keys are tolerated only in a root document, and numeric types compare flexibly. Anything unimplemented is a SKIP with a reason, never a pass, and the one assertion class not yet checked -- expectEvents, i.e. command monitoring -- is disclosed at the top of the scorecard so `pass` reads as an upper bound. First honest run: 131 pass, 161 fail, 195 skip over 175 files, zero timeouts. Getting there took four attempts, and the failures are documented in the README because each would have shipped a scorecard claiming a compatibility gap that did not exist. Two were genuine leaks in this runner (clients left open when a case timed out; clients registered for cleanup only after `await connect()`, plus abandoned cases still creating more). The third I misdiagnosed as machine load. The fourth attempt found the real cause: a leaked catalog lock in the engine, fixed separately, which alone accounts for the jump from 45 passes to 131. So the runner carries its own guards: per-operation CSOT timeouts so work is never abandoned, an active-handle census per file, an end-of-run tripwire for stray timers, a hard stop if the server dies rather than emitting hundreds of misleading ECONNREFUSED failures, and --skip/--limit for bisecting a run whose failures depend on position. The README states the rule plainly -- a long unbroken tail of timeouts is a harness bug until proven otherwise -- and the two commands that settle it. Also fixes bench-run.sh, which copied its report over bench-latest.txt unconditionally, including after a run that only warned -- so a degraded run could silently replace the baseline that PLAN D7.5 makes a milestone gate.
This commit is contained in:
743
tests/spec/run.js
Normal file
743
tests/spec/run.js
Normal file
@@ -0,0 +1,743 @@
|
||||
'use strict';
|
||||
// MongoDB unified-test-format runner, pointed at MultiforaDB.
|
||||
//
|
||||
// PLAN D2 makes the official spec suites the gate for command semantics, and
|
||||
// PLAN D7.6 asks for the harness to exist at M0 with a red baseline -- so this
|
||||
// is deliberately a *measuring* tool, not a passing one. What it must never do
|
||||
// is report a pass it did not earn: a compatibility scorecard that flatters
|
||||
// the engine is worse than no scorecard. Everything unimplemented is reported
|
||||
// as SKIP with a reason, never as PASS.
|
||||
//
|
||||
// bash tests/spec/fetch.sh # pinned suites
|
||||
// node tests/spec/run.js # spawns its own server
|
||||
// node tests/spec/run.js --url mongodb://127.0.0.1:27020 # external server
|
||||
// node tests/spec/run.js --file find.json --verbose
|
||||
// node tests/spec/run.js --scorecard # rewrite tests/spec/scorecard.txt
|
||||
//
|
||||
// Matching follows the pseudo-code in the spec's "Evaluating Matches"
|
||||
// section verbatim, including the two rules that are easy to get wrong and
|
||||
// that decide whether a result is a real pass: extra keys are tolerated only
|
||||
// in a *root* document, and numeric types compare flexibly.
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { spawn } = require('child_process');
|
||||
|
||||
const DRIVER = path.join(__dirname, '..', 'e2e', 'node_modules', 'mongodb');
|
||||
const { MongoClient } = require(DRIVER);
|
||||
const { EJSON, Long, Int32, Double, Decimal128, ObjectId, Binary, Timestamp } = require(path.join(DRIVER, 'lib', 'bson.js'));
|
||||
|
||||
const SUITE_DIR = path.join(__dirname, 'specifications', 'source', 'crud', 'tests', 'unified');
|
||||
const SCORECARD = path.join(__dirname, 'scorecard.txt');
|
||||
const REPO = path.join(__dirname, '..', '..');
|
||||
|
||||
// The runner implements schema 1.0-1.9 features that CRUD tests actually use.
|
||||
// A file declaring more than this is skipped whole rather than half-run.
|
||||
const MAX_SCHEMA = [1, 24];
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
function opt(name, dflt) {
|
||||
const i = argv.indexOf('--' + name);
|
||||
if (i < 0) return dflt;
|
||||
const v = argv[i + 1];
|
||||
return v === undefined || v.startsWith('--') ? true : v;
|
||||
}
|
||||
const VERBOSE = !!opt('verbose', false);
|
||||
const ONLY_FILE = opt('file', null);
|
||||
const WRITE_SCORECARD = !!opt('scorecard', false);
|
||||
const EXTERNAL_URL = opt('url', null);
|
||||
const PORT = parseInt(opt('port', '27222'), 10);
|
||||
const BIN = process.env.MFDB_BIN || path.join(REPO, 'zig-out', 'bin', 'multiforadb');
|
||||
const DB_PATH = process.env.MFDB_SPEC_DB || path.join(REPO, '.zig-cache', 'spec.log');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Server lifecycle (mirrors tests/e2e/e2e6.js: poll with a real ping, and
|
||||
// treat an exited child as a hard failure so we cannot silently measure a
|
||||
// stale server on the same port).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let server = null;
|
||||
let serverDead = false;
|
||||
let serverExit = null;
|
||||
let serverOut = [];
|
||||
// Persisted, not just buffered in memory: when the server dies mid-run its
|
||||
// panic is the only thing that explains the hundreds of ECONNREFUSED failures
|
||||
// that follow, and an in-memory buffer is lost if the runner itself is killed.
|
||||
const SERVER_LOG = path.join(REPO, '.zig-cache', 'spec-server.out');
|
||||
|
||||
function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
||||
|
||||
// A single case must not be able to stall the baseline run. A timeout is
|
||||
// reported as a FAIL, not a SKIP: "the engine never answered" is a result, and
|
||||
// hiding it in the skip column would overstate compatibility.
|
||||
//
|
||||
// Per-operation, enforced by the driver (CSOT `timeoutMS`) so the promise
|
||||
// settles instead of being abandoned. The budget exists because some commands
|
||||
// produce no reply at all rather than an error, and a driver would otherwise
|
||||
// wait indefinitely.
|
||||
//
|
||||
// 10 s, not the 3 s that seems ample for an in-process engine on loopback
|
||||
// where a clean suite file runs in well under a second. A tight budget makes
|
||||
// the scorecard a function of machine load rather than of the engine: with a
|
||||
// concurrent `zig build test` (which compiles and runs multi-threaded tests),
|
||||
// a 3 s budget turned 166 perfectly good cases into timeout FAILs, and the
|
||||
// same files passed alone. A generous budget costs wall-clock only on cases
|
||||
// that genuinely hang, which are few. Don't tighten it to speed up a run.
|
||||
const OP_TIMEOUT_MS = parseInt(opt('op-timeout-ms', '10000'), 10);
|
||||
// Backstop only, for a hang the driver's own timeout cannot see. It should
|
||||
// essentially never fire -- when it does, the case is abandoned, and abandoned
|
||||
// work is what corrupted two earlier baselines, so it is deliberately far
|
||||
// above OP_TIMEOUT_MS rather than close to it.
|
||||
const CASE_TIMEOUT_MS = parseInt(opt('case-timeout-ms', '60000'), 10);
|
||||
|
||||
// Live handle census. Two baselines were wrecked by the runner accumulating
|
||||
// something across a single process's 175 files, and a count taken only at the
|
||||
// end could not say *when* it started. Printed per file so the transition is
|
||||
// visible in the log.
|
||||
function resourceTag() {
|
||||
if (!process.getActiveResourcesInfo) return '';
|
||||
const counts = {};
|
||||
for (const r of process.getActiveResourcesInfo()) counts[r] = (counts[r] || 0) + 1;
|
||||
const interesting = Object.entries(counts)
|
||||
.filter(([k]) => k !== 'Immediate' && k !== 'TTYWrap' && k !== 'ProcessWrap' && k !== 'PipeWrap')
|
||||
.map(([k, v]) => `${k}=${v}`)
|
||||
.join(' ');
|
||||
return interesting ? ` [${interesting}]` : '';
|
||||
}
|
||||
|
||||
// Server round-trip time, so "the engine is getting slower as the run goes on"
|
||||
// is a number in the log rather than a guess. Also counts databases, since
|
||||
// state accumulating on the server is the obvious candidate.
|
||||
async function pingTag() {
|
||||
try {
|
||||
const t0 = process.hrtime.bigint();
|
||||
await harness.db('admin').command({ ping: 1 });
|
||||
const ms = Number(process.hrtime.bigint() - t0) / 1e6;
|
||||
let ndbs = '?';
|
||||
try {
|
||||
const r = await harness.db('admin').command({ listDatabases: 1, nameOnly: true });
|
||||
ndbs = String((r.databases || []).length);
|
||||
} catch (e) {
|
||||
ndbs = `ERR(${e.name}: ${String(e.message).slice(0, 60)})`;
|
||||
}
|
||||
return `ping=${ms.toFixed(1)}ms dbs=${ndbs}`;
|
||||
} catch (e) {
|
||||
return `ping=FAILED(${e.name})`;
|
||||
}
|
||||
}
|
||||
|
||||
async function withTimeout(fn, ms, msg) {
|
||||
let timer;
|
||||
const guard = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(msg)), ms); });
|
||||
try { return await Promise.race([fn(), guard]); } finally { clearTimeout(timer); }
|
||||
}
|
||||
|
||||
async function startServer() {
|
||||
try { fs.unlinkSync(DB_PATH); } catch (e) { /* first run */ }
|
||||
if (!fs.existsSync(BIN)) throw new Error(`${BIN} missing - run \`zig build\` first`);
|
||||
server = spawn(BIN, ['--port', String(PORT), '--db', DB_PATH, '--ttl-sweep-secs', '0'], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
try { fs.unlinkSync(SERVER_LOG); } catch (e) { /* first run */ }
|
||||
const sink = fs.createWriteStream(SERVER_LOG, { flags: 'a' });
|
||||
const grab = (b) => { sink.write(b); serverOut.push(b.toString()); if (serverOut.length > 400) serverOut.shift(); };
|
||||
server.stdout.on('data', grab);
|
||||
server.stderr.on('data', grab);
|
||||
server.on('exit', (code, signal) => { serverDead = true; serverExit = signal ? `signal ${signal}` : `exit code ${code}`; });
|
||||
|
||||
const url = `mongodb://127.0.0.1:${PORT}`;
|
||||
const deadline = Date.now() + 20000;
|
||||
while (Date.now() < deadline) {
|
||||
if (serverDead) throw new Error('server exited during startup:\n' + serverOut.join(''));
|
||||
try {
|
||||
const c = new MongoClient(url, { serverSelectionTimeoutMS: 500 });
|
||||
await c.connect();
|
||||
await c.db('admin').command({ ping: 1 });
|
||||
await c.close();
|
||||
return url;
|
||||
} catch (e) { await sleep(100); }
|
||||
}
|
||||
throw new Error('server did not become ready:\n' + serverOut.join(''));
|
||||
}
|
||||
|
||||
function stopServer() {
|
||||
if (server && !serverDead) server.kill('SIGKILL');
|
||||
}
|
||||
process.on('exit', stopServer);
|
||||
process.on('SIGINT', () => { stopServer(); process.exit(130); });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Matching -- the spec's Evaluating Matches algorithm.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
class MatchError extends Error {}
|
||||
|
||||
function fail(pathStr, msg) { throw new MatchError(`${pathStr || '<root>'}: ${msg}`); }
|
||||
|
||||
function isPlainDoc(v) {
|
||||
if (v === null || typeof v !== 'object' || Array.isArray(v)) return false;
|
||||
// BSON scalars are objects too; they are values, not documents.
|
||||
return !(v instanceof ObjectId || v instanceof Binary || v instanceof Long ||
|
||||
v instanceof Int32 || v instanceof Double || v instanceof Decimal128 ||
|
||||
v instanceof Timestamp || v instanceof Date || v instanceof RegExp);
|
||||
}
|
||||
|
||||
function specialKey(doc) {
|
||||
if (!isPlainDoc(doc)) return null;
|
||||
const ks = Object.keys(doc);
|
||||
return ks.length === 1 && ks[0].startsWith('$$') ? ks[0] : null;
|
||||
}
|
||||
|
||||
function numeric(v) {
|
||||
if (typeof v === 'number') return v;
|
||||
if (v instanceof Int32 || v instanceof Double) return v.valueOf();
|
||||
if (v instanceof Long) return Number(v.toString());
|
||||
return null;
|
||||
}
|
||||
|
||||
// The spec's type aliases, restricted to what the CRUD suites use.
|
||||
function matchesType(actual, alias) {
|
||||
switch (alias) {
|
||||
case 'double': return typeof actual === 'number' || actual instanceof Double;
|
||||
case 'int': return actual instanceof Int32 || (typeof actual === 'number' && Number.isInteger(actual));
|
||||
case 'long': return actual instanceof Long || (typeof actual === 'number' && Number.isInteger(actual));
|
||||
case 'string': return typeof actual === 'string';
|
||||
case 'object': return isPlainDoc(actual);
|
||||
case 'array': return Array.isArray(actual);
|
||||
case 'binData': return actual instanceof Binary;
|
||||
case 'objectId': return actual instanceof ObjectId;
|
||||
case 'bool': return typeof actual === 'boolean';
|
||||
case 'date': return actual instanceof Date;
|
||||
case 'null': return actual === null;
|
||||
case 'regex': return actual instanceof RegExp;
|
||||
case 'timestamp': return actual instanceof Timestamp;
|
||||
case 'decimal': return actual instanceof Decimal128;
|
||||
case 'number': return numeric(actual) !== null || actual instanceof Decimal128;
|
||||
default: return null; // unknown alias -> caller reports unsupported
|
||||
}
|
||||
}
|
||||
|
||||
function scalarEqual(expected, actual) {
|
||||
const en = numeric(expected), an = numeric(actual);
|
||||
if (en !== null && an !== null) return en === an; // flexible numerics
|
||||
if (expected === null) return actual === null;
|
||||
if (expected instanceof Date) return actual instanceof Date && +expected === +actual;
|
||||
if (expected instanceof ObjectId) return actual instanceof ObjectId && expected.equals(actual);
|
||||
if (expected instanceof Binary) return actual instanceof Binary && Buffer.compare(expected.buffer, actual.buffer) === 0 && expected.sub_type === actual.sub_type;
|
||||
if (expected instanceof Decimal128) return actual instanceof Decimal128 && expected.toString() === actual.toString();
|
||||
if (expected instanceof Timestamp) return actual instanceof Timestamp && expected.equals(actual);
|
||||
if (expected instanceof RegExp) return actual instanceof RegExp && String(expected) === String(actual);
|
||||
if (typeof expected !== typeof actual) return false;
|
||||
return expected === actual;
|
||||
}
|
||||
|
||||
// `root` implements the one rule that separates a real pass from a lenient
|
||||
// one: only a root document may carry keys the expectation does not mention.
|
||||
function match(expected, actual, entities, pathStr = '', root = true) {
|
||||
const sk = specialKey(expected);
|
||||
if (sk) return special(sk, expected[sk], actual, entities, pathStr, true);
|
||||
|
||||
if (isPlainDoc(expected)) {
|
||||
if (!isPlainDoc(actual)) fail(pathStr, `expected a document, got ${describe(actual)}`);
|
||||
for (const [k, v] of Object.entries(expected)) {
|
||||
const kp = pathStr ? `${pathStr}.${k}` : k;
|
||||
const vsk = specialKey(v);
|
||||
if (vsk) {
|
||||
const consumed = special(vsk, v[vsk], actual[k], entities, kp, Object.prototype.hasOwnProperty.call(actual, k));
|
||||
if (!consumed) continue;
|
||||
continue;
|
||||
}
|
||||
if (!Object.prototype.hasOwnProperty.call(actual, k)) fail(kp, 'missing from actual');
|
||||
match(v, actual[k], entities, kp, false);
|
||||
}
|
||||
if (!root) {
|
||||
const extra = Object.keys(actual).filter((k) => !Object.prototype.hasOwnProperty.call(expected, k));
|
||||
if (extra.length) fail(pathStr, `unexpected extra keys ${JSON.stringify(extra)}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (Array.isArray(expected)) {
|
||||
if (!Array.isArray(actual)) fail(pathStr, `expected an array, got ${describe(actual)}`);
|
||||
if (expected.length !== actual.length) fail(pathStr, `expected ${expected.length} elements, got ${actual.length}`);
|
||||
// Array elements are not root documents: `expectResult: [ {...} ]` from
|
||||
// find must match its documents exactly, extra fields included.
|
||||
expected.forEach((e, i) => match(e, actual[i], entities, `${pathStr}[${i}]`, false));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!scalarEqual(expected, actual)) fail(pathStr, `expected ${describe(expected)}, got ${describe(actual)}`);
|
||||
}
|
||||
|
||||
// Returns whether the actual value still has to be matched by the caller.
|
||||
function special(op, arg, actual, entities, pathStr, present) {
|
||||
switch (op) {
|
||||
case '$$exists':
|
||||
if (arg && !present) fail(pathStr, 'expected the key to exist');
|
||||
if (!arg && present) fail(pathStr, 'expected the key to be absent');
|
||||
return false;
|
||||
case '$$type': {
|
||||
const aliases = Array.isArray(arg) ? arg : [arg];
|
||||
const results = aliases.map((a) => matchesType(actual, a));
|
||||
if (results.some((r) => r === null)) throw new Unsupported(`$$type alias ${JSON.stringify(arg)}`);
|
||||
if (!results.some((r) => r === true)) fail(pathStr, `expected type ${JSON.stringify(arg)}, got ${describe(actual)}`);
|
||||
return false;
|
||||
}
|
||||
case '$$unsetOrMatches':
|
||||
if (!present || actual === undefined) return false;
|
||||
match(arg, actual, entities, pathStr, false);
|
||||
return false;
|
||||
case '$$matchesEntity': {
|
||||
if (!(arg in entities.map)) fail(pathStr, `entity ${arg} not found`);
|
||||
match(entities.map[arg], actual, entities, pathStr, false);
|
||||
return false;
|
||||
}
|
||||
case '$$matchesHexBytes':
|
||||
if (!(actual instanceof Binary)) fail(pathStr, 'expected binary data');
|
||||
if (actual.buffer.toString('hex') !== String(arg).toLowerCase()) fail(pathStr, 'hex bytes differ');
|
||||
return false;
|
||||
default:
|
||||
throw new Unsupported(`match operator ${op}`);
|
||||
}
|
||||
}
|
||||
|
||||
function describe(v) {
|
||||
if (v === undefined) return 'undefined';
|
||||
if (v === null) return 'null';
|
||||
try { return EJSON.stringify(v, { relaxed: true }); } catch (e) { return String(v); }
|
||||
}
|
||||
|
||||
class Unsupported extends Error {}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Operations
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Argument keys the spec passes positionally rather than as driver options.
|
||||
const POSITIONAL = new Set(['filter', 'document', 'documents', 'update', 'replacement', 'pipeline', 'fieldName', 'models', 'requests', 'keys', 'name', 'indexes', 'command', 'session', 'entity', 'to']);
|
||||
|
||||
function options(args, drop = []) {
|
||||
const o = {};
|
||||
for (const [k, v] of Object.entries(args || {})) {
|
||||
if (POSITIONAL.has(k) || drop.includes(k)) continue;
|
||||
o[k] = v;
|
||||
}
|
||||
return Object.keys(o).length ? o : undefined;
|
||||
}
|
||||
|
||||
function requireNoSession(args) {
|
||||
if (args && args.session) throw new Unsupported('explicit sessions (M4)');
|
||||
}
|
||||
|
||||
async function runOperation(op, entities) {
|
||||
const args = op.arguments || {};
|
||||
requireNoSession(args);
|
||||
const target = entities.map[op.object];
|
||||
if (op.object === 'testRunner') throw new Unsupported(`testRunner operation ${op.name}`);
|
||||
if (target === undefined) throw new Unsupported(`entity ${op.object}`);
|
||||
|
||||
switch (op.name) {
|
||||
// -- read ----------------------------------------------------------
|
||||
case 'find': return await target.find(args.filter || {}, options(args)).toArray();
|
||||
case 'findOne': return await target.findOne(args.filter || {}, options(args));
|
||||
case 'aggregate': return await target.aggregate(args.pipeline, options(args)).toArray();
|
||||
case 'countDocuments': return await target.countDocuments(args.filter || {}, options(args));
|
||||
case 'estimatedDocumentCount': return await target.estimatedDocumentCount(options(args));
|
||||
case 'distinct': return await target.distinct(args.fieldName, args.filter || {}, options(args));
|
||||
case 'listIndexes': return await target.listIndexes(options(args)).toArray();
|
||||
case 'listIndexNames': return await target.indexes(options(args)).then((ix) => ix.map((i) => i.name));
|
||||
case 'listCollections': return await target.listCollections(args.filter || {}, options(args)).toArray();
|
||||
case 'listCollectionNames': return await target.listCollections(args.filter || {}, options(args)).toArray().then((cs) => cs.map((c) => c.name));
|
||||
case 'runCommand': return await target.command(args.command, options(args, ['commandName']));
|
||||
|
||||
// -- write ---------------------------------------------------------
|
||||
case 'insertOne': return plain(await target.insertOne(args.document, options(args)));
|
||||
case 'insertMany': return plain(await target.insertMany(args.documents, options(args)));
|
||||
case 'updateOne': return plain(await target.updateOne(args.filter, args.update, options(args)));
|
||||
case 'updateMany': return plain(await target.updateMany(args.filter, args.update, options(args)));
|
||||
case 'replaceOne': return plain(await target.replaceOne(args.filter, args.replacement, options(args)));
|
||||
case 'deleteOne': return plain(await target.deleteOne(args.filter, options(args)));
|
||||
case 'deleteMany': return plain(await target.deleteMany(args.filter, options(args)));
|
||||
case 'bulkWrite': return plain(await target.bulkWrite(args.requests, options(args)));
|
||||
case 'findOneAndUpdate': return unwrapFam(await target.findOneAndUpdate(args.filter, args.update, options(args)));
|
||||
case 'findOneAndReplace': return unwrapFam(await target.findOneAndReplace(args.filter, args.replacement, options(args)));
|
||||
case 'findOneAndDelete': return unwrapFam(await target.findOneAndDelete(args.filter, options(args)));
|
||||
|
||||
// -- collection / index management ----------------------------------
|
||||
case 'createIndex': return await target.createIndex(args.keys, options(args));
|
||||
case 'dropIndex': return await target.dropIndex(args.name, options(args));
|
||||
case 'createCollection': return void (await target.createCollection(args.collection, options(args, ['collection'])));
|
||||
case 'dropCollection': return void (await target.dropCollection(args.collection, options(args, ['collection'])));
|
||||
case 'assertCollectionExists': case 'assertCollectionNotExists':
|
||||
throw new Unsupported(`testRunner operation ${op.name}`);
|
||||
default:
|
||||
throw new Unsupported(`operation ${op.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Driver result objects carry class instances and `acknowledged`; the spec
|
||||
// expects plain documents. Turning them into plain objects keeps the
|
||||
// root-document rule meaningful (extra keys tolerated only at the root).
|
||||
function plain(r) {
|
||||
if (r === null || typeof r !== 'object') return r;
|
||||
const o = {};
|
||||
for (const k of Object.keys(r)) {
|
||||
if (k === 'acknowledged') continue;
|
||||
o[k] = r[k];
|
||||
}
|
||||
if (typeof r.getUpsertedIds === 'function') {
|
||||
// BulkWriteResult exposes counts through getters, not own keys.
|
||||
for (const k of ['insertedCount', 'matchedCount', 'modifiedCount', 'deletedCount', 'upsertedCount', 'insertedIds', 'upsertedIds']) {
|
||||
if (r[k] !== undefined) o[k] = r[k];
|
||||
}
|
||||
}
|
||||
return o;
|
||||
}
|
||||
|
||||
// Driver 5+ returns the document itself; 4.x wrapped it in `{value}`. e2e.js
|
||||
// tolerates both the same way (`fam.value ?? fam`).
|
||||
function unwrapFam(r) {
|
||||
if (r && typeof r === 'object' && 'value' in r && 'ok' in r) return r.value;
|
||||
return r;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// runOnRequirements
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function cmpVersion(a, b) {
|
||||
const pa = String(a).split('.').map(Number), pb = String(b).split('.').map(Number);
|
||||
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
||||
const x = pa[i] || 0, y = pb[i] || 0;
|
||||
if (x !== y) return x < y ? -1 : 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function unmetRequirement(reqs, server) {
|
||||
if (!reqs) return null;
|
||||
const reasons = [];
|
||||
for (const r of reqs) {
|
||||
const why = [];
|
||||
if (r.minServerVersion && cmpVersion(server.version, r.minServerVersion) < 0) why.push(`needs server >= ${r.minServerVersion}`);
|
||||
if (r.maxServerVersion && cmpVersion(server.version, r.maxServerVersion) > 0) why.push(`needs server <= ${r.maxServerVersion}`);
|
||||
if (r.topologies && !r.topologies.includes('single')) why.push(`needs topology ${r.topologies.join('/')}`);
|
||||
if (r.auth === true) why.push('needs auth (M7)');
|
||||
if (r.csfle) why.push('needs csfle');
|
||||
if (r.serverless === 'require') why.push('needs serverless');
|
||||
if (r.serverParameters) why.push('needs server parameters');
|
||||
if (why.length === 0) return null; // this alternative is satisfied
|
||||
reasons.push(why.join(', '));
|
||||
}
|
||||
return reasons[0] || 'unmet requirement';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// One file
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// One client for the whole run, used for seeding and outcome checks.
|
||||
//
|
||||
// It used to be one client per seed and per outcome check. That, plus per-test
|
||||
// entity clients that leaked whenever a case timed out before buildEntities
|
||||
// returned, meant hundreds of live MongoClients accumulated -- each with its
|
||||
// own pool and heartbeat timer. Node's event loop starved, the per-case timer
|
||||
// then fired before operations could complete, and every case from that point
|
||||
// on "failed" with a timeout. A ping from a separate process still answered,
|
||||
// which made it look convincingly like a server-side wedge. It was the runner.
|
||||
//
|
||||
// The lesson worth keeping: a harness whose own failures look like engine
|
||||
// failures will silently invent a compatibility gap. Hence also the assertion
|
||||
// below that clients never accumulate.
|
||||
let harness = null;
|
||||
|
||||
async function seedInitialData(initialData) {
|
||||
if (!initialData) return;
|
||||
for (const spec of initialData) {
|
||||
const coll = harness.db(spec.databaseName).collection(spec.collectionName);
|
||||
await coll.drop().catch(() => { });
|
||||
if (spec.documents && spec.documents.length) await coll.insertMany(spec.documents);
|
||||
else await harness.db(spec.databaseName).createCollection(spec.collectionName).catch(() => { });
|
||||
}
|
||||
}
|
||||
|
||||
// `clients` is supplied by the caller so that entities created before a
|
||||
// failure are still closed: returning them only on success is what leaked.
|
||||
async function buildEntities(url, createEntities, clients) {
|
||||
const map = {};
|
||||
for (const spec of createEntities || []) {
|
||||
const [kind, def] = Object.entries(spec)[0];
|
||||
switch (kind) {
|
||||
case 'client': {
|
||||
if (def.useMultipleMongoses === true) { /* single topology: harmless */ }
|
||||
// `timeoutMS` (CSOT) makes each operation reject on its own
|
||||
// rather than being abandoned by an outer race. That matters
|
||||
// more than it looks: an abandoned operation keeps running,
|
||||
// keeps its client alive, and -- since it may still be inside
|
||||
// buildEntities -- can create further clients *after* cleanup
|
||||
// has already run. That is what leaked, and what turned into
|
||||
// 190 phantom timeout FAILs.
|
||||
const c = new MongoClient(url, Object.assign({
|
||||
serverSelectionTimeoutMS: 2000,
|
||||
connectTimeoutMS: 2000,
|
||||
timeoutMS: OP_TIMEOUT_MS,
|
||||
}, def.uriOptions || {}));
|
||||
// Registered before connect, so a client whose connect throws
|
||||
// or is abandoned is still closed by the caller.
|
||||
clients.push(c);
|
||||
await c.connect();
|
||||
if (clients.abandoned) {
|
||||
// Cleanup already ran; this client would otherwise linger.
|
||||
await c.close().catch(() => { });
|
||||
throw new Error('case abandoned');
|
||||
}
|
||||
map[def.id] = c;
|
||||
break;
|
||||
}
|
||||
case 'database': map[def.id] = map[def.client].db(def.databaseName); break;
|
||||
case 'collection': map[def.id] = map[def.database].collection(def.collectionName); break;
|
||||
case 'session': throw new Unsupported('session entities (M4)');
|
||||
case 'bucket': throw new Unsupported('gridfs bucket entities');
|
||||
case 'clientEncryption': throw new Unsupported('client-side encryption');
|
||||
default: throw new Unsupported(`entity type ${kind}`);
|
||||
}
|
||||
}
|
||||
return { map, clients };
|
||||
}
|
||||
|
||||
async function verifyOutcome(outcome, entities) {
|
||||
if (!outcome) return;
|
||||
for (const spec of outcome) {
|
||||
const actual = await harness.db(spec.databaseName).collection(spec.collectionName)
|
||||
.find({}, { sort: { _id: 1 } }).toArray();
|
||||
match(spec.documents, actual, entities, `outcome ${spec.databaseName}.${spec.collectionName}`, false);
|
||||
}
|
||||
}
|
||||
|
||||
async function runFile(file, url, server) {
|
||||
const text = fs.readFileSync(path.join(SUITE_DIR, file), 'utf8');
|
||||
const doc = EJSON.parse(text, { relaxed: false });
|
||||
const out = { file, pass: 0, fail: 0, skip: 0, cases: [] };
|
||||
|
||||
const schema = String(doc.schemaVersion || '1.0').split('.').map(Number);
|
||||
if (schema[0] > MAX_SCHEMA[0] || (schema[0] === MAX_SCHEMA[0] && (schema[1] || 0) > MAX_SCHEMA[1])) {
|
||||
out.skip = (doc.tests || []).length;
|
||||
out.cases.push({ name: '*', status: 'SKIP', reason: `schemaVersion ${doc.schemaVersion} > ${MAX_SCHEMA.join('.')}` });
|
||||
return out;
|
||||
}
|
||||
const fileUnmet = unmetRequirement(doc.runOnRequirements, server);
|
||||
if (fileUnmet) {
|
||||
out.skip = (doc.tests || []).length;
|
||||
out.cases.push({ name: '*', status: 'SKIP', reason: fileUnmet });
|
||||
return out;
|
||||
}
|
||||
|
||||
for (const test of doc.tests || []) {
|
||||
const name = test.description;
|
||||
if (test.skipReason) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: 'upstream skipReason: ' + test.skipReason }); continue; }
|
||||
const unmet = unmetRequirement(test.runOnRequirements, server);
|
||||
if (unmet) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: unmet }); continue; }
|
||||
|
||||
// Owned out here, not by buildEntities, so a case that dies partway
|
||||
// still has every client it managed to open closed below.
|
||||
const clients = [];
|
||||
try {
|
||||
await withTimeout(async () => {
|
||||
await seedInitialData(doc.initialData);
|
||||
const entities = await buildEntities(url, doc.createEntities, clients);
|
||||
for (const op of test.operations) await runOne(op, entities);
|
||||
await verifyOutcome(test.outcome, entities);
|
||||
}, CASE_TIMEOUT_MS, `case exceeded ${CASE_TIMEOUT_MS} ms`);
|
||||
out.pass++;
|
||||
out.cases.push({ name, status: 'PASS' });
|
||||
} catch (e) {
|
||||
if (e instanceof Unsupported) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: 'runner: ' + e.message }); }
|
||||
else { out.fail++; out.cases.push({ name, status: 'FAIL', reason: (e instanceof MatchError ? '' : (e.constructor.name + ': ')) + e.message.split('\n')[0].slice(0, 220) }); }
|
||||
} finally {
|
||||
// Unconditional, and the flag matters: if the case was abandoned,
|
||||
// its continuation may still be running and about to open another
|
||||
// client, which buildEntities closes itself on seeing this.
|
||||
clients.abandoned = true;
|
||||
for (const c of clients) await c.close().catch(() => { });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function runOne(op, entities) {
|
||||
if (op.name === 'failPoint' || op.name === 'targetedFailPoint') throw new Unsupported('failPoint');
|
||||
let result, err = null;
|
||||
try {
|
||||
result = await runOperation(op, entities);
|
||||
} catch (e) {
|
||||
if (e instanceof Unsupported) throw e;
|
||||
err = e;
|
||||
}
|
||||
|
||||
if (op.expectError) {
|
||||
if (!err) fail(op.name, 'expected an error, the operation succeeded');
|
||||
checkError(op.expectError, err, entities, op.name);
|
||||
return;
|
||||
}
|
||||
if (err) throw err;
|
||||
if (op.ignoreResultAndError) return;
|
||||
if ('expectResult' in op) match(op.expectResult, result, entities, op.name, true);
|
||||
if (op.saveResultAsEntity) entities.map[op.saveResultAsEntity] = result;
|
||||
}
|
||||
|
||||
function checkError(exp, err, entities, where) {
|
||||
if (exp.isError === false) fail(where, 'expected no error');
|
||||
if (exp.isClientError === true && err.code !== undefined && err.code !== null) {
|
||||
fail(where, `expected a client-side error, got server code ${err.code}`);
|
||||
}
|
||||
if (exp.errorContains) {
|
||||
const hay = String(err.message || '').toLowerCase();
|
||||
if (!hay.includes(String(exp.errorContains).toLowerCase())) fail(where, `error message ${JSON.stringify(String(err.message).slice(0, 120))} does not contain ${JSON.stringify(exp.errorContains)}`);
|
||||
}
|
||||
if (exp.errorCode !== undefined) {
|
||||
const actual = numeric(err.code);
|
||||
if (actual !== numeric(exp.errorCode)) fail(where, `expected error code ${exp.errorCode}, got ${err.code}`);
|
||||
}
|
||||
if (exp.errorCodeName !== undefined) {
|
||||
const actual = err.codeName || (err.result && err.result.codeName);
|
||||
if (actual !== exp.errorCodeName) fail(where, `expected codeName ${exp.errorCodeName}, got ${actual}`);
|
||||
}
|
||||
if (exp.errorLabelsContain) {
|
||||
for (const l of exp.errorLabelsContain) if (!(err.errorLabels || []).includes(l)) fail(where, `expected error label ${l}`);
|
||||
}
|
||||
if (exp.errorLabelsOmit) {
|
||||
for (const l of exp.errorLabelsOmit) if ((err.errorLabels || []).includes(l)) fail(where, `expected no error label ${l}`);
|
||||
}
|
||||
if (exp.errorResponse) {
|
||||
const resp = err.result || err.errorResponse || err;
|
||||
match(exp.errorResponse, resp, entities, where + '.errorResponse', true);
|
||||
}
|
||||
if (exp.writeErrors || exp.writeConcernErrors) throw new Unsupported('clientBulkWrite error assertions');
|
||||
if ('expectResult' in exp) {
|
||||
const r = err.result || (err.writeErrors ? plain(err) : undefined);
|
||||
if (r === undefined) throw new Unsupported('expectResult on an error the driver does not expose');
|
||||
match(exp.expectResult, r, entities, where + '.expectResult', true);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
(async () => {
|
||||
if (!fs.existsSync(SUITE_DIR)) {
|
||||
console.error(`missing ${SUITE_DIR}\nrun: bash tests/spec/fetch.sh`);
|
||||
process.exit(2);
|
||||
}
|
||||
let files = fs.readdirSync(SUITE_DIR).filter((f) => f.endsWith('.json'))
|
||||
.filter((f) => !ONLY_FILE || f === ONLY_FILE || f === ONLY_FILE + '.json')
|
||||
.sort();
|
||||
// --skip/--limit exist for bisecting a run whose failures depend on
|
||||
// position rather than content, which has happened more than once.
|
||||
const skip = parseInt(opt('skip', '0'), 10);
|
||||
if (skip > 0) files = files.slice(skip);
|
||||
const limit = parseInt(opt('limit', '0'), 10);
|
||||
if (limit > 0) files = files.slice(0, limit);
|
||||
if (!files.length) { console.error('no matching suite files'); process.exit(2); }
|
||||
|
||||
const url = EXTERNAL_URL || await startServer();
|
||||
|
||||
harness = new MongoClient(url, { serverSelectionTimeoutMS: 3000, connectTimeoutMS: 3000, socketTimeoutMS: 8000 });
|
||||
await harness.connect();
|
||||
const bi = await harness.db('admin').command({ buildInfo: 1 }).catch(() => ({ version: '0.0.0' }));
|
||||
const hello = await harness.db('admin').command({ hello: 1 }).catch(() => ({}));
|
||||
const server = { version: bi.version || '0.0.0', maxWireVersion: hello.maxWireVersion };
|
||||
|
||||
const results = [];
|
||||
let died = null;
|
||||
for (const f of files) {
|
||||
// A dead server turns every remaining case into ECONNREFUSED, which
|
||||
// would land in the scorecard as ~700 engine failures and bury the one
|
||||
// fact that matters: it crashed, and where. Stop at the crash instead.
|
||||
if (serverDead && !EXTERNAL_URL) {
|
||||
died = { after: results.length ? results[results.length - 1].file : '<startup>', next: f };
|
||||
break;
|
||||
}
|
||||
let r;
|
||||
try {
|
||||
r = await runFile(f, url, server);
|
||||
} catch (e) {
|
||||
r = { file: f, pass: 0, fail: 0, skip: 0, cases: [{ name: '*', status: 'ERROR', reason: e.message.split('\n')[0].slice(0, 200) }], errored: true };
|
||||
}
|
||||
results.push(r);
|
||||
const tag = r.errored ? 'ERROR' : `${r.pass} pass, ${r.fail} fail, ${r.skip} skip`;
|
||||
console.log(`${f.padEnd(48)} ${tag}${resourceTag()} ${await pingTag()}`);
|
||||
if (VERBOSE) for (const c of r.cases) if (c.status !== 'PASS') console.log(` ${c.status.padEnd(5)} ${c.name}${c.reason ? ' -- ' + c.reason : ''}`);
|
||||
}
|
||||
|
||||
const tot = results.reduce((a, r) => ({ pass: a.pass + r.pass, fail: a.fail + r.fail, skip: a.skip + r.skip }), { pass: 0, fail: 0, skip: 0 });
|
||||
const errored = results.filter((r) => r.errored).length;
|
||||
console.log(`\ncrud+aggregate: ${tot.pass} pass, ${tot.fail} fail, ${tot.skip} skip across ${results.length}/${files.length} files (${errored} files errored)`);
|
||||
|
||||
if (died) {
|
||||
console.error(`\n*** THE SERVER DIED (${serverExit}) ***`);
|
||||
console.error(`last file completed: ${died.after}; would have run next: ${died.next}`);
|
||||
console.error(`server output: ${path.relative(REPO, SERVER_LOG)}`);
|
||||
console.error('--- tail ---\n' + serverOut.slice(-40).join(''));
|
||||
console.error('Refusing to write a scorecard from a partial run.');
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
if (WRITE_SCORECARD) {
|
||||
fs.writeFileSync(SCORECARD, scorecardText(results, tot, errored, server, files.length));
|
||||
console.log(`wrote ${path.relative(REPO, SCORECARD)}`);
|
||||
}
|
||||
// A runner leak once turned into ~77 phantom "engine timeouts" (see the
|
||||
// comment on `harness`). This is the tripwire for that class of bug: if the
|
||||
// harness is the only client left standing, nothing accumulated.
|
||||
const leaked = process.getActiveResourcesInfo
|
||||
? process.getActiveResourcesInfo().filter((r) => r === 'Timeout').length
|
||||
: 0;
|
||||
if (leaked > 8) {
|
||||
console.error(`\nWARNING: ${leaked} timers still active at the end of the run.`);
|
||||
console.error('That is how a client leak looks; treat the timeout FAILs above as suspect.');
|
||||
}
|
||||
await harness.close(true).catch(() => { });
|
||||
|
||||
// The runner's own exit code reports whether it *ran*, not whether the
|
||||
// engine passed -- M0's gate is a recorded baseline, and a red baseline is
|
||||
// the expected state. A non-zero exit here would make it useless as a gate.
|
||||
console.log('SPEC_RUNNER_OK');
|
||||
stopServer();
|
||||
process.exit(0);
|
||||
})().catch((e) => {
|
||||
console.error('spec runner failed:', e);
|
||||
if (serverOut.length) console.error('--- server output ---\n' + serverOut.slice(-30).join(''));
|
||||
stopServer();
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
function scorecardText(results, tot, errored, server, nfiles) {
|
||||
const L = [];
|
||||
L.push('# MongoDB spec-test scorecard (PLAN D2) -- crud + aggregate, unified format');
|
||||
L.push(`# specs: mongodb/specifications @ 615e0f9 (pinned in tests/spec/fetch.sh)`);
|
||||
L.push(`# driver: mongodb@${require(path.join(DRIVER, 'package.json')).version} (pinned in tests/e2e/package-lock.json)`);
|
||||
L.push(`# server: MultiforaDB reporting version ${server.version}, maxWireVersion ${server.maxWireVersion}`);
|
||||
L.push('# reproduce: bash tests/spec/fetch.sh && node tests/spec/run.js --scorecard');
|
||||
L.push('#');
|
||||
L.push('# What SKIP means here, so the totals are not read as better than they are:');
|
||||
L.push('# - the suite needs a feature whose milestone has not landed (sessions M4,');
|
||||
L.push('# auth M7, failPoints, gridfs) -- counted as skip, never as pass;');
|
||||
L.push('# - or the runner itself does not implement the operation/matcher yet.');
|
||||
L.push('# Deliberately NOT asserted yet: expectEvents (command monitoring). Those');
|
||||
L.push('# assertions are about driver-visible command shape rather than result');
|
||||
L.push('# semantics; ignoring them makes some cases pass that a full runner would');
|
||||
L.push('# fail, so treat `pass` as an upper bound until M1 wires events up.');
|
||||
L.push('');
|
||||
L.push(`total\t${tot.pass} pass\t${tot.fail} fail\t${tot.skip} skip\t${nfiles} files\t${errored} errored`);
|
||||
L.push('');
|
||||
L.push('# per-file: name\tpass\tfail\tskip');
|
||||
for (const r of results) L.push(`${r.file}\t${r.pass}\t${r.fail}\t${r.skip}${r.errored ? '\terrored' : ''}`);
|
||||
L.push('');
|
||||
L.push('# every non-passing case, with its reason');
|
||||
for (const r of results) {
|
||||
for (const c of r.cases) {
|
||||
if (c.status === 'PASS') continue;
|
||||
L.push(`${r.file}\t${c.status}\t${c.name}\t${c.reason || ''}`);
|
||||
}
|
||||
}
|
||||
return L.join('\n') + '\n';
|
||||
}
|
||||
Reference in New Issue
Block a user