7f426cdd33
`buildEntities` built every collection as `db.collection(name)` and every
database as `client.db(name)`, dropping `collectionOptions` and
`databaseOptions` on the floor. 15 collection entities declare a
`writeConcern`, 7 a `readConcern`, one a `readPreference` -- and the 15 are all
`{w: 0}`, so every "unacknowledged write" case in this corpus has been running
an acknowledged write against a driver that was never told otherwise.
They passed anyway, because an acknowledged and an unacknowledged write of the
same document produce results a `$$unsetOrMatches` expectation accepts either
way. Only the command on the wire distinguished them, and nothing was reading
the command until the previous commits. This is the first thing the event
assertions found, and it is a fair answer to what they cost.
Option documents are unwrapped from their BSON types on the way to the driver.
The suites are parsed with `relaxed: false`, so `{w: 0}` arrives as an Int32
and the driver gates `writeConcern.w` on `typeof w === 'number'` -- the same
trap NUMERIC_OPTIONS already documents for operation options, and a silent one:
the option would simply not apply. Wholesale unwrapping is safe here in a way
it is not there, since these are settings the driver consumes rather than
values an assertion compares. An option key outside the spec's
`collectionOrDatabaseOptions` set is reported unsupported rather than ignored,
which is the lesson of the bug itself.
159/132/196 becomes 173/118/196. 14 cases fixed, none broken.
The other 10 unacknowledged cases now fail differently, and that is progress
of a sort: with `w: 0` actually applied, the driver refuses client-side to send
`hint` on a delete or findAndModify to a server older than 4.4. This engine
reports itself as 4.4.0 with maxWireVersion 8, and 4.4 is wire 9. That
inconsistency is ours, it is the same one behind the `comment`-on-getMore
failures, and it gets the next commit.
955 lines
48 KiB
JavaScript
955 lines
48 KiB
JavaScript
'use strict';
|
|
// MongoDB unified-test-format runner, pointed at MultiforaDB.
|
|
//
|
|
// PLAN D2 makes the official spec suites the gate for command semantics, and
|
|
// PLAN D7.6 asks for the harness to exist at M0 with a red baseline -- so this
|
|
// is deliberately a *measuring* tool, not a passing one. What it must never do
|
|
// is report a pass it did not earn: a compatibility scorecard that flatters
|
|
// the engine is worse than no scorecard. Everything unimplemented is reported
|
|
// as SKIP with a reason, never as PASS.
|
|
//
|
|
// bash tests/spec/fetch.sh # pinned suites
|
|
// node tests/spec/run.js # spawns its own server
|
|
// node tests/spec/run.js --url mongodb://127.0.0.1:27020 # external server
|
|
// node tests/spec/run.js --file find.json --verbose
|
|
// node tests/spec/run.js --scorecard # rewrite tests/spec/scorecard.txt
|
|
//
|
|
// Matching follows the pseudo-code in the spec's "Evaluating Matches"
|
|
// section verbatim, including the two rules that are easy to get wrong and
|
|
// that decide whether a result is a real pass: extra keys are tolerated only
|
|
// in a *root* document, and numeric types compare flexibly.
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { spawn } = require('child_process');
|
|
|
|
const DRIVER = path.join(__dirname, '..', 'e2e', 'node_modules', 'mongodb');
|
|
const { MongoClient } = require(DRIVER);
|
|
const { EJSON, Long, Int32, Double, Decimal128, ObjectId, Binary, Timestamp } = require(path.join(DRIVER, 'lib', 'bson.js'));
|
|
|
|
const SUITE_DIR = path.join(__dirname, 'specifications', 'source', 'crud', 'tests', 'unified');
|
|
const SCORECARD = path.join(__dirname, 'scorecard.txt');
|
|
const REPO = path.join(__dirname, '..', '..');
|
|
|
|
// The runner implements the schema features that CRUD tests actually use, up
|
|
// to this version. A file declaring more than this is skipped whole rather
|
|
// than half-run.
|
|
const MAX_SCHEMA = [1, 24];
|
|
|
|
const argv = process.argv.slice(2);
|
|
function opt(name, dflt) {
|
|
const i = argv.indexOf('--' + name);
|
|
if (i < 0) return dflt;
|
|
const v = argv[i + 1];
|
|
return v === undefined || v.startsWith('--') ? true : v;
|
|
}
|
|
const VERBOSE = !!opt('verbose', false);
|
|
const ONLY_FILE = opt('file', null);
|
|
const WRITE_SCORECARD = !!opt('scorecard', false);
|
|
const EXTERNAL_URL = opt('url', null);
|
|
const PORT = parseInt(opt('port', '27222'), 10);
|
|
const BIN = process.env.MFDB_BIN || path.join(REPO, 'zig-out', 'bin', 'multiforadb');
|
|
const DB_PATH = process.env.MFDB_SPEC_DB || path.join(REPO, '.zig-cache', 'spec.log');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Server lifecycle (mirrors tests/e2e/e2e6.js: poll with a real ping, and
|
|
// treat an exited child as a hard failure so we cannot silently measure a
|
|
// stale server on the same port).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
let server = null;
|
|
let serverDead = false;
|
|
let serverExit = null;
|
|
let serverOut = [];
|
|
// Persisted, not just buffered in memory: when the server dies mid-run its
|
|
// panic is the only thing that explains the hundreds of ECONNREFUSED failures
|
|
// that follow, and an in-memory buffer is lost if the runner itself is killed.
|
|
const SERVER_LOG = path.join(REPO, '.zig-cache', 'spec-server.out');
|
|
|
|
function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
|
|
|
// A single case must not be able to stall the baseline run. A timeout is
|
|
// reported as a FAIL, not a SKIP: "the engine never answered" is a result, and
|
|
// hiding it in the skip column would overstate compatibility.
|
|
//
|
|
// Per-operation, enforced by the driver (CSOT `timeoutMS`) so the promise
|
|
// settles instead of being abandoned. The budget exists because some commands
|
|
// produce no reply at all rather than an error, and a driver would otherwise
|
|
// wait indefinitely.
|
|
//
|
|
// 10 s, not the 3 s that seems ample for an in-process engine on loopback
|
|
// where a clean suite file runs in well under a second. A tight budget makes
|
|
// the scorecard a function of machine load rather than of the engine: with a
|
|
// concurrent `zig build test` (which compiles and runs multi-threaded tests),
|
|
// a 3 s budget turned 166 perfectly good cases into timeout FAILs, and the
|
|
// same files passed alone. A generous budget costs wall-clock only on cases
|
|
// that genuinely hang, which are few. Don't tighten it to speed up a run.
|
|
const OP_TIMEOUT_MS = parseInt(opt('op-timeout-ms', '10000'), 10);
|
|
// Backstop only, for a hang the driver's own timeout cannot see. It should
|
|
// essentially never fire -- when it does, the case is abandoned, and abandoned
|
|
// work is what corrupted two earlier baselines, so it is deliberately far
|
|
// above OP_TIMEOUT_MS rather than close to it.
|
|
const CASE_TIMEOUT_MS = parseInt(opt('case-timeout-ms', '60000'), 10);
|
|
|
|
// Live handle census. Two baselines were wrecked by the runner accumulating
|
|
// something across a single process's 175 files, and a count taken only at the
|
|
// end could not say *when* it started. Printed per file so the transition is
|
|
// visible in the log.
|
|
function resourceTag() {
|
|
if (!process.getActiveResourcesInfo) return '';
|
|
const counts = {};
|
|
for (const r of process.getActiveResourcesInfo()) counts[r] = (counts[r] || 0) + 1;
|
|
const interesting = Object.entries(counts)
|
|
.filter(([k]) => k !== 'Immediate' && k !== 'TTYWrap' && k !== 'ProcessWrap' && k !== 'PipeWrap')
|
|
.map(([k, v]) => `${k}=${v}`)
|
|
.join(' ');
|
|
return interesting ? ` [${interesting}]` : '';
|
|
}
|
|
|
|
// Server round-trip time, so "the engine is getting slower as the run goes on"
|
|
// is a number in the log rather than a guess. Also counts databases, since
|
|
// state accumulating on the server is the obvious candidate.
|
|
async function pingTag() {
|
|
try {
|
|
const t0 = process.hrtime.bigint();
|
|
await harness.db('admin').command({ ping: 1 });
|
|
const ms = Number(process.hrtime.bigint() - t0) / 1e6;
|
|
let ndbs = '?';
|
|
try {
|
|
const r = await harness.db('admin').command({ listDatabases: 1, nameOnly: true });
|
|
ndbs = String((r.databases || []).length);
|
|
} catch (e) {
|
|
ndbs = `ERR(${e.name}: ${String(e.message).slice(0, 60)})`;
|
|
}
|
|
return `ping=${ms.toFixed(1)}ms dbs=${ndbs}`;
|
|
} catch (e) {
|
|
return `ping=FAILED(${e.name})`;
|
|
}
|
|
}
|
|
|
|
async function withTimeout(fn, ms, msg) {
|
|
let timer;
|
|
const guard = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(msg)), ms); });
|
|
try { return await Promise.race([fn(), guard]); } finally { clearTimeout(timer); }
|
|
}
|
|
|
|
async function startServer() {
|
|
try { fs.unlinkSync(DB_PATH); } catch (e) { /* first run */ }
|
|
if (!fs.existsSync(BIN)) throw new Error(`${BIN} missing - run \`zig build\` first`);
|
|
server = spawn(BIN, ['--port', String(PORT), '--db', DB_PATH, '--ttl-sweep-secs', '0'], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
try { fs.unlinkSync(SERVER_LOG); } catch (e) { /* first run */ }
|
|
const sink = fs.createWriteStream(SERVER_LOG, { flags: 'a' });
|
|
const grab = (b) => { sink.write(b); serverOut.push(b.toString()); if (serverOut.length > 400) serverOut.shift(); };
|
|
server.stdout.on('data', grab);
|
|
server.stderr.on('data', grab);
|
|
server.on('exit', (code, signal) => { serverDead = true; serverExit = signal ? `signal ${signal}` : `exit code ${code}`; });
|
|
|
|
const url = `mongodb://127.0.0.1:${PORT}`;
|
|
const deadline = Date.now() + 20000;
|
|
while (Date.now() < deadline) {
|
|
if (serverDead) throw new Error('server exited during startup:\n' + serverOut.join(''));
|
|
try {
|
|
const c = new MongoClient(url, { serverSelectionTimeoutMS: 500 });
|
|
await c.connect();
|
|
await c.db('admin').command({ ping: 1 });
|
|
await c.close();
|
|
return url;
|
|
} catch (e) { await sleep(100); }
|
|
}
|
|
throw new Error('server did not become ready:\n' + serverOut.join(''));
|
|
}
|
|
|
|
function stopServer() {
|
|
if (server && !serverDead) server.kill('SIGKILL');
|
|
}
|
|
process.on('exit', stopServer);
|
|
process.on('SIGINT', () => { stopServer(); process.exit(130); });
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Matching -- the spec's Evaluating Matches algorithm.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
class MatchError extends Error {}
|
|
|
|
function fail(pathStr, msg) { throw new MatchError(`${pathStr || '<root>'}: ${msg}`); }
|
|
|
|
function isPlainDoc(v) {
|
|
if (v === null || typeof v !== 'object' || Array.isArray(v)) return false;
|
|
// BSON scalars are objects too; they are values, not documents.
|
|
return !(v instanceof ObjectId || v instanceof Binary || v instanceof Long ||
|
|
v instanceof Int32 || v instanceof Double || v instanceof Decimal128 ||
|
|
v instanceof Timestamp || v instanceof Date || v instanceof RegExp);
|
|
}
|
|
|
|
function specialKey(doc) {
|
|
if (!isPlainDoc(doc)) return null;
|
|
const ks = Object.keys(doc);
|
|
return ks.length === 1 && ks[0].startsWith('$$') ? ks[0] : null;
|
|
}
|
|
|
|
function numeric(v) {
|
|
if (typeof v === 'number') return v;
|
|
if (v instanceof Int32 || v instanceof Double) return v.valueOf();
|
|
if (v instanceof Long) return Number(v.toString());
|
|
return null;
|
|
}
|
|
|
|
// The spec's type aliases, restricted to what the CRUD suites use.
|
|
function matchesType(actual, alias) {
|
|
switch (alias) {
|
|
case 'double': return typeof actual === 'number' || actual instanceof Double;
|
|
case 'int': return actual instanceof Int32 || (typeof actual === 'number' && Number.isInteger(actual));
|
|
case 'long': return actual instanceof Long || (typeof actual === 'number' && Number.isInteger(actual));
|
|
case 'string': return typeof actual === 'string';
|
|
case 'object': return isPlainDoc(actual);
|
|
case 'array': return Array.isArray(actual);
|
|
case 'binData': return actual instanceof Binary;
|
|
case 'objectId': return actual instanceof ObjectId;
|
|
case 'bool': return typeof actual === 'boolean';
|
|
case 'date': return actual instanceof Date;
|
|
case 'null': return actual === null;
|
|
case 'regex': return actual instanceof RegExp;
|
|
case 'timestamp': return actual instanceof Timestamp;
|
|
case 'decimal': return actual instanceof Decimal128;
|
|
case 'number': return numeric(actual) !== null || actual instanceof Decimal128;
|
|
default: return null; // unknown alias -> caller reports unsupported
|
|
}
|
|
}
|
|
|
|
function scalarEqual(expected, actual) {
|
|
const en = numeric(expected), an = numeric(actual);
|
|
if (en !== null && an !== null) return en === an; // flexible numerics
|
|
if (expected === null) return actual === null;
|
|
if (expected instanceof Date) return actual instanceof Date && +expected === +actual;
|
|
if (expected instanceof ObjectId) return actual instanceof ObjectId && expected.equals(actual);
|
|
if (expected instanceof Binary) return actual instanceof Binary && Buffer.compare(expected.buffer, actual.buffer) === 0 && expected.sub_type === actual.sub_type;
|
|
if (expected instanceof Decimal128) return actual instanceof Decimal128 && expected.toString() === actual.toString();
|
|
if (expected instanceof Timestamp) return actual instanceof Timestamp && expected.equals(actual);
|
|
if (expected instanceof RegExp) return actual instanceof RegExp && String(expected) === String(actual);
|
|
if (typeof expected !== typeof actual) return false;
|
|
return expected === actual;
|
|
}
|
|
|
|
// `root` implements the one rule that separates a real pass from a lenient
|
|
// one: only a root document may carry keys the expectation does not mention.
|
|
function match(expected, actual, entities, pathStr = '', root = true) {
|
|
const sk = specialKey(expected);
|
|
if (sk) return special(sk, expected[sk], actual, entities, pathStr, true, root);
|
|
|
|
if (isPlainDoc(expected)) {
|
|
if (!isPlainDoc(actual)) fail(pathStr, `expected a document, got ${describe(actual)}`);
|
|
for (const [k, v] of Object.entries(expected)) {
|
|
const kp = pathStr ? `${pathStr}.${k}` : k;
|
|
const vsk = specialKey(v);
|
|
if (vsk) {
|
|
const consumed = special(vsk, v[vsk], actual[k], entities, kp, Object.prototype.hasOwnProperty.call(actual, k), false);
|
|
if (!consumed) continue;
|
|
continue;
|
|
}
|
|
if (!Object.prototype.hasOwnProperty.call(actual, k)) fail(kp, 'missing from actual');
|
|
match(v, actual[k], entities, kp, false);
|
|
}
|
|
if (!root) {
|
|
const extra = Object.keys(actual).filter((k) => !Object.prototype.hasOwnProperty.call(expected, k));
|
|
if (extra.length) fail(pathStr, `unexpected extra keys ${JSON.stringify(extra)}`);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (Array.isArray(expected)) {
|
|
if (!Array.isArray(actual)) fail(pathStr, `expected an array, got ${describe(actual)}`);
|
|
if (expected.length !== actual.length) fail(pathStr, `expected ${expected.length} elements, got ${actual.length}`);
|
|
// Array elements are not root documents: `expectResult: [ {...} ]` from
|
|
// find must match its documents exactly, extra fields included.
|
|
expected.forEach((e, i) => match(e, actual[i], entities, `${pathStr}[${i}]`, false));
|
|
return;
|
|
}
|
|
|
|
if (!scalarEqual(expected, actual)) fail(pathStr, `expected ${describe(expected)}, got ${describe(actual)}`);
|
|
}
|
|
|
|
// Returns whether the actual value still has to be matched by the caller.
|
|
// `root` is the root-ness of the value the operator stands in for: these
|
|
// operators wrap a value, they do not reposition it. A `$$unsetOrMatches` at
|
|
// the top of an `expectResult` still matches a root document, so the actual
|
|
// document may carry fields the expectation does not mention.
|
|
function special(op, arg, actual, entities, pathStr, present, root) {
|
|
switch (op) {
|
|
case '$$exists':
|
|
if (arg && !present) fail(pathStr, 'expected the key to exist');
|
|
if (!arg && present) fail(pathStr, 'expected the key to be absent');
|
|
return false;
|
|
case '$$type': {
|
|
const aliases = Array.isArray(arg) ? arg : [arg];
|
|
const results = aliases.map((a) => matchesType(actual, a));
|
|
if (results.some((r) => r === null)) throw new Unsupported(`$$type alias ${JSON.stringify(arg)}`);
|
|
if (!results.some((r) => r === true)) fail(pathStr, `expected type ${JSON.stringify(arg)}, got ${describe(actual)}`);
|
|
return false;
|
|
}
|
|
case '$$unsetOrMatches':
|
|
if (!present || actual === undefined) return false;
|
|
match(arg, actual, entities, pathStr, root);
|
|
return false;
|
|
case '$$matchesEntity': {
|
|
if (!(arg in entities.map)) fail(pathStr, `entity ${arg} not found`);
|
|
match(entities.map[arg], actual, entities, pathStr, root);
|
|
return false;
|
|
}
|
|
case '$$matchesHexBytes':
|
|
if (!(actual instanceof Binary)) fail(pathStr, 'expected binary data');
|
|
if (actual.buffer.toString('hex') !== String(arg).toLowerCase()) fail(pathStr, 'hex bytes differ');
|
|
return false;
|
|
default:
|
|
throw new Unsupported(`match operator ${op}`);
|
|
}
|
|
}
|
|
|
|
function describe(v) {
|
|
if (v === undefined) return 'undefined';
|
|
if (v === null) return 'null';
|
|
try { return EJSON.stringify(v, { relaxed: true }); } catch (e) { return String(v); }
|
|
}
|
|
|
|
class Unsupported extends Error {}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Operations
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Argument keys the spec passes positionally rather than as driver options.
|
|
const POSITIONAL = new Set(['filter', 'document', 'documents', 'update', 'replacement', 'pipeline', 'fieldName', 'models', 'requests', 'keys', 'name', 'indexes', 'command', 'session', 'entity', 'to']);
|
|
|
|
// Driver options the driver only honours as a JavaScript number. The suites are
|
|
// parsed with `EJSON.parse(text, {relaxed: false})` so that `$numberLong` and
|
|
// friends keep their exact BSON type in *data* -- but that also turns a plain
|
|
// JSON `2` in an *option* into a BSON Int32 object, and the driver gates every
|
|
// one of these on `typeof options.skip === 'number'`
|
|
// (node_modules/mongodb/lib/operations/find.js:68-95). A BSON wrapper therefore
|
|
// failed the check and the option was dropped on the floor: `skip`, `limit` and
|
|
// `batchSize` never reached the wire at all, and three find.json cases failed
|
|
// with the *unclipped* match count while the engine was applying both correctly.
|
|
// Read as an engine bug for a whole milestone. Coerce by name, not by shape:
|
|
// unwrapping every numeric-looking value would rewrite the wire type of the
|
|
// `comment` and `hint` values that other suites assert on.
|
|
const NUMERIC_OPTIONS = new Set([
|
|
'skip',
|
|
'limit',
|
|
'batchSize',
|
|
'maxTimeMS',
|
|
'maxAwaitTimeMS',
|
|
'expireAfterSeconds',
|
|
]);
|
|
|
|
function numeric_option(v) {
|
|
if (v === null || typeof v !== 'object' || typeof v.valueOf !== 'function') return v;
|
|
const n = v.valueOf();
|
|
return typeof n === 'number' ? n : v;
|
|
}
|
|
|
|
function options(args, drop = []) {
|
|
const o = {};
|
|
for (const [k, v] of Object.entries(args || {})) {
|
|
if (POSITIONAL.has(k) || drop.includes(k)) continue;
|
|
o[k] = NUMERIC_OPTIONS.has(k) ? numeric_option(v) : v;
|
|
}
|
|
return Object.keys(o).length ? o : undefined;
|
|
}
|
|
|
|
function requireNoSession(args) {
|
|
if (args && args.session) throw new Unsupported('explicit sessions (M4)');
|
|
}
|
|
|
|
async function runOperation(op, entities) {
|
|
const args = op.arguments || {};
|
|
requireNoSession(args);
|
|
const target = entities.map[op.object];
|
|
if (op.object === 'testRunner') throw new Unsupported(`testRunner operation ${op.name}`);
|
|
if (target === undefined) throw new Unsupported(`entity ${op.object}`);
|
|
|
|
switch (op.name) {
|
|
// -- read ----------------------------------------------------------
|
|
case 'find': return await target.find(args.filter || {}, options(args)).toArray();
|
|
case 'findOne': return await target.findOne(args.filter || {}, options(args));
|
|
case 'aggregate': return await target.aggregate(args.pipeline, options(args)).toArray();
|
|
case 'countDocuments': return await target.countDocuments(args.filter || {}, options(args));
|
|
case 'estimatedDocumentCount': return await target.estimatedDocumentCount(options(args));
|
|
case 'distinct': return await target.distinct(args.fieldName, args.filter || {}, options(args));
|
|
case 'listIndexes': return await target.listIndexes(options(args)).toArray();
|
|
case 'listIndexNames': return await target.indexes(options(args)).then((ix) => ix.map((i) => i.name));
|
|
case 'listCollections': return await target.listCollections(args.filter || {}, options(args)).toArray();
|
|
case 'listCollectionNames': return await target.listCollections(args.filter || {}, options(args)).toArray().then((cs) => cs.map((c) => c.name));
|
|
case 'runCommand': return await target.command(args.command, options(args, ['commandName']));
|
|
|
|
// -- write ---------------------------------------------------------
|
|
case 'insertOne': return plain(await target.insertOne(args.document, options(args)));
|
|
case 'insertMany': return plain(await target.insertMany(args.documents, options(args)));
|
|
case 'updateOne': return plain(await target.updateOne(args.filter, args.update, options(args)));
|
|
case 'updateMany': return plain(await target.updateMany(args.filter, args.update, options(args)));
|
|
case 'replaceOne': return plain(await target.replaceOne(args.filter, args.replacement, options(args)));
|
|
case 'deleteOne': return plain(await target.deleteOne(args.filter, options(args)));
|
|
case 'deleteMany': return plain(await target.deleteMany(args.filter, options(args)));
|
|
case 'bulkWrite': return plain(await target.bulkWrite(args.requests, options(args)));
|
|
case 'findOneAndUpdate': return unwrapFam(await target.findOneAndUpdate(args.filter, args.update, options(args)));
|
|
case 'findOneAndReplace': return unwrapFam(await target.findOneAndReplace(args.filter, args.replacement, options(args)));
|
|
case 'findOneAndDelete': return unwrapFam(await target.findOneAndDelete(args.filter, options(args)));
|
|
|
|
// -- collection / index management ----------------------------------
|
|
case 'createIndex': return await target.createIndex(args.keys, options(args));
|
|
case 'dropIndex': return await target.dropIndex(args.name, options(args));
|
|
case 'createCollection': return void (await target.createCollection(args.collection, options(args, ['collection'])));
|
|
case 'dropCollection': return void (await target.dropCollection(args.collection, options(args, ['collection'])));
|
|
case 'assertCollectionExists': case 'assertCollectionNotExists':
|
|
throw new Unsupported(`testRunner operation ${op.name}`);
|
|
default:
|
|
throw new Unsupported(`operation ${op.name}`);
|
|
}
|
|
}
|
|
|
|
// Driver result objects carry class instances and `acknowledged`; the spec
|
|
// expects plain documents. Turning them into plain objects keeps the
|
|
// root-document rule meaningful (extra keys tolerated only at the root).
|
|
function plain(r) {
|
|
if (r === null || typeof r !== 'object') return r;
|
|
const o = {};
|
|
for (const k of Object.keys(r)) {
|
|
if (k === 'acknowledged') continue;
|
|
o[k] = r[k];
|
|
}
|
|
if (typeof r.getUpsertedIds === 'function') {
|
|
// BulkWriteResult exposes counts through getters, not own keys.
|
|
for (const k of ['insertedCount', 'matchedCount', 'modifiedCount', 'deletedCount', 'upsertedCount', 'insertedIds', 'upsertedIds']) {
|
|
if (r[k] !== undefined) o[k] = r[k];
|
|
}
|
|
}
|
|
return o;
|
|
}
|
|
|
|
// Driver 5+ returns the document itself; 4.x wrapped it in `{value}`. e2e.js
|
|
// tolerates both the same way (`fam.value ?? fam`).
|
|
function unwrapFam(r) {
|
|
if (r && typeof r === 'object' && 'value' in r && 'ok' in r) return r.value;
|
|
return r;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// runOnRequirements
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function cmpVersion(a, b) {
|
|
const pa = String(a).split('.').map(Number), pb = String(b).split('.').map(Number);
|
|
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
|
const x = pa[i] || 0, y = pb[i] || 0;
|
|
if (x !== y) return x < y ? -1 : 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
function unmetRequirement(reqs, server) {
|
|
if (!reqs) return null;
|
|
const reasons = [];
|
|
for (const r of reqs) {
|
|
const why = [];
|
|
if (r.minServerVersion && cmpVersion(server.version, r.minServerVersion) < 0) why.push(`needs server >= ${r.minServerVersion}`);
|
|
if (r.maxServerVersion && cmpVersion(server.version, r.maxServerVersion) > 0) why.push(`needs server <= ${r.maxServerVersion}`);
|
|
if (r.topologies && !r.topologies.includes('single')) why.push(`needs topology ${r.topologies.join('/')}`);
|
|
if (r.auth === true) why.push('needs auth (M7)');
|
|
if (r.csfle) why.push('needs csfle');
|
|
if (r.serverless === 'require') why.push('needs serverless');
|
|
if (r.serverParameters) why.push('needs server parameters');
|
|
if (why.length === 0) return null; // this alternative is satisfied
|
|
reasons.push(why.join(', '));
|
|
}
|
|
return reasons[0] || 'unmet requirement';
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// One file
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// One client for the whole run, used for seeding and outcome checks.
|
|
//
|
|
// It used to be one client per seed and per outcome check. That, plus per-test
|
|
// entity clients that leaked whenever a case timed out before buildEntities
|
|
// returned, meant hundreds of live MongoClients accumulated -- each with its
|
|
// own pool and heartbeat timer. Node's event loop starved, the per-case timer
|
|
// then fired before operations could complete, and every case from that point
|
|
// on "failed" with a timeout. A ping from a separate process still answered,
|
|
// which made it look convincingly like a server-side wedge. It was the runner.
|
|
//
|
|
// The lesson worth keeping: a harness whose own failures look like engine
|
|
// failures will silently invent a compatibility gap. Hence also the assertion
|
|
// below that clients never accumulate.
|
|
let harness = null;
|
|
|
|
async function seedInitialData(initialData) {
|
|
if (!initialData) return;
|
|
for (const spec of initialData) {
|
|
const coll = harness.db(spec.databaseName).collection(spec.collectionName);
|
|
await coll.drop().catch(() => { });
|
|
if (spec.documents && spec.documents.length) await coll.insertMany(spec.documents);
|
|
else await harness.db(spec.databaseName).createCollection(spec.collectionName).catch(() => { });
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Command monitoring
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// The three event types a `client` entity can ask to observe that this runner
|
|
// can produce, mapped to the driver's own event names. `cmap` and `sdam` types
|
|
// are simply not collected; a test that goes on to *assert* them is reported
|
|
// unsupported at that point rather than here, so declaring an observation this
|
|
// runner ignores costs a case nothing.
|
|
const COMMAND_EVENTS = {
|
|
commandStartedEvent: 'commandStarted',
|
|
commandSucceededEvent: 'commandSucceeded',
|
|
commandFailedEvent: 'commandFailed',
|
|
};
|
|
|
|
// Sensitive commands, per the command-logging-and-monitoring spec's Security
|
|
// section. Events for these are dropped unless the entity sets
|
|
// `observeSensitiveCommands` (unified-test-format.md:3070-3075). None of them
|
|
// can be issued by this engine yet -- there is no auth and no user management
|
|
// before M7 -- so this is here to keep the rule where the rule belongs rather
|
|
// than to filter anything today.
|
|
const SENSITIVE_COMMANDS = new Set([
|
|
'authenticate', 'saslstart', 'saslcontinue', 'getnonce', 'createuser',
|
|
'updateuser', 'copydbgetnonce', 'copydbsaslstart', 'copydb',
|
|
]);
|
|
|
|
function isSensitive(ev) {
|
|
const name = String(ev.commandName || '').toLowerCase();
|
|
if (SENSITIVE_COMMANDS.has(name)) return true;
|
|
// `hello` and legacy hello are sensitive only when they carry
|
|
// `speculativeAuthenticate`, which the driver does not report either way --
|
|
// it redacts both the command and the reply to an empty document, and the
|
|
// spec says to infer sensitivity from exactly that.
|
|
if (name === 'hello' || name === 'ismaster') {
|
|
const body = ev.command || ev.reply;
|
|
return !!body && Object.keys(body).length === 0;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Listeners are disabled rather than removed, and disabled before the outcome
|
|
// check rather than after it (unified-test-format.md:3081): the teardown that
|
|
// follows a case issues commands of its own, and a buffer that kept growing
|
|
// through it would make the assertion a function of the harness.
|
|
function disableEvents(events) {
|
|
if (process.env.MFDB_DUMP_EVENTS) {
|
|
for (const [id, buf] of events) console.log(` events ${id}: ${buf.map((e) => `${e.kind}:${e.ev.commandName}`).join(', ')}`);
|
|
}
|
|
for (const buf of events.values()) buf.enabled = false;
|
|
}
|
|
|
|
// `collectionOrDatabaseOptions` (unified-test-format.md, entity definitions).
|
|
// Anything outside this set is refused rather than silently ignored -- that is
|
|
// the whole lesson of the bug this function exists to fix.
|
|
const ENTITY_OPTIONS = new Set(['readConcern', 'readPreference', 'writeConcern']);
|
|
|
|
// These documents are parsed with `relaxed: false` like everything else in the
|
|
// file, so a plain JSON `0` arrives as a BSON Int32 -- and the driver gates
|
|
// `writeConcern.w` on `typeof w === 'number'`, which is exactly the trap
|
|
// NUMERIC_OPTIONS documents for operation options. Unwrapping wholesale is safe
|
|
// here in a way it is not there: these are settings the driver consumes, not
|
|
// values any assertion ever compares.
|
|
function plainOptions(spec, what) {
|
|
for (const k of Object.keys(spec || {})) {
|
|
if (!ENTITY_OPTIONS.has(k)) throw new Unsupported(`${what} ${k}`);
|
|
}
|
|
const walk = (v) => {
|
|
if (Array.isArray(v)) return v.map(walk);
|
|
if (typeof v === 'object' && v !== null && numeric(v) !== null) return numeric(v);
|
|
if (!isPlainDoc(v)) return v;
|
|
const out = {};
|
|
for (const [k, x] of Object.entries(v)) out[k] = walk(x);
|
|
return out;
|
|
};
|
|
return walk(spec || {});
|
|
}
|
|
|
|
// `clients` is supplied by the caller so that entities created before a
|
|
// failure are still closed: returning them only on success is what leaked.
|
|
// `events` is supplied for the same reason -- a case that dies partway still
|
|
// has to be able to turn its listeners off.
|
|
async function buildEntities(url, createEntities, clients, events) {
|
|
const map = {};
|
|
for (const spec of createEntities || []) {
|
|
const [kind, def] = Object.entries(spec)[0];
|
|
switch (kind) {
|
|
case 'client': {
|
|
if (def.useMultipleMongoses === true) { /* single topology: harmless */ }
|
|
// `timeoutMS` (CSOT) makes each operation reject on its own
|
|
// rather than being abandoned by an outer race. That matters
|
|
// more than it looks: an abandoned operation keeps running,
|
|
// keeps its client alive, and -- since it may still be inside
|
|
// buildEntities -- can create further clients *after* cleanup
|
|
// has already run. That is what leaked, and what turned into
|
|
// 190 phantom timeout FAILs.
|
|
const observed = (def.observeEvents || []).filter((e) => e in COMMAND_EVENTS);
|
|
const c = new MongoClient(url, Object.assign({
|
|
serverSelectionTimeoutMS: 2000,
|
|
connectTimeoutMS: 2000,
|
|
timeoutMS: OP_TIMEOUT_MS,
|
|
monitorCommands: observed.length > 0,
|
|
}, def.uriOptions || {}));
|
|
// Registered before connect, so a client whose connect throws
|
|
// or is abandoned is still closed by the caller.
|
|
clients.push(c);
|
|
await c.connect();
|
|
if (clients.abandoned) {
|
|
// Cleanup already ran; this client would otherwise linger.
|
|
await c.close().catch(() => { });
|
|
throw new Error('case abandoned');
|
|
}
|
|
// Subscribed after connect, so the handshake this client just
|
|
// performed is not in its own buffer.
|
|
if (observed.length) {
|
|
const ignore = new Set((def.ignoreCommandMonitoringEvents || []).map((s) => String(s).toLowerCase()));
|
|
const buf = [];
|
|
buf.enabled = true;
|
|
events.set(def.id, buf);
|
|
for (const name of observed) {
|
|
const kind = name.replace(/Event$/, '');
|
|
c.on(COMMAND_EVENTS[name], (ev) => {
|
|
if (!buf.enabled) return;
|
|
if (ignore.has(String(ev.commandName).toLowerCase())) return;
|
|
if (!def.observeSensitiveCommands && isSensitive(ev)) return;
|
|
buf.push({ kind, ev });
|
|
});
|
|
}
|
|
}
|
|
map[def.id] = c;
|
|
break;
|
|
}
|
|
case 'database':
|
|
map[def.id] = map[def.client].db(def.databaseName, plainOptions(def.databaseOptions, 'databaseOptions'));
|
|
break;
|
|
case 'collection':
|
|
map[def.id] = map[def.database].collection(def.collectionName, plainOptions(def.collectionOptions, 'collectionOptions'));
|
|
break;
|
|
case 'session': throw new Unsupported('session entities (M4)');
|
|
case 'bucket': throw new Unsupported('gridfs bucket entities');
|
|
case 'clientEncryption': throw new Unsupported('client-side encryption');
|
|
default: throw new Unsupported(`entity type ${kind}`);
|
|
}
|
|
}
|
|
return { map, clients, events };
|
|
}
|
|
|
|
// Two rules decide how much of the corpus this assertion can reach, and both
|
|
// come from the spec rather than from taste:
|
|
//
|
|
// - `command` and `reply` are matched as *root* documents
|
|
// (unified-test-format.md:1020-1022 and :1037-1039). The driver puts
|
|
// `lsid`, `$db` and `$readPreference` on nearly everything it sends;
|
|
// matched as nested documents, almost every case in the corpus would fail
|
|
// on keys its expectation was never written to mention.
|
|
// - the event list is exact in number and order, not a prefix
|
|
// (unified-test-format.md:3088-3091). 23 cases here expect an empty list,
|
|
// and a prefix rule would pass every one of them without looking.
|
|
function verifyEvents(expectEvents, entities) {
|
|
if (!expectEvents) return;
|
|
for (const spec of expectEvents) {
|
|
const type = spec.eventType || 'command';
|
|
if (type !== 'command') throw new Unsupported(`${type} events`);
|
|
if (spec.ignoreExtraEvents) throw new Unsupported('ignoreExtraEvents');
|
|
const buf = entities.events.get(spec.client);
|
|
// Not treated as an empty list: a client that never subscribed and a
|
|
// client that saw nothing are the same thing to a comparison and very
|
|
// different things to a runner, and the second one is a runner bug
|
|
// that would quietly pass the 23 empty-list cases.
|
|
if (!buf) fail(`events ${spec.client}`, 'the client entity is not observing command events');
|
|
const expected = spec.events || [];
|
|
if (expected.length !== buf.length) {
|
|
fail(`events ${spec.client}`, `expected ${expected.length} events, observed ${buf.length}` +
|
|
` [${buf.map((e) => `${e.kind}:${e.ev.commandName}`).join(', ')}]`);
|
|
}
|
|
expected.forEach((e, i) => matchEvent(e, buf[i], entities, `events ${spec.client}[${i}]`));
|
|
}
|
|
}
|
|
|
|
function matchEvent(expected, actual, entities, pathStr) {
|
|
const [name, body] = Object.entries(expected)[0];
|
|
if (!(name in COMMAND_EVENTS)) throw new Unsupported(`event ${name}`);
|
|
const kind = name.replace(/Event$/, '');
|
|
if (actual.kind !== kind) fail(pathStr, `expected ${kind}, got ${actual.kind} of ${actual.ev.commandName}`);
|
|
for (const [k, v] of Object.entries(body || {})) {
|
|
switch (k) {
|
|
case 'command':
|
|
// The one assertion this runner cannot make, and the only
|
|
// escape hatch in it. Every client entity carries CSOT
|
|
// `timeoutMS` (see OP_TIMEOUT_MS), and CSOT overwrites
|
|
// `maxTimeMS` on every command with what is left of that
|
|
// budget -- so the value on the wire is the harness's, not the
|
|
// test's. Dropping `timeoutMS` is not the alternative: it is
|
|
// what replaced the outer race that produced ~190 phantom
|
|
// timeout FAILs, and it would cost far more than one case.
|
|
// Refused unconditionally rather than only when the values
|
|
// differ, so this can never turn into a pass by coincidence.
|
|
if (isPlainDoc(v) && Object.prototype.hasOwnProperty.call(v, 'maxTimeMS')) {
|
|
throw new Unsupported('maxTimeMS in an expected command (CSOT rewrites it)');
|
|
}
|
|
match(v, actual.ev[k], entities, `${pathStr}.${k}`, true);
|
|
break;
|
|
case 'reply':
|
|
match(v, actual.ev[k], entities, `${pathStr}.${k}`, true);
|
|
break;
|
|
case 'commandName':
|
|
case 'databaseName':
|
|
match(v, actual.ev[k], entities, `${pathStr}.${k}`, false);
|
|
break;
|
|
default:
|
|
throw new Unsupported(`event assertion ${name}.${k}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function verifyOutcome(outcome, entities) {
|
|
if (!outcome) return;
|
|
for (const spec of outcome) {
|
|
const actual = await harness.db(spec.databaseName).collection(spec.collectionName)
|
|
.find({}, { sort: { _id: 1 } }).toArray();
|
|
match(spec.documents, actual, entities, `outcome ${spec.databaseName}.${spec.collectionName}`, false);
|
|
}
|
|
}
|
|
|
|
async function runFile(file, url, server) {
|
|
const text = fs.readFileSync(path.join(SUITE_DIR, file), 'utf8');
|
|
const doc = EJSON.parse(text, { relaxed: false });
|
|
const out = { file, pass: 0, fail: 0, skip: 0, cases: [] };
|
|
|
|
const schema = String(doc.schemaVersion || '1.0').split('.').map(Number);
|
|
if (schema[0] > MAX_SCHEMA[0] || (schema[0] === MAX_SCHEMA[0] && (schema[1] || 0) > MAX_SCHEMA[1])) {
|
|
out.skip = (doc.tests || []).length;
|
|
out.cases.push({ name: '*', status: 'SKIP', reason: `schemaVersion ${doc.schemaVersion} > ${MAX_SCHEMA.join('.')}` });
|
|
return out;
|
|
}
|
|
const fileUnmet = unmetRequirement(doc.runOnRequirements, server);
|
|
if (fileUnmet) {
|
|
out.skip = (doc.tests || []).length;
|
|
out.cases.push({ name: '*', status: 'SKIP', reason: fileUnmet });
|
|
return out;
|
|
}
|
|
|
|
for (const test of doc.tests || []) {
|
|
const name = test.description;
|
|
if (test.skipReason) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: 'upstream skipReason: ' + test.skipReason }); continue; }
|
|
const unmet = unmetRequirement(test.runOnRequirements, server);
|
|
if (unmet) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: unmet }); continue; }
|
|
|
|
// Owned out here, not by buildEntities, so a case that dies partway
|
|
// still has every client it managed to open closed below.
|
|
const clients = [];
|
|
const events = new Map();
|
|
try {
|
|
await withTimeout(async () => {
|
|
await seedInitialData(doc.initialData);
|
|
const entities = await buildEntities(url, doc.createEntities, clients, events);
|
|
for (const op of test.operations) await runOne(op, entities);
|
|
disableEvents(events);
|
|
verifyEvents(test.expectEvents, entities);
|
|
await verifyOutcome(test.outcome, entities);
|
|
}, CASE_TIMEOUT_MS, `case exceeded ${CASE_TIMEOUT_MS} ms`);
|
|
out.pass++;
|
|
out.cases.push({ name, status: 'PASS' });
|
|
} catch (e) {
|
|
if (e instanceof Unsupported) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: 'runner: ' + e.message }); }
|
|
else { out.fail++; out.cases.push({ name, status: 'FAIL', reason: (e instanceof MatchError ? '' : (e.constructor.name + ': ')) + e.message.split('\n')[0].slice(0, 220) }); }
|
|
} finally {
|
|
// Unconditional, and the flag matters: if the case was abandoned,
|
|
// its continuation may still be running and about to open another
|
|
// client, which buildEntities closes itself on seeing this.
|
|
clients.abandoned = true;
|
|
disableEvents(events);
|
|
for (const c of clients) await c.close().catch(() => { });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function runOne(op, entities) {
|
|
if (op.name === 'failPoint' || op.name === 'targetedFailPoint') throw new Unsupported('failPoint');
|
|
let result, err = null;
|
|
try {
|
|
result = await runOperation(op, entities);
|
|
} catch (e) {
|
|
if (e instanceof Unsupported) throw e;
|
|
err = e;
|
|
}
|
|
|
|
if (op.expectError) {
|
|
if (!err) fail(op.name, 'expected an error, the operation succeeded');
|
|
checkError(op.expectError, err, entities, op.name);
|
|
return;
|
|
}
|
|
if (err) throw err;
|
|
if (op.ignoreResultAndError) return;
|
|
if ('expectResult' in op) match(op.expectResult, result, entities, op.name, true);
|
|
if (op.saveResultAsEntity) entities.map[op.saveResultAsEntity] = result;
|
|
}
|
|
|
|
function checkError(exp, err, entities, where) {
|
|
if (exp.isError === false) fail(where, 'expected no error');
|
|
if (exp.isClientError === true && err.code !== undefined && err.code !== null) {
|
|
fail(where, `expected a client-side error, got server code ${err.code}`);
|
|
}
|
|
if (exp.errorContains) {
|
|
const hay = String(err.message || '').toLowerCase();
|
|
if (!hay.includes(String(exp.errorContains).toLowerCase())) fail(where, `error message ${JSON.stringify(String(err.message).slice(0, 120))} does not contain ${JSON.stringify(exp.errorContains)}`);
|
|
}
|
|
if (exp.errorCode !== undefined) {
|
|
const actual = numeric(err.code);
|
|
if (actual !== numeric(exp.errorCode)) fail(where, `expected error code ${exp.errorCode}, got ${err.code}`);
|
|
}
|
|
if (exp.errorCodeName !== undefined) {
|
|
const actual = err.codeName || (err.result && err.result.codeName);
|
|
if (actual !== exp.errorCodeName) fail(where, `expected codeName ${exp.errorCodeName}, got ${actual}`);
|
|
}
|
|
if (exp.errorLabelsContain) {
|
|
for (const l of exp.errorLabelsContain) if (!(err.errorLabels || []).includes(l)) fail(where, `expected error label ${l}`);
|
|
}
|
|
if (exp.errorLabelsOmit) {
|
|
for (const l of exp.errorLabelsOmit) if ((err.errorLabels || []).includes(l)) fail(where, `expected no error label ${l}`);
|
|
}
|
|
if (exp.errorResponse) {
|
|
const resp = err.result || err.errorResponse || err;
|
|
match(exp.errorResponse, resp, entities, where + '.errorResponse', true);
|
|
}
|
|
if (exp.writeErrors || exp.writeConcernErrors) throw new Unsupported('clientBulkWrite error assertions');
|
|
if ('expectResult' in exp) {
|
|
const r = err.result || (err.writeErrors ? plain(err) : undefined);
|
|
if (r === undefined) throw new Unsupported('expectResult on an error the driver does not expose');
|
|
match(exp.expectResult, r, entities, where + '.expectResult', true);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
(async () => {
|
|
if (!fs.existsSync(SUITE_DIR)) {
|
|
console.error(`missing ${SUITE_DIR}\nrun: bash tests/spec/fetch.sh`);
|
|
process.exit(2);
|
|
}
|
|
let files = fs.readdirSync(SUITE_DIR).filter((f) => f.endsWith('.json'))
|
|
.filter((f) => !ONLY_FILE || f === ONLY_FILE || f === ONLY_FILE + '.json')
|
|
.sort();
|
|
// --skip/--limit exist for bisecting a run whose failures depend on
|
|
// position rather than content, which has happened more than once.
|
|
const skip = parseInt(opt('skip', '0'), 10);
|
|
if (skip > 0) files = files.slice(skip);
|
|
const limit = parseInt(opt('limit', '0'), 10);
|
|
if (limit > 0) files = files.slice(0, limit);
|
|
if (!files.length) { console.error('no matching suite files'); process.exit(2); }
|
|
|
|
const url = EXTERNAL_URL || await startServer();
|
|
|
|
harness = new MongoClient(url, { serverSelectionTimeoutMS: 3000, connectTimeoutMS: 3000, socketTimeoutMS: 8000 });
|
|
await harness.connect();
|
|
const bi = await harness.db('admin').command({ buildInfo: 1 }).catch(() => ({ version: '0.0.0' }));
|
|
const hello = await harness.db('admin').command({ hello: 1 }).catch(() => ({}));
|
|
const server = { version: bi.version || '0.0.0', maxWireVersion: hello.maxWireVersion };
|
|
|
|
const results = [];
|
|
let died = null;
|
|
for (const f of files) {
|
|
// A dead server turns every remaining case into ECONNREFUSED, which
|
|
// would land in the scorecard as ~700 engine failures and bury the one
|
|
// fact that matters: it crashed, and where. Stop at the crash instead.
|
|
if (serverDead && !EXTERNAL_URL) {
|
|
died = { after: results.length ? results[results.length - 1].file : '<startup>', next: f };
|
|
break;
|
|
}
|
|
let r;
|
|
try {
|
|
r = await runFile(f, url, server);
|
|
} catch (e) {
|
|
r = { file: f, pass: 0, fail: 0, skip: 0, cases: [{ name: '*', status: 'ERROR', reason: e.message.split('\n')[0].slice(0, 200) }], errored: true };
|
|
}
|
|
results.push(r);
|
|
const tag = r.errored ? 'ERROR' : `${r.pass} pass, ${r.fail} fail, ${r.skip} skip`;
|
|
console.log(`${f.padEnd(48)} ${tag}${resourceTag()} ${await pingTag()}`);
|
|
if (VERBOSE) for (const c of r.cases) if (c.status !== 'PASS') console.log(` ${c.status.padEnd(5)} ${c.name}${c.reason ? ' -- ' + c.reason : ''}`);
|
|
}
|
|
|
|
const tot = results.reduce((a, r) => ({ pass: a.pass + r.pass, fail: a.fail + r.fail, skip: a.skip + r.skip }), { pass: 0, fail: 0, skip: 0 });
|
|
const errored = results.filter((r) => r.errored).length;
|
|
console.log(`\ncrud+aggregate: ${tot.pass} pass, ${tot.fail} fail, ${tot.skip} skip across ${results.length}/${files.length} files (${errored} files errored)`);
|
|
|
|
if (died) {
|
|
console.error(`\n*** THE SERVER DIED (${serverExit}) ***`);
|
|
console.error(`last file completed: ${died.after}; would have run next: ${died.next}`);
|
|
console.error(`server output: ${path.relative(REPO, SERVER_LOG)}`);
|
|
console.error('--- tail ---\n' + serverOut.slice(-40).join(''));
|
|
console.error('Refusing to write a scorecard from a partial run.');
|
|
process.exit(3);
|
|
}
|
|
|
|
if (WRITE_SCORECARD) {
|
|
fs.writeFileSync(SCORECARD, scorecardText(results, tot, errored, server, files.length));
|
|
console.log(`wrote ${path.relative(REPO, SCORECARD)}`);
|
|
}
|
|
// A runner leak once turned into ~77 phantom "engine timeouts" (see the
|
|
// comment on `harness`). This is the tripwire for that class of bug: if the
|
|
// harness is the only client left standing, nothing accumulated.
|
|
const leaked = process.getActiveResourcesInfo
|
|
? process.getActiveResourcesInfo().filter((r) => r === 'Timeout').length
|
|
: 0;
|
|
if (leaked > 8) {
|
|
console.error(`\nWARNING: ${leaked} timers still active at the end of the run.`);
|
|
console.error('That is how a client leak looks; treat the timeout FAILs above as suspect.');
|
|
}
|
|
await harness.close(true).catch(() => { });
|
|
|
|
// The runner's own exit code reports whether it *ran*, not whether the
|
|
// engine passed -- M0's gate is a recorded baseline, and a red baseline is
|
|
// the expected state. A non-zero exit here would make it useless as a gate.
|
|
console.log('SPEC_RUNNER_OK');
|
|
stopServer();
|
|
process.exit(0);
|
|
})().catch((e) => {
|
|
console.error('spec runner failed:', e);
|
|
if (serverOut.length) console.error('--- server output ---\n' + serverOut.slice(-30).join(''));
|
|
stopServer();
|
|
process.exit(1);
|
|
});
|
|
|
|
function scorecardText(results, tot, errored, server, nfiles) {
|
|
const L = [];
|
|
L.push('# MongoDB spec-test scorecard (PLAN D2) -- crud + aggregate, unified format');
|
|
L.push(`# specs: mongodb/specifications @ 615e0f9 (pinned in tests/spec/fetch.sh)`);
|
|
L.push(`# driver: mongodb@${require(path.join(DRIVER, 'package.json')).version} (pinned in tests/e2e/package-lock.json)`);
|
|
L.push(`# server: MultiforaDB reporting version ${server.version}, maxWireVersion ${server.maxWireVersion}`);
|
|
L.push('# reproduce: bash tests/spec/fetch.sh && node tests/spec/run.js --scorecard');
|
|
L.push('#');
|
|
L.push('# What SKIP means here, so the totals are not read as better than they are:');
|
|
L.push('# - the suite needs a feature whose milestone has not landed (sessions M4,');
|
|
L.push('# auth M7, failPoints, gridfs) -- counted as skip, never as pass;');
|
|
L.push('# - or the runner itself does not implement the operation/matcher yet.');
|
|
L.push('# expectEvents IS asserted: the commands the driver sent are compared to the');
|
|
L.push('# expectation exactly, in number and in order, with `command` and `reply`');
|
|
L.push('# matched as root documents. A pass therefore means the engine answered');
|
|
L.push('# correctly *and* was asked the right question. Not comparable to any');
|
|
L.push('# scorecard recorded before that landed, where `pass` was an upper bound.');
|
|
L.push('# Still unasserted, each reported as skip where it is asserted, never as');
|
|
L.push('# pass: cmap and sdam events, ignoreExtraEvents, hasServiceId,');
|
|
L.push('# hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites');
|
|
L.push('# it -- the only assertion this runner declines to make).');
|
|
L.push('');
|
|
L.push(`total\t${tot.pass} pass\t${tot.fail} fail\t${tot.skip} skip\t${nfiles} files\t${errored} errored`);
|
|
L.push('');
|
|
L.push('# per-file: name\tpass\tfail\tskip');
|
|
for (const r of results) L.push(`${r.file}\t${r.pass}\t${r.fail}\t${r.skip}${r.errored ? '\terrored' : ''}`);
|
|
L.push('');
|
|
L.push('# every non-passing case, with its reason');
|
|
for (const r of results) {
|
|
for (const c of r.cases) {
|
|
if (c.status === 'PASS') continue;
|
|
L.push(`${r.file}\t${c.status}\t${c.name}\t${c.reason || ''}`);
|
|
}
|
|
}
|
|
return L.join('\n') + '\n';
|
|
}
|