3044a38d1c
M2.5's gate, built before the milestone it gates -- the same order that put
`expectEvents` before the free list in M1 and Tier 0 before everything in M2.
`mongodb/specifications` has no aggregation suite, which is amendment A6's
central finding, so this milestone has to bring its own. The hazard in a corpus
we author is obvious and fatal: it can encode our own bugs as expectations and
then agree with us forever. So the split is enforced by the tooling.
`sources/*.json` holds documents and pipelines and nothing else; `record.js`
asks a real mongod 8.3.7 what each pipeline answers and writes the unified-
format file from the reply. Inputs authored, expectations measured -- the
discipline that corrected three assumptions in M1's session work and every
error code in M2, where the alternative would have shipped both times.
No second runner. `run.js --suite-dir` points the existing one somewhere else,
so the entity model, the matchers, the skip accounting and `expectEvents` come
for free; a second runner would drift from the first exactly where it mattered.
`--scorecard` is refused with `--suite-dir`, because `scorecard.txt` is the crud
corpus's record and the milestones are compared against it -- writing it from an
unrelated run would replace that record silently.
Errors record the code and not the message: message text is mongod's to change
between releases. Group pipelines end in a `$sort`, because group output order
is unspecified and a case depending on it would fail for the wrong reason on
either server.
The first source covers `$group`: nine accumulators including the edge cases
that decide an implementation -- `$avg` over a group whose values are not
numbers, `$min` of a field no document has, `$push` skipping a missing field,
`$first`/`$last` against input order, grouping on an array, a compound `_id`.
Where it starts, run against the M2 tip:
group-accumulators.json 9 pass 10 fail 0 skip
The nine include the four refusals M2 added, which answer with mongod's own
codes -- so the corpus already confirms that half. The ten are the milestone.
The crud corpus is unchanged at 201/90/196.
994 lines
50 KiB
JavaScript
994 lines
50 KiB
JavaScript
'use strict';
|
|
// MongoDB unified-test-format runner, pointed at MultiforaDB.
|
|
//
|
|
// PLAN D2 makes the official spec suites the gate for command semantics, and
|
|
// PLAN D7.6 asks for the harness to exist at M0 with a red baseline -- so this
|
|
// is deliberately a *measuring* tool, not a passing one. What it must never do
|
|
// is report a pass it did not earn: a compatibility scorecard that flatters
|
|
// the engine is worse than no scorecard. Everything unimplemented is reported
|
|
// as SKIP with a reason, never as PASS.
|
|
//
|
|
// bash tests/spec/fetch.sh # pinned suites
|
|
// node tests/spec/run.js # spawns its own server
|
|
// node tests/spec/run.js --url mongodb://127.0.0.1:27020 # external server
|
|
// node tests/spec/run.js --file find.json --verbose
|
|
// node tests/spec/run.js --scorecard # rewrite tests/spec/scorecard.txt
|
|
//
|
|
// Matching follows the pseudo-code in the spec's "Evaluating Matches"
|
|
// section verbatim, including the two rules that are easy to get wrong and
|
|
// that decide whether a result is a real pass: extra keys are tolerated only
|
|
// in a *root* document, and numeric types compare flexibly.
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const { spawn } = require('child_process');
|
|
|
|
const DRIVER = path.join(__dirname, '..', 'e2e', 'node_modules', 'mongodb');
|
|
const { MongoClient } = require(DRIVER);
|
|
const { EJSON, Long, Int32, Double, Decimal128, ObjectId, Binary, Timestamp } = require(path.join(DRIVER, 'lib', 'bson.js'));
|
|
|
|
const SCORECARD = path.join(__dirname, 'scorecard.txt');
|
|
const REPO = path.join(__dirname, '..', '..');
|
|
|
|
// The runner implements the schema features that CRUD tests actually use, up
|
|
// to this version. A file declaring more than this is skipped whole rather
|
|
// than half-run.
|
|
const MAX_SCHEMA = [1, 24];
|
|
|
|
const argv = process.argv.slice(2);
|
|
function opt(name, dflt) {
|
|
const i = argv.indexOf('--' + name);
|
|
if (i < 0) return dflt;
|
|
const v = argv[i + 1];
|
|
return v === undefined || v.startsWith('--') ? true : v;
|
|
}
|
|
const VERBOSE = !!opt('verbose', false);
|
|
// The corpus. Defaults to the pinned crud suite; `--suite-dir` points the same
|
|
// runner at another one, which is how `tests/spec/aggregate/` is run. Sharing
|
|
// the runner rather than writing a second one is the point: the entity model,
|
|
// the matchers, the skip accounting and `expectEvents` all come for free, and a
|
|
// second runner would drift from this one exactly where it mattered.
|
|
const CUSTOM_SUITE = opt('suite-dir', null);
|
|
const SUITE_DIR = CUSTOM_SUITE
|
|
? path.resolve(CUSTOM_SUITE)
|
|
: path.join(__dirname, 'specifications', 'source', 'crud', 'tests', 'unified');
|
|
const ONLY_FILE = opt('file', null);
|
|
const WRITE_SCORECARD = !!opt('scorecard', false);
|
|
// `scorecard.txt` is the crud corpus's number and the milestones are compared
|
|
// against it. Writing it from a run over some other corpus would silently
|
|
// replace that record with an unrelated one, so the combination is refused
|
|
// rather than made to mean something.
|
|
if (WRITE_SCORECARD && CUSTOM_SUITE) {
|
|
console.error('--scorecard writes the crud corpus\'s record; it cannot be combined with --suite-dir');
|
|
process.exit(2);
|
|
}
|
|
const EXTERNAL_URL = opt('url', null);
|
|
const PORT = parseInt(opt('port', '27222'), 10);
|
|
const BIN = process.env.MFDB_BIN || path.join(REPO, 'zig-out', 'bin', 'multiforadb');
|
|
const DB_PATH = process.env.MFDB_SPEC_DB || path.join(REPO, '.zig-cache', 'spec.log');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Server lifecycle (mirrors tests/e2e/e2e6.js: poll with a real ping, and
|
|
// treat an exited child as a hard failure so we cannot silently measure a
|
|
// stale server on the same port).
|
|
// ---------------------------------------------------------------------------
|
|
|
|
let server = null;
|
|
let serverDead = false;
|
|
let serverExit = null;
|
|
let serverOut = [];
|
|
// Persisted, not just buffered in memory: when the server dies mid-run its
|
|
// panic is the only thing that explains the hundreds of ECONNREFUSED failures
|
|
// that follow, and an in-memory buffer is lost if the runner itself is killed.
|
|
const SERVER_LOG = path.join(REPO, '.zig-cache', 'spec-server.out');
|
|
|
|
function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
|
|
|
// A single case must not be able to stall the baseline run. A timeout is
|
|
// reported as a FAIL, not a SKIP: "the engine never answered" is a result, and
|
|
// hiding it in the skip column would overstate compatibility.
|
|
//
|
|
// Per-operation, enforced by the driver (CSOT `timeoutMS`) so the promise
|
|
// settles instead of being abandoned. The budget exists because some commands
|
|
// produce no reply at all rather than an error, and a driver would otherwise
|
|
// wait indefinitely.
|
|
//
|
|
// 10 s, not the 3 s that seems ample for an in-process engine on loopback
|
|
// where a clean suite file runs in well under a second. A tight budget makes
|
|
// the scorecard a function of machine load rather than of the engine: with a
|
|
// concurrent `zig build test` (which compiles and runs multi-threaded tests),
|
|
// a 3 s budget turned 166 perfectly good cases into timeout FAILs, and the
|
|
// same files passed alone. A generous budget costs wall-clock only on cases
|
|
// that genuinely hang, which are few. Don't tighten it to speed up a run.
|
|
const OP_TIMEOUT_MS = parseInt(opt('op-timeout-ms', '10000'), 10);
|
|
// Backstop only, for a hang the driver's own timeout cannot see. It should
|
|
// essentially never fire -- when it does, the case is abandoned, and abandoned
|
|
// work is what corrupted two earlier baselines, so it is deliberately far
|
|
// above OP_TIMEOUT_MS rather than close to it.
|
|
const CASE_TIMEOUT_MS = parseInt(opt('case-timeout-ms', '60000'), 10);
|
|
|
|
// Live handle census. Two baselines were wrecked by the runner accumulating
|
|
// something across a single process's 175 files, and a count taken only at the
|
|
// end could not say *when* it started. Printed per file so the transition is
|
|
// visible in the log.
|
|
function resourceTag() {
|
|
if (!process.getActiveResourcesInfo) return '';
|
|
const counts = {};
|
|
for (const r of process.getActiveResourcesInfo()) counts[r] = (counts[r] || 0) + 1;
|
|
const interesting = Object.entries(counts)
|
|
.filter(([k]) => k !== 'Immediate' && k !== 'TTYWrap' && k !== 'ProcessWrap' && k !== 'PipeWrap')
|
|
.map(([k, v]) => `${k}=${v}`)
|
|
.join(' ');
|
|
return interesting ? ` [${interesting}]` : '';
|
|
}
|
|
|
|
// Server round-trip time, so "the engine is getting slower as the run goes on"
|
|
// is a number in the log rather than a guess. Also counts databases, since
|
|
// state accumulating on the server is the obvious candidate.
|
|
async function pingTag() {
|
|
try {
|
|
const t0 = process.hrtime.bigint();
|
|
await harness.db('admin').command({ ping: 1 });
|
|
const ms = Number(process.hrtime.bigint() - t0) / 1e6;
|
|
let ndbs = '?';
|
|
try {
|
|
const r = await harness.db('admin').command({ listDatabases: 1, nameOnly: true });
|
|
ndbs = String((r.databases || []).length);
|
|
} catch (e) {
|
|
ndbs = `ERR(${e.name}: ${String(e.message).slice(0, 60)})`;
|
|
}
|
|
return `ping=${ms.toFixed(1)}ms dbs=${ndbs}`;
|
|
} catch (e) {
|
|
return `ping=FAILED(${e.name})`;
|
|
}
|
|
}
|
|
|
|
async function withTimeout(fn, ms, msg) {
|
|
let timer;
|
|
const guard = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(msg)), ms); });
|
|
try { return await Promise.race([fn(), guard]); } finally { clearTimeout(timer); }
|
|
}
|
|
|
|
async function startServer() {
|
|
try { fs.unlinkSync(DB_PATH); } catch (e) { /* first run */ }
|
|
if (!fs.existsSync(BIN)) throw new Error(`${BIN} missing - run \`zig build\` first`);
|
|
server = spawn(BIN, ['--port', String(PORT), '--db', DB_PATH, '--ttl-sweep-secs', '0'], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
try { fs.unlinkSync(SERVER_LOG); } catch (e) { /* first run */ }
|
|
const sink = fs.createWriteStream(SERVER_LOG, { flags: 'a' });
|
|
const grab = (b) => { sink.write(b); serverOut.push(b.toString()); if (serverOut.length > 400) serverOut.shift(); };
|
|
server.stdout.on('data', grab);
|
|
server.stderr.on('data', grab);
|
|
server.on('exit', (code, signal) => { serverDead = true; serverExit = signal ? `signal ${signal}` : `exit code ${code}`; });
|
|
|
|
const url = `mongodb://127.0.0.1:${PORT}`;
|
|
const deadline = Date.now() + 20000;
|
|
while (Date.now() < deadline) {
|
|
if (serverDead) throw new Error('server exited during startup:\n' + serverOut.join(''));
|
|
try {
|
|
const c = new MongoClient(url, { serverSelectionTimeoutMS: 500 });
|
|
await c.connect();
|
|
await c.db('admin').command({ ping: 1 });
|
|
await c.close();
|
|
return url;
|
|
} catch (e) { await sleep(100); }
|
|
}
|
|
throw new Error('server did not become ready:\n' + serverOut.join(''));
|
|
}
|
|
|
|
function stopServer() {
|
|
if (server && !serverDead) server.kill('SIGKILL');
|
|
}
|
|
process.on('exit', stopServer);
|
|
process.on('SIGINT', () => { stopServer(); process.exit(130); });
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Matching -- the spec's Evaluating Matches algorithm.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
class MatchError extends Error {}
|
|
|
|
function fail(pathStr, msg) { throw new MatchError(`${pathStr || '<root>'}: ${msg}`); }
|
|
|
|
function isPlainDoc(v) {
|
|
if (v === null || typeof v !== 'object' || Array.isArray(v)) return false;
|
|
// BSON scalars are objects too; they are values, not documents.
|
|
return !(v instanceof ObjectId || v instanceof Binary || v instanceof Long ||
|
|
v instanceof Int32 || v instanceof Double || v instanceof Decimal128 ||
|
|
v instanceof Timestamp || v instanceof Date || v instanceof RegExp);
|
|
}
|
|
|
|
function specialKey(doc) {
|
|
if (!isPlainDoc(doc)) return null;
|
|
const ks = Object.keys(doc);
|
|
return ks.length === 1 && ks[0].startsWith('$$') ? ks[0] : null;
|
|
}
|
|
|
|
function numeric(v) {
|
|
if (typeof v === 'number') return v;
|
|
if (v instanceof Int32 || v instanceof Double) return v.valueOf();
|
|
if (v instanceof Long) return Number(v.toString());
|
|
return null;
|
|
}
|
|
|
|
// The spec's type aliases, restricted to what the CRUD suites use.
|
|
function matchesType(actual, alias) {
|
|
switch (alias) {
|
|
case 'double': return typeof actual === 'number' || actual instanceof Double;
|
|
case 'int': return actual instanceof Int32 || (typeof actual === 'number' && Number.isInteger(actual));
|
|
case 'long': return actual instanceof Long || (typeof actual === 'number' && Number.isInteger(actual));
|
|
case 'string': return typeof actual === 'string';
|
|
case 'object': return isPlainDoc(actual);
|
|
case 'array': return Array.isArray(actual);
|
|
case 'binData': return actual instanceof Binary;
|
|
case 'objectId': return actual instanceof ObjectId;
|
|
case 'bool': return typeof actual === 'boolean';
|
|
case 'date': return actual instanceof Date;
|
|
case 'null': return actual === null;
|
|
case 'regex': return actual instanceof RegExp;
|
|
case 'timestamp': return actual instanceof Timestamp;
|
|
case 'decimal': return actual instanceof Decimal128;
|
|
case 'number': return numeric(actual) !== null || actual instanceof Decimal128;
|
|
default: return null; // unknown alias -> caller reports unsupported
|
|
}
|
|
}
|
|
|
|
function scalarEqual(expected, actual) {
|
|
const en = numeric(expected), an = numeric(actual);
|
|
if (en !== null && an !== null) return en === an; // flexible numerics
|
|
if (expected === null) return actual === null;
|
|
if (expected instanceof Date) return actual instanceof Date && +expected === +actual;
|
|
if (expected instanceof ObjectId) return actual instanceof ObjectId && expected.equals(actual);
|
|
if (expected instanceof Binary) return actual instanceof Binary && Buffer.compare(expected.buffer, actual.buffer) === 0 && expected.sub_type === actual.sub_type;
|
|
if (expected instanceof Decimal128) return actual instanceof Decimal128 && expected.toString() === actual.toString();
|
|
if (expected instanceof Timestamp) return actual instanceof Timestamp && expected.equals(actual);
|
|
if (expected instanceof RegExp) return actual instanceof RegExp && String(expected) === String(actual);
|
|
if (typeof expected !== typeof actual) return false;
|
|
return expected === actual;
|
|
}
|
|
|
|
// `root` implements the one rule that separates a real pass from a lenient
|
|
// one: only a root document may carry keys the expectation does not mention.
|
|
function match(expected, actual, entities, pathStr = '', root = true) {
|
|
const sk = specialKey(expected);
|
|
if (sk) return special(sk, expected[sk], actual, entities, pathStr, true, root);
|
|
|
|
if (isPlainDoc(expected)) {
|
|
if (!isPlainDoc(actual)) fail(pathStr, `expected a document, got ${describe(actual)}`);
|
|
for (const [k, v] of Object.entries(expected)) {
|
|
const kp = pathStr ? `${pathStr}.${k}` : k;
|
|
const vsk = specialKey(v);
|
|
if (vsk) {
|
|
const consumed = special(vsk, v[vsk], actual[k], entities, kp, Object.prototype.hasOwnProperty.call(actual, k), false);
|
|
if (!consumed) continue;
|
|
continue;
|
|
}
|
|
if (!Object.prototype.hasOwnProperty.call(actual, k)) fail(kp, 'missing from actual');
|
|
match(v, actual[k], entities, kp, false);
|
|
}
|
|
if (!root) {
|
|
const extra = Object.keys(actual).filter((k) => !Object.prototype.hasOwnProperty.call(expected, k));
|
|
if (extra.length) fail(pathStr, `unexpected extra keys ${JSON.stringify(extra)}`);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (Array.isArray(expected)) {
|
|
if (!Array.isArray(actual)) fail(pathStr, `expected an array, got ${describe(actual)}`);
|
|
if (expected.length !== actual.length) fail(pathStr, `expected ${expected.length} elements, got ${actual.length}`);
|
|
// Array elements are not root documents: `expectResult: [ {...} ]` from
|
|
// find must match its documents exactly, extra fields included.
|
|
expected.forEach((e, i) => match(e, actual[i], entities, `${pathStr}[${i}]`, false));
|
|
return;
|
|
}
|
|
|
|
if (!scalarEqual(expected, actual)) fail(pathStr, `expected ${describe(expected)}, got ${describe(actual)}`);
|
|
}
|
|
|
|
// Returns whether the actual value still has to be matched by the caller.
|
|
// `root` is the root-ness of the value the operator stands in for: these
|
|
// operators wrap a value, they do not reposition it. A `$$unsetOrMatches` at
|
|
// the top of an `expectResult` still matches a root document, so the actual
|
|
// document may carry fields the expectation does not mention.
|
|
function special(op, arg, actual, entities, pathStr, present, root) {
|
|
switch (op) {
|
|
case '$$exists':
|
|
if (arg && !present) fail(pathStr, 'expected the key to exist');
|
|
if (!arg && present) fail(pathStr, 'expected the key to be absent');
|
|
return false;
|
|
case '$$type': {
|
|
const aliases = Array.isArray(arg) ? arg : [arg];
|
|
const results = aliases.map((a) => matchesType(actual, a));
|
|
if (results.some((r) => r === null)) throw new Unsupported(`$$type alias ${JSON.stringify(arg)}`);
|
|
if (!results.some((r) => r === true)) fail(pathStr, `expected type ${JSON.stringify(arg)}, got ${describe(actual)}`);
|
|
return false;
|
|
}
|
|
case '$$unsetOrMatches':
|
|
if (!present || actual === undefined) return false;
|
|
match(arg, actual, entities, pathStr, root);
|
|
return false;
|
|
case '$$matchesEntity': {
|
|
if (!(arg in entities.map)) fail(pathStr, `entity ${arg} not found`);
|
|
match(entities.map[arg], actual, entities, pathStr, root);
|
|
return false;
|
|
}
|
|
case '$$matchesHexBytes':
|
|
if (!(actual instanceof Binary)) fail(pathStr, 'expected binary data');
|
|
if (actual.buffer.toString('hex') !== String(arg).toLowerCase()) fail(pathStr, 'hex bytes differ');
|
|
return false;
|
|
default:
|
|
throw new Unsupported(`match operator ${op}`);
|
|
}
|
|
}
|
|
|
|
function describe(v) {
|
|
if (v === undefined) return 'undefined';
|
|
if (v === null) return 'null';
|
|
try { return EJSON.stringify(v, { relaxed: true }); } catch (e) { return String(v); }
|
|
}
|
|
|
|
class Unsupported extends Error {}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Operations
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Argument keys the spec passes positionally rather than as driver options.
|
|
const POSITIONAL = new Set(['filter', 'document', 'documents', 'update', 'replacement', 'pipeline', 'fieldName', 'models', 'requests', 'keys', 'name', 'indexes', 'command', 'session', 'entity', 'to']);
|
|
|
|
// Driver options the driver only honours as a JavaScript number. The suites are
|
|
// parsed with `EJSON.parse(text, {relaxed: false})` so that `$numberLong` and
|
|
// friends keep their exact BSON type in *data* -- but that also turns a plain
|
|
// JSON `2` in an *option* into a BSON Int32 object, and the driver gates every
|
|
// one of these on `typeof options.skip === 'number'`
|
|
// (node_modules/mongodb/lib/operations/find.js:68-95). A BSON wrapper therefore
|
|
// failed the check and the option was dropped on the floor: `skip`, `limit` and
|
|
// `batchSize` never reached the wire at all, and three find.json cases failed
|
|
// with the *unclipped* match count while the engine was applying both correctly.
|
|
// Read as an engine bug for a whole milestone. Coerce by name, not by shape:
|
|
// unwrapping every numeric-looking value would rewrite the wire type of the
|
|
// `comment` and `hint` values that other suites assert on.
|
|
const NUMERIC_OPTIONS = new Set([
|
|
'skip',
|
|
'limit',
|
|
'batchSize',
|
|
'maxTimeMS',
|
|
'maxAwaitTimeMS',
|
|
'expireAfterSeconds',
|
|
]);
|
|
|
|
function numeric_option(v) {
|
|
if (v === null || typeof v !== 'object' || typeof v.valueOf !== 'function') return v;
|
|
const n = v.valueOf();
|
|
return typeof n === 'number' ? n : v;
|
|
}
|
|
|
|
function options(args, drop = []) {
|
|
const o = {};
|
|
for (const [k, v] of Object.entries(args || {})) {
|
|
if (POSITIONAL.has(k) || drop.includes(k)) continue;
|
|
o[k] = NUMERIC_OPTIONS.has(k) ? numeric_option(v) : v;
|
|
}
|
|
return Object.keys(o).length ? o : undefined;
|
|
}
|
|
|
|
function requireNoSession(args) {
|
|
if (args && args.session) throw new Unsupported('explicit sessions (M4)');
|
|
}
|
|
|
|
async function runOperation(op, entities) {
|
|
const args = op.arguments || {};
|
|
requireNoSession(args);
|
|
const target = entities.map[op.object];
|
|
if (op.object === 'testRunner') throw new Unsupported(`testRunner operation ${op.name}`);
|
|
if (target === undefined) throw new Unsupported(`entity ${op.object}`);
|
|
|
|
switch (op.name) {
|
|
// -- read ----------------------------------------------------------
|
|
case 'find': return await target.find(args.filter || {}, options(args)).toArray();
|
|
case 'findOne': return await target.findOne(args.filter || {}, options(args));
|
|
case 'aggregate': return await target.aggregate(args.pipeline, options(args)).toArray();
|
|
case 'countDocuments': return await target.countDocuments(args.filter || {}, options(args));
|
|
case 'estimatedDocumentCount': return await target.estimatedDocumentCount(options(args));
|
|
case 'distinct': return await target.distinct(args.fieldName, args.filter || {}, options(args));
|
|
case 'listIndexes': return await target.listIndexes(options(args)).toArray();
|
|
case 'listIndexNames': return await target.indexes(options(args)).then((ix) => ix.map((i) => i.name));
|
|
case 'listCollections': return await target.listCollections(args.filter || {}, options(args)).toArray();
|
|
case 'listCollectionNames': return await target.listCollections(args.filter || {}, options(args)).toArray().then((cs) => cs.map((c) => c.name));
|
|
case 'runCommand': return await target.command(args.command, options(args, ['commandName']));
|
|
|
|
// -- write ---------------------------------------------------------
|
|
case 'insertOne': return plain(await target.insertOne(args.document, options(args)));
|
|
case 'insertMany': return plain(await target.insertMany(args.documents, options(args)));
|
|
case 'updateOne': return plain(await target.updateOne(args.filter, args.update, options(args)));
|
|
case 'updateMany': return plain(await target.updateMany(args.filter, args.update, options(args)));
|
|
case 'replaceOne': return plain(await target.replaceOne(args.filter, args.replacement, options(args)));
|
|
case 'deleteOne': return plain(await target.deleteOne(args.filter, options(args)));
|
|
case 'deleteMany': return plain(await target.deleteMany(args.filter, options(args)));
|
|
case 'bulkWrite': return plain(await target.bulkWrite(args.requests, options(args)));
|
|
case 'findOneAndUpdate': return unwrapFam(await target.findOneAndUpdate(args.filter, args.update, options(args)));
|
|
case 'findOneAndReplace': return unwrapFam(await target.findOneAndReplace(args.filter, args.replacement, options(args)));
|
|
case 'findOneAndDelete': return unwrapFam(await target.findOneAndDelete(args.filter, options(args)));
|
|
|
|
// -- collection / index management ----------------------------------
|
|
case 'createIndex': return await target.createIndex(args.keys, options(args));
|
|
case 'dropIndex': return await target.dropIndex(args.name, options(args));
|
|
case 'createCollection': return void (await target.createCollection(args.collection, options(args, ['collection'])));
|
|
case 'dropCollection': return void (await target.dropCollection(args.collection, options(args, ['collection'])));
|
|
case 'assertCollectionExists': case 'assertCollectionNotExists':
|
|
throw new Unsupported(`testRunner operation ${op.name}`);
|
|
default:
|
|
throw new Unsupported(`operation ${op.name}`);
|
|
}
|
|
}
|
|
|
|
// Driver result objects carry class instances and `acknowledged`; the spec
|
|
// expects plain documents. Turning them into plain objects keeps the
|
|
// root-document rule meaningful (extra keys tolerated only at the root).
|
|
function plain(r) {
|
|
if (r === null || typeof r !== 'object') return r;
|
|
const o = {};
|
|
for (const k of Object.keys(r)) {
|
|
if (k === 'acknowledged') continue;
|
|
o[k] = r[k];
|
|
}
|
|
if (typeof r.getUpsertedIds === 'function') {
|
|
// BulkWriteResult exposes counts through getters, not own keys.
|
|
for (const k of ['insertedCount', 'matchedCount', 'modifiedCount', 'deletedCount', 'upsertedCount', 'insertedIds', 'upsertedIds']) {
|
|
if (r[k] !== undefined) o[k] = r[k];
|
|
}
|
|
}
|
|
return o;
|
|
}
|
|
|
|
// Driver 5+ returns the document itself; 4.x wrapped it in `{value}`. e2e.js
|
|
// tolerates both the same way (`fam.value ?? fam`).
|
|
function unwrapFam(r) {
|
|
if (r && typeof r === 'object' && 'value' in r && 'ok' in r) return r.value;
|
|
return r;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// runOnRequirements
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function cmpVersion(a, b) {
|
|
const pa = String(a).split('.').map(Number), pb = String(b).split('.').map(Number);
|
|
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
|
const x = pa[i] || 0, y = pb[i] || 0;
|
|
if (x !== y) return x < y ? -1 : 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
function unmetRequirement(reqs, server) {
|
|
if (!reqs) return null;
|
|
const reasons = [];
|
|
for (const r of reqs) {
|
|
const why = [];
|
|
if (r.minServerVersion && cmpVersion(server.version, r.minServerVersion) < 0) why.push(`needs server >= ${r.minServerVersion}`);
|
|
if (r.maxServerVersion && cmpVersion(server.version, r.maxServerVersion) > 0) why.push(`needs server <= ${r.maxServerVersion}`);
|
|
if (r.topologies && !r.topologies.includes('single')) why.push(`needs topology ${r.topologies.join('/')}`);
|
|
if (r.auth === true) why.push('needs auth (M7)');
|
|
if (r.csfle) why.push('needs csfle');
|
|
if (r.serverless === 'require') why.push('needs serverless');
|
|
if (r.serverParameters) why.push('needs server parameters');
|
|
if (why.length === 0) return null; // this alternative is satisfied
|
|
reasons.push(why.join(', '));
|
|
}
|
|
return reasons[0] || 'unmet requirement';
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// One file
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// One client for the whole run, used for seeding and outcome checks.
|
|
//
|
|
// It used to be one client per seed and per outcome check. That, plus per-test
|
|
// entity clients that leaked whenever a case timed out before buildEntities
|
|
// returned, meant hundreds of live MongoClients accumulated -- each with its
|
|
// own pool and heartbeat timer. Node's event loop starved, the per-case timer
|
|
// then fired before operations could complete, and every case from that point
|
|
// on "failed" with a timeout. A ping from a separate process still answered,
|
|
// which made it look convincingly like a server-side wedge. It was the runner.
|
|
//
|
|
// The lesson worth keeping: a harness whose own failures look like engine
|
|
// failures will silently invent a compatibility gap. Hence also the assertion
|
|
// below that clients never accumulate.
|
|
let harness = null;
|
|
|
|
async function seedInitialData(initialData) {
|
|
if (!initialData) return;
|
|
for (const spec of initialData) {
|
|
const coll = harness.db(spec.databaseName).collection(spec.collectionName);
|
|
await coll.drop().catch(() => { });
|
|
if (spec.documents && spec.documents.length) await coll.insertMany(spec.documents);
|
|
else await harness.db(spec.databaseName).createCollection(spec.collectionName).catch(() => { });
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Command monitoring
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// The three event types a `client` entity can ask to observe that this runner
|
|
// can produce, mapped to the driver's own event names. `cmap` and `sdam` types
|
|
// are simply not collected; a test that goes on to *assert* them is reported
|
|
// unsupported at that point rather than here, so declaring an observation this
|
|
// runner ignores costs a case nothing.
|
|
const COMMAND_EVENTS = {
|
|
commandStartedEvent: 'commandStarted',
|
|
commandSucceededEvent: 'commandSucceeded',
|
|
commandFailedEvent: 'commandFailed',
|
|
};
|
|
|
|
// Sensitive commands, per the command-logging-and-monitoring spec's Security
|
|
// section. Events for these are dropped unless the entity sets
|
|
// `observeSensitiveCommands` (unified-test-format.md:3070-3075). None of them
|
|
// can be issued by this engine yet -- there is no auth and no user management
|
|
// before M7 -- so this is here to keep the rule where the rule belongs rather
|
|
// than to filter anything today.
|
|
const SENSITIVE_COMMANDS = new Set([
|
|
'authenticate', 'saslstart', 'saslcontinue', 'getnonce', 'createuser',
|
|
'updateuser', 'copydbgetnonce', 'copydbsaslstart', 'copydb',
|
|
]);
|
|
|
|
function isSensitive(ev) {
|
|
const name = String(ev.commandName || '').toLowerCase();
|
|
if (SENSITIVE_COMMANDS.has(name)) return true;
|
|
// `hello` and legacy hello are sensitive only when they carry
|
|
// `speculativeAuthenticate`, which the driver does not report either way --
|
|
// it redacts both the command and the reply to an empty document, and the
|
|
// spec says to infer sensitivity from exactly that.
|
|
if (name === 'hello' || name === 'ismaster') {
|
|
const body = ev.command || ev.reply;
|
|
return !!body && Object.keys(body).length === 0;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// Listeners are disabled rather than removed, and disabled before the outcome
|
|
// check rather than after it (unified-test-format.md:3081): the teardown that
|
|
// follows a case issues commands of its own, and a buffer that kept growing
|
|
// through it would make the assertion a function of the harness.
|
|
function disableEvents(events) {
|
|
if (process.env.MFDB_DUMP_EVENTS) {
|
|
for (const [id, buf] of events) console.log(` events ${id}: ${buf.map((e) => `${e.kind}:${e.ev.commandName}`).join(', ')}`);
|
|
}
|
|
for (const buf of events.values()) buf.enabled = false;
|
|
}
|
|
|
|
// `collectionOrDatabaseOptions` (unified-test-format.md, entity definitions).
|
|
// Anything outside this set is refused rather than silently ignored -- that is
|
|
// the whole lesson of the bug this function exists to fix.
|
|
const ENTITY_OPTIONS = new Set(['readConcern', 'readPreference', 'writeConcern']);
|
|
|
|
// These documents are parsed with `relaxed: false` like everything else in the
|
|
// file, so a plain JSON `0` arrives as a BSON Int32 -- and the driver gates
|
|
// `writeConcern.w` on `typeof w === 'number'`, which is exactly the trap
|
|
// NUMERIC_OPTIONS documents for operation options. Unwrapping wholesale is safe
|
|
// here in a way it is not there: these are settings the driver consumes, not
|
|
// values any assertion ever compares.
|
|
function plainOptions(spec, what) {
|
|
for (const k of Object.keys(spec || {})) {
|
|
if (!ENTITY_OPTIONS.has(k)) throw new Unsupported(`${what} ${k}`);
|
|
}
|
|
const walk = (v) => {
|
|
if (Array.isArray(v)) return v.map(walk);
|
|
if (typeof v === 'object' && v !== null && numeric(v) !== null) return numeric(v);
|
|
if (!isPlainDoc(v)) return v;
|
|
const out = {};
|
|
for (const [k, x] of Object.entries(v)) out[k] = walk(x);
|
|
return out;
|
|
};
|
|
return walk(spec || {});
|
|
}
|
|
|
|
// `clients` is supplied by the caller so that entities created before a
|
|
// failure are still closed: returning them only on success is what leaked.
|
|
// `events` is supplied for the same reason -- a case that dies partway still
|
|
// has to be able to turn its listeners off.
|
|
async function buildEntities(url, createEntities, clients, events) {
|
|
const map = {};
|
|
for (const spec of createEntities || []) {
|
|
const [kind, def] = Object.entries(spec)[0];
|
|
switch (kind) {
|
|
case 'client': {
|
|
if (def.useMultipleMongoses === true) { /* single topology: harmless */ }
|
|
// `timeoutMS` (CSOT) makes each operation reject on its own
|
|
// rather than being abandoned by an outer race. That matters
|
|
// more than it looks: an abandoned operation keeps running,
|
|
// keeps its client alive, and -- since it may still be inside
|
|
// buildEntities -- can create further clients *after* cleanup
|
|
// has already run. That is what leaked, and what turned into
|
|
// 190 phantom timeout FAILs.
|
|
const observed = (def.observeEvents || []).filter((e) => e in COMMAND_EVENTS);
|
|
const c = new MongoClient(url, Object.assign({
|
|
serverSelectionTimeoutMS: 2000,
|
|
connectTimeoutMS: 2000,
|
|
timeoutMS: OP_TIMEOUT_MS,
|
|
monitorCommands: observed.length > 0,
|
|
}, def.uriOptions || {}));
|
|
// Registered before connect, so a client whose connect throws
|
|
// or is abandoned is still closed by the caller.
|
|
clients.push(c);
|
|
await c.connect();
|
|
if (clients.abandoned) {
|
|
// Cleanup already ran; this client would otherwise linger.
|
|
await c.close().catch(() => { });
|
|
throw new Error('case abandoned');
|
|
}
|
|
// Subscribed after connect, so the handshake this client just
|
|
// performed is not in its own buffer.
|
|
if (observed.length) {
|
|
const ignore = new Set((def.ignoreCommandMonitoringEvents || []).map((s) => String(s).toLowerCase()));
|
|
const buf = [];
|
|
buf.enabled = true;
|
|
events.set(def.id, buf);
|
|
for (const name of observed) {
|
|
const kind = name.replace(/Event$/, '');
|
|
c.on(COMMAND_EVENTS[name], (ev) => {
|
|
if (!buf.enabled) return;
|
|
if (ignore.has(String(ev.commandName).toLowerCase())) return;
|
|
if (!def.observeSensitiveCommands && isSensitive(ev)) return;
|
|
buf.push({ kind, ev });
|
|
});
|
|
}
|
|
}
|
|
map[def.id] = c;
|
|
break;
|
|
}
|
|
case 'database':
|
|
map[def.id] = map[def.client].db(def.databaseName, plainOptions(def.databaseOptions, 'databaseOptions'));
|
|
break;
|
|
case 'collection':
|
|
map[def.id] = map[def.database].collection(def.collectionName, plainOptions(def.collectionOptions, 'collectionOptions'));
|
|
break;
|
|
case 'session': throw new Unsupported('session entities (M4)');
|
|
case 'bucket': throw new Unsupported('gridfs bucket entities');
|
|
case 'clientEncryption': throw new Unsupported('client-side encryption');
|
|
default: throw new Unsupported(`entity type ${kind}`);
|
|
}
|
|
}
|
|
return { map, clients, events };
|
|
}
|
|
|
|
// Two rules decide how much of the corpus this assertion can reach, and both
|
|
// come from the spec rather than from taste:
|
|
//
|
|
// - `command` and `reply` are matched as *root* documents
|
|
// (unified-test-format.md:1020-1022 and :1037-1039). The driver puts
|
|
// `lsid`, `$db` and `$readPreference` on nearly everything it sends;
|
|
// matched as nested documents, almost every case in the corpus would fail
|
|
// on keys its expectation was never written to mention.
|
|
// - the event list is exact in number and order, not a prefix
|
|
// (unified-test-format.md:3088-3091). 23 cases here expect an empty list,
|
|
// and a prefix rule would pass every one of them without looking.
|
|
function verifyEvents(expectEvents, entities) {
|
|
if (!expectEvents) return;
|
|
for (const spec of expectEvents) {
|
|
const type = spec.eventType || 'command';
|
|
if (type !== 'command') throw new Unsupported(`${type} events`);
|
|
if (spec.ignoreExtraEvents) throw new Unsupported('ignoreExtraEvents');
|
|
const buf = entities.events.get(spec.client);
|
|
// Not treated as an empty list: a client that never subscribed and a
|
|
// client that saw nothing are the same thing to a comparison and very
|
|
// different things to a runner, and the second one is a runner bug
|
|
// that would quietly pass the 23 empty-list cases.
|
|
if (!buf) fail(`events ${spec.client}`, 'the client entity is not observing command events');
|
|
const expected = spec.events || [];
|
|
if (expected.length !== buf.length) {
|
|
fail(`events ${spec.client}`, `expected ${expected.length} events, observed ${buf.length}` +
|
|
` [${buf.map((e) => `${e.kind}:${e.ev.commandName}`).join(', ')}]`);
|
|
}
|
|
expected.forEach((e, i) => matchEvent(e, buf[i], entities, `events ${spec.client}[${i}]`));
|
|
}
|
|
}
|
|
|
|
// A monitored event hands over the command as the driver holds it in memory,
|
|
// which is not always the shape it puts on the wire: a sort is a JS `Map`
|
|
// (driver lib/sort.js). `Object.keys` on a Map is empty, so the matcher
|
|
// reported every key of an expected sort as missing from a command that in
|
|
// fact carried it. EJSON serializes a Map exactly like a document, which is
|
|
// why a dump of the event looks perfectly correct and this had to be measured
|
|
// rather than read. Converted only for the comparison, and at every depth,
|
|
// since a sort also appears inside `updates[i]`.
|
|
function wireShape(v) {
|
|
if (v instanceof Map) {
|
|
const out = {};
|
|
for (const [k, x] of v) out[k] = wireShape(x);
|
|
return out;
|
|
}
|
|
if (Array.isArray(v)) return v.map(wireShape);
|
|
if (!isPlainDoc(v)) return v;
|
|
const out = {};
|
|
for (const [k, x] of Object.entries(v)) out[k] = wireShape(x);
|
|
return out;
|
|
}
|
|
|
|
function matchEvent(expected, actual, entities, pathStr) {
|
|
const [name, body] = Object.entries(expected)[0];
|
|
if (!(name in COMMAND_EVENTS)) throw new Unsupported(`event ${name}`);
|
|
const kind = name.replace(/Event$/, '');
|
|
if (actual.kind !== kind) fail(pathStr, `expected ${kind}, got ${actual.kind} of ${actual.ev.commandName}`);
|
|
for (const [k, v] of Object.entries(body || {})) {
|
|
switch (k) {
|
|
case 'command':
|
|
// The one assertion this runner cannot make, and the only
|
|
// escape hatch in it. Every client entity carries CSOT
|
|
// `timeoutMS` (see OP_TIMEOUT_MS), and CSOT overwrites
|
|
// `maxTimeMS` on every command with what is left of that
|
|
// budget -- so the value on the wire is the harness's, not the
|
|
// test's. Dropping `timeoutMS` is not the alternative: it is
|
|
// what replaced the outer race that produced ~190 phantom
|
|
// timeout FAILs, and it would cost far more than one case.
|
|
// Refused unconditionally rather than only when the values
|
|
// differ, so this can never turn into a pass by coincidence.
|
|
if (isPlainDoc(v) && Object.prototype.hasOwnProperty.call(v, 'maxTimeMS')) {
|
|
throw new Unsupported('maxTimeMS in an expected command (CSOT rewrites it)');
|
|
}
|
|
match(v, wireShape(actual.ev[k]), entities, `${pathStr}.${k}`, true);
|
|
break;
|
|
case 'reply':
|
|
match(v, wireShape(actual.ev[k]), entities, `${pathStr}.${k}`, true);
|
|
break;
|
|
case 'commandName':
|
|
case 'databaseName':
|
|
match(v, actual.ev[k], entities, `${pathStr}.${k}`, false);
|
|
break;
|
|
default:
|
|
throw new Unsupported(`event assertion ${name}.${k}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function verifyOutcome(outcome, entities) {
|
|
if (!outcome) return;
|
|
for (const spec of outcome) {
|
|
const actual = await harness.db(spec.databaseName).collection(spec.collectionName)
|
|
.find({}, { sort: { _id: 1 } }).toArray();
|
|
match(spec.documents, actual, entities, `outcome ${spec.databaseName}.${spec.collectionName}`, false);
|
|
}
|
|
}
|
|
|
|
async function runFile(file, url, server) {
|
|
const text = fs.readFileSync(path.join(SUITE_DIR, file), 'utf8');
|
|
const doc = EJSON.parse(text, { relaxed: false });
|
|
const out = { file, pass: 0, fail: 0, skip: 0, cases: [] };
|
|
|
|
const schema = String(doc.schemaVersion || '1.0').split('.').map(Number);
|
|
if (schema[0] > MAX_SCHEMA[0] || (schema[0] === MAX_SCHEMA[0] && (schema[1] || 0) > MAX_SCHEMA[1])) {
|
|
out.skip = (doc.tests || []).length;
|
|
out.cases.push({ name: '*', status: 'SKIP', reason: `schemaVersion ${doc.schemaVersion} > ${MAX_SCHEMA.join('.')}` });
|
|
return out;
|
|
}
|
|
const fileUnmet = unmetRequirement(doc.runOnRequirements, server);
|
|
if (fileUnmet) {
|
|
out.skip = (doc.tests || []).length;
|
|
out.cases.push({ name: '*', status: 'SKIP', reason: fileUnmet });
|
|
return out;
|
|
}
|
|
|
|
for (const test of doc.tests || []) {
|
|
const name = test.description;
|
|
if (test.skipReason) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: 'upstream skipReason: ' + test.skipReason }); continue; }
|
|
const unmet = unmetRequirement(test.runOnRequirements, server);
|
|
if (unmet) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: unmet }); continue; }
|
|
|
|
// Owned out here, not by buildEntities, so a case that dies partway
|
|
// still has every client it managed to open closed below.
|
|
const clients = [];
|
|
const events = new Map();
|
|
try {
|
|
await withTimeout(async () => {
|
|
await seedInitialData(doc.initialData);
|
|
const entities = await buildEntities(url, doc.createEntities, clients, events);
|
|
for (const op of test.operations) await runOne(op, entities);
|
|
disableEvents(events);
|
|
verifyEvents(test.expectEvents, entities);
|
|
await verifyOutcome(test.outcome, entities);
|
|
}, CASE_TIMEOUT_MS, `case exceeded ${CASE_TIMEOUT_MS} ms`);
|
|
out.pass++;
|
|
out.cases.push({ name, status: 'PASS' });
|
|
} catch (e) {
|
|
if (e instanceof Unsupported) { out.skip++; out.cases.push({ name, status: 'SKIP', reason: 'runner: ' + e.message }); }
|
|
else { out.fail++; out.cases.push({ name, status: 'FAIL', reason: (e instanceof MatchError ? '' : (e.constructor.name + ': ')) + e.message.split('\n')[0].slice(0, 220) }); }
|
|
} finally {
|
|
// Unconditional, and the flag matters: if the case was abandoned,
|
|
// its continuation may still be running and about to open another
|
|
// client, which buildEntities closes itself on seeing this.
|
|
clients.abandoned = true;
|
|
disableEvents(events);
|
|
for (const c of clients) await c.close().catch(() => { });
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function runOne(op, entities) {
|
|
if (op.name === 'failPoint' || op.name === 'targetedFailPoint') throw new Unsupported('failPoint');
|
|
let result, err = null;
|
|
try {
|
|
result = await runOperation(op, entities);
|
|
} catch (e) {
|
|
if (e instanceof Unsupported) throw e;
|
|
err = e;
|
|
}
|
|
|
|
if (op.expectError) {
|
|
if (!err) fail(op.name, 'expected an error, the operation succeeded');
|
|
checkError(op.expectError, err, entities, op.name);
|
|
return;
|
|
}
|
|
if (err) throw err;
|
|
if (op.ignoreResultAndError) return;
|
|
if ('expectResult' in op) match(op.expectResult, result, entities, op.name, true);
|
|
if (op.saveResultAsEntity) entities.map[op.saveResultAsEntity] = result;
|
|
}
|
|
|
|
function checkError(exp, err, entities, where) {
|
|
if (exp.isError === false) fail(where, 'expected no error');
|
|
if (exp.isClientError === true && err.code !== undefined && err.code !== null) {
|
|
fail(where, `expected a client-side error, got server code ${err.code}`);
|
|
}
|
|
if (exp.errorContains) {
|
|
const hay = String(err.message || '').toLowerCase();
|
|
if (!hay.includes(String(exp.errorContains).toLowerCase())) fail(where, `error message ${JSON.stringify(String(err.message).slice(0, 120))} does not contain ${JSON.stringify(exp.errorContains)}`);
|
|
}
|
|
if (exp.errorCode !== undefined) {
|
|
const actual = numeric(err.code);
|
|
if (actual !== numeric(exp.errorCode)) fail(where, `expected error code ${exp.errorCode}, got ${err.code}`);
|
|
}
|
|
if (exp.errorCodeName !== undefined) {
|
|
const actual = err.codeName || (err.result && err.result.codeName);
|
|
if (actual !== exp.errorCodeName) fail(where, `expected codeName ${exp.errorCodeName}, got ${actual}`);
|
|
}
|
|
if (exp.errorLabelsContain) {
|
|
for (const l of exp.errorLabelsContain) if (!(err.errorLabels || []).includes(l)) fail(where, `expected error label ${l}`);
|
|
}
|
|
if (exp.errorLabelsOmit) {
|
|
for (const l of exp.errorLabelsOmit) if ((err.errorLabels || []).includes(l)) fail(where, `expected no error label ${l}`);
|
|
}
|
|
if (exp.errorResponse) {
|
|
const resp = err.result || err.errorResponse || err;
|
|
match(exp.errorResponse, resp, entities, where + '.errorResponse', true);
|
|
}
|
|
if (exp.writeErrors || exp.writeConcernErrors) throw new Unsupported('clientBulkWrite error assertions');
|
|
if ('expectResult' in exp) {
|
|
const r = err.result || (err.writeErrors ? plain(err) : undefined);
|
|
if (r === undefined) throw new Unsupported('expectResult on an error the driver does not expose');
|
|
match(exp.expectResult, r, entities, where + '.expectResult', true);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
(async () => {
|
|
if (!fs.existsSync(SUITE_DIR)) {
|
|
console.error(CUSTOM_SUITE
|
|
? `missing ${SUITE_DIR}`
|
|
: `missing ${SUITE_DIR}\nrun: bash tests/spec/fetch.sh`);
|
|
process.exit(2);
|
|
}
|
|
let files = fs.readdirSync(SUITE_DIR).filter((f) => f.endsWith('.json'))
|
|
.filter((f) => !ONLY_FILE || f === ONLY_FILE || f === ONLY_FILE + '.json')
|
|
.sort();
|
|
// --skip/--limit exist for bisecting a run whose failures depend on
|
|
// position rather than content, which has happened more than once.
|
|
const skip = parseInt(opt('skip', '0'), 10);
|
|
if (skip > 0) files = files.slice(skip);
|
|
const limit = parseInt(opt('limit', '0'), 10);
|
|
if (limit > 0) files = files.slice(0, limit);
|
|
if (!files.length) { console.error('no matching suite files'); process.exit(2); }
|
|
|
|
const url = EXTERNAL_URL || await startServer();
|
|
|
|
harness = new MongoClient(url, { serverSelectionTimeoutMS: 3000, connectTimeoutMS: 3000, socketTimeoutMS: 8000 });
|
|
await harness.connect();
|
|
const bi = await harness.db('admin').command({ buildInfo: 1 }).catch(() => ({ version: '0.0.0' }));
|
|
const hello = await harness.db('admin').command({ hello: 1 }).catch(() => ({}));
|
|
const server = { version: bi.version || '0.0.0', maxWireVersion: hello.maxWireVersion };
|
|
|
|
const results = [];
|
|
let died = null;
|
|
for (const f of files) {
|
|
// A dead server turns every remaining case into ECONNREFUSED, which
|
|
// would land in the scorecard as ~700 engine failures and bury the one
|
|
// fact that matters: it crashed, and where. Stop at the crash instead.
|
|
if (serverDead && !EXTERNAL_URL) {
|
|
died = { after: results.length ? results[results.length - 1].file : '<startup>', next: f };
|
|
break;
|
|
}
|
|
let r;
|
|
try {
|
|
r = await runFile(f, url, server);
|
|
} catch (e) {
|
|
r = { file: f, pass: 0, fail: 0, skip: 0, cases: [{ name: '*', status: 'ERROR', reason: e.message.split('\n')[0].slice(0, 200) }], errored: true };
|
|
}
|
|
results.push(r);
|
|
const tag = r.errored ? 'ERROR' : `${r.pass} pass, ${r.fail} fail, ${r.skip} skip`;
|
|
console.log(`${f.padEnd(48)} ${tag}${resourceTag()} ${await pingTag()}`);
|
|
if (VERBOSE) for (const c of r.cases) if (c.status !== 'PASS') console.log(` ${c.status.padEnd(5)} ${c.name}${c.reason ? ' -- ' + c.reason : ''}`);
|
|
}
|
|
|
|
const tot = results.reduce((a, r) => ({ pass: a.pass + r.pass, fail: a.fail + r.fail, skip: a.skip + r.skip }), { pass: 0, fail: 0, skip: 0 });
|
|
const errored = results.filter((r) => r.errored).length;
|
|
console.log(`\ncrud+aggregate: ${tot.pass} pass, ${tot.fail} fail, ${tot.skip} skip across ${results.length}/${files.length} files (${errored} files errored)`);
|
|
|
|
if (died) {
|
|
console.error(`\n*** THE SERVER DIED (${serverExit}) ***`);
|
|
console.error(`last file completed: ${died.after}; would have run next: ${died.next}`);
|
|
console.error(`server output: ${path.relative(REPO, SERVER_LOG)}`);
|
|
console.error('--- tail ---\n' + serverOut.slice(-40).join(''));
|
|
console.error('Refusing to write a scorecard from a partial run.');
|
|
process.exit(3);
|
|
}
|
|
|
|
if (WRITE_SCORECARD) {
|
|
fs.writeFileSync(SCORECARD, scorecardText(results, tot, errored, server, files.length));
|
|
console.log(`wrote ${path.relative(REPO, SCORECARD)}`);
|
|
}
|
|
// A runner leak once turned into ~77 phantom "engine timeouts" (see the
|
|
// comment on `harness`). This is the tripwire for that class of bug: if the
|
|
// harness is the only client left standing, nothing accumulated.
|
|
const leaked = process.getActiveResourcesInfo
|
|
? process.getActiveResourcesInfo().filter((r) => r === 'Timeout').length
|
|
: 0;
|
|
if (leaked > 8) {
|
|
console.error(`\nWARNING: ${leaked} timers still active at the end of the run.`);
|
|
console.error('That is how a client leak looks; treat the timeout FAILs above as suspect.');
|
|
}
|
|
await harness.close(true).catch(() => { });
|
|
|
|
// The runner's own exit code reports whether it *ran*, not whether the
|
|
// engine passed -- M0's gate is a recorded baseline, and a red baseline is
|
|
// the expected state. A non-zero exit here would make it useless as a gate.
|
|
console.log('SPEC_RUNNER_OK');
|
|
stopServer();
|
|
process.exit(0);
|
|
})().catch((e) => {
|
|
console.error('spec runner failed:', e);
|
|
if (serverOut.length) console.error('--- server output ---\n' + serverOut.slice(-30).join(''));
|
|
stopServer();
|
|
process.exit(1);
|
|
});
|
|
|
|
function scorecardText(results, tot, errored, server, nfiles) {
|
|
const L = [];
|
|
L.push('# MongoDB spec-test scorecard (PLAN D2) -- crud + aggregate, unified format');
|
|
L.push(`# specs: mongodb/specifications @ 615e0f9 (pinned in tests/spec/fetch.sh)`);
|
|
L.push(`# driver: mongodb@${require(path.join(DRIVER, 'package.json')).version} (pinned in tests/e2e/package-lock.json)`);
|
|
L.push(`# server: MultiforaDB reporting version ${server.version}, maxWireVersion ${server.maxWireVersion}`);
|
|
L.push('# reproduce: bash tests/spec/fetch.sh && node tests/spec/run.js --scorecard');
|
|
L.push('#');
|
|
L.push('# What SKIP means here, so the totals are not read as better than they are:');
|
|
L.push('# - the suite needs a feature whose milestone has not landed (sessions M4,');
|
|
L.push('# auth M7, failPoints, gridfs) -- counted as skip, never as pass;');
|
|
L.push('# - or the runner itself does not implement the operation/matcher yet.');
|
|
L.push('# expectEvents IS asserted: the commands the driver sent are compared to the');
|
|
L.push('# expectation exactly, in number and in order, with `command` and `reply`');
|
|
L.push('# matched as root documents. A pass therefore means the engine answered');
|
|
L.push('# correctly *and* was asked the right question. Not comparable to any');
|
|
L.push('# scorecard recorded before that landed, where `pass` was an upper bound.');
|
|
L.push('# Still unasserted, each reported as skip where it is asserted, never as');
|
|
L.push('# pass: cmap and sdam events, ignoreExtraEvents, hasServiceId,');
|
|
L.push('# hasServerConnectionId, and maxTimeMS in an expected command (CSOT rewrites');
|
|
L.push('# it -- the only assertion this runner declines to make).');
|
|
L.push('');
|
|
L.push(`total\t${tot.pass} pass\t${tot.fail} fail\t${tot.skip} skip\t${nfiles} files\t${errored} errored`);
|
|
L.push('');
|
|
L.push('# per-file: name\tpass\tfail\tskip');
|
|
for (const r of results) L.push(`${r.file}\t${r.pass}\t${r.fail}\t${r.skip}${r.errored ? '\terrored' : ''}`);
|
|
L.push('');
|
|
L.push('# every non-passing case, with its reason');
|
|
for (const r of results) {
|
|
for (const c of r.cases) {
|
|
if (c.status === 'PASS') continue;
|
|
L.push(`${r.file}\t${c.status}\t${c.name}\t${c.reason || ''}`);
|
|
}
|
|
}
|
|
return L.join('\n') + '\n';
|
|
}
|