diff --git a/tests/spec/aggregate/README.md b/tests/spec/aggregate/README.md new file mode 100644 index 0000000..eb7f6de --- /dev/null +++ b/tests/spec/aggregate/README.md @@ -0,0 +1,69 @@ +# The aggregation corpus + +`mongodb/specifications` has no aggregation suite. The thirteen +`aggregate-*.json` files this project runs come from `crud` and test the +aggregate *command* — cursors, read concern, the write stages, collation, +`let`. They touch stages barely and expressions not at all: `$lookup`, +`$unwind`, `$facet`, `$addFields` and `$replaceRoot` appear nowhere in the +pinned corpus. That is PLAN amendment A6, and this directory is its +consequence: M2.5 has to bring its own gate. + +## The one rule + +**Inputs are authored here; expectations are measured against a real mongod.** + +A corpus we write is a corpus that can encode our own bugs as expectations, and +it would then agree with us forever. So `sources/*.json` holds documents and +pipelines and nothing else, and `record.js` asks mongod 8.3.7 what each pipeline +answers. It is the same discipline that corrected three assumptions in M1's +session work and every error code in M2 — the alternative, in both cases, would +have shipped. + +## Running it + +```sh +node tests/spec/run.js --suite-dir tests/spec/aggregate +``` + +The same runner as the crud corpus, pointed elsewhere. Sharing it is the point: +the entity model, the matchers, the skip accounting and `expectEvents` come for +free, and a second runner would drift from the first exactly where it mattered. + +`--scorecard` is refused with `--suite-dir`: `tests/spec/scorecard.txt` is the +crud corpus's record and the milestones are compared against it. + +## Re-recording + +```sh +mongod --port 27099 --dbpath /tmp/mongo-corpus & +node tests/spec/aggregate/record.js --mongod-port 27099 +``` + +Writes `.json` for every `sources/.json`. The generated files are +committed: they *are* the corpus, and regenerating them is how a disagreement +with mongod gets re-measured rather than argued about. + +Two things to know when adding cases: + +- **End a `$group` pipeline with a `$sort`.** Group output order is unspecified, + and a case that depended on it would fail for the wrong reason on either + server. +- **Errors record the code, not the message.** Message text is mongod's to + change between releases; a corpus that pinned it would break for the wrong + reason. + +Leave out any case whose answer depends on a server newer than the 4.4 this +server reports — recording it from mongod 8.x and judging it against a 4.4 +answer measures the version gap, not the engine. + +## Where it stands + +Recorded against mongod 8.3.7, run against the M2 tip: + +``` +group-accumulators.json 9 pass 10 fail 0 skip +``` + +The nine include the four refusals M2 added, which answer with mongod's own +codes. The ten are M2.5's work: `$avg`, `$min`, `$max`, `$first`, `$last`, +`$push`, `$addToSet`, `$count`, and a compound `_id`. diff --git a/tests/spec/aggregate/group-accumulators.json b/tests/spec/aggregate/group-accumulators.json new file mode 100644 index 0000000..7ad872d --- /dev/null +++ b/tests/spec/aggregate/group-accumulators.json @@ -0,0 +1,807 @@ +{ + "description": "group-accumulators", + "schemaVersion": "1.4", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "aggregate-corpus" + } + }, + { + "collection": { + "id": "collection0", + "database": "database0", + "collectionName": "coll" + } + } + ], + "initialData": [ + { + "collectionName": "coll", + "databaseName": "aggregate-corpus", + "documents": [ + { + "_id": 1, + "g": "a", + "x": 10, + "s": "p", + "t": [ + 1, + 2 + ] + }, + { + "_id": 2, + "g": "a", + "x": 20, + "s": "q", + "t": [ + 2, + 3 + ] + }, + { + "_id": 3, + "g": "b", + "x": 30, + "s": "p", + "t": [] + }, + { + "_id": 4, + "g": "b", + "x": 7, + "s": "r" + }, + { + "_id": 5, + "g": "b", + "x": "not a number", + "s": "p", + "t": [ + 4 + ] + } + ] + } + ], + "tests": [ + { + "description": "$sum over a field path", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$sum": "$x" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "v": 30 + }, + { + "_id": "b", + "v": 37 + } + ] + } + ] + }, + { + "description": "$sum counts with a constant", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$sum": 1 + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "v": 2 + }, + { + "_id": "b", + "v": 3 + } + ] + } + ] + }, + { + "description": "$avg ignores non-numeric values", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$avg": "$x" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "v": 15 + }, + { + "_id": "b", + "v": 18.5 + } + ] + } + ] + }, + { + "description": "$avg of a group with no numeric value at all", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$s", + "v": { + "$avg": "$missing" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "p", + "v": null + }, + { + "_id": "q", + "v": null + }, + { + "_id": "r", + "v": null + } + ] + } + ] + }, + { + "description": "$min and $max compare across types", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "lo": { + "$min": "$x" + }, + "hi": { + "$max": "$x" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "lo": 10, + "hi": 20 + }, + { + "_id": "b", + "lo": 7, + "hi": "not a number" + } + ] + } + ] + }, + { + "description": "$min of a field no document has", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$min": "$missing" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "v": null + }, + { + "_id": "b", + "v": null + } + ] + } + ] + }, + { + "description": "$first and $last follow the input order", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$sort": { + "_id": 1 + } + }, + { + "$group": { + "_id": "$g", + "f": { + "$first": "$x" + }, + "l": { + "$last": "$x" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "f": 10, + "l": 20 + }, + { + "_id": "b", + "f": 30, + "l": "not a number" + } + ] + } + ] + }, + { + "description": "$push keeps duplicates and order", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$sort": { + "_id": 1 + } + }, + { + "$group": { + "_id": "$s", + "v": { + "$push": "$g" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "p", + "v": [ + "a", + "b", + "b" + ] + }, + { + "_id": "q", + "v": [ + "a" + ] + }, + { + "_id": "r", + "v": [ + "b" + ] + } + ] + } + ] + }, + { + "description": "$push of a missing field skips it", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$sort": { + "_id": 1 + } + }, + { + "$group": { + "_id": "$g", + "v": { + "$push": "$t" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "v": [ + [ + 1, + 2 + ], + [ + 2, + 3 + ] + ] + }, + { + "_id": "b", + "v": [ + [], + [ + 4 + ] + ] + } + ] + } + ] + }, + { + "description": "$addToSet drops duplicates", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$s", + "v": { + "$addToSet": "$g" + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "p", + "v": [ + "a", + "b" + ] + }, + { + "_id": "q", + "v": [ + "a" + ] + }, + { + "_id": "r", + "v": [ + "b" + ] + } + ] + } + ] + }, + { + "description": "$count is the number of documents in the group", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$count": {} + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": "a", + "v": 2 + }, + { + "_id": "b", + "v": 3 + } + ] + } + ] + }, + { + "description": "a compound _id groups on every field", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": { + "g": "$g", + "s": "$s" + }, + "n": { + "$sum": 1 + } + } + }, + { + "$sort": { + "_id.g": 1, + "_id.s": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": { + "g": "a", + "s": "p" + }, + "n": 1 + }, + { + "_id": { + "g": "a", + "s": "q" + }, + "n": 1 + }, + { + "_id": { + "g": "b", + "s": "p" + }, + "n": 2 + }, + { + "_id": { + "g": "b", + "s": "r" + }, + "n": 1 + } + ] + } + ] + }, + { + "description": "a constant _id puts everything in one group", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": null, + "n": { + "$sum": 1 + } + } + } + ] + }, + "expectResult": [ + { + "_id": null, + "n": 5 + } + ] + } + ] + }, + { + "description": "an _id path that no document has", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$missing", + "n": { + "$sum": 1 + } + } + } + ] + }, + "expectResult": [ + { + "_id": null, + "n": 5 + } + ] + } + ] + }, + { + "description": "grouping on an array field", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$t", + "n": { + "$sum": 1 + } + } + }, + { + "$sort": { + "_id": 1 + } + } + ] + }, + "expectResult": [ + { + "_id": null, + "n": 1 + }, + { + "_id": [], + "n": 1 + }, + { + "_id": [ + 1, + 2 + ], + "n": 1 + }, + { + "_id": [ + 2, + 3 + ], + "n": 1 + }, + { + "_id": [ + 4 + ], + "n": 1 + } + ] + } + ] + }, + { + "description": "an unknown accumulator is refused", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$bogusAcc": "$x" + } + } + } + ] + }, + "expectError": { + "isError": true, + "errorCode": 15952 + } + } + ] + }, + { + "description": "an accumulator that is not a document is refused", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": "$x" + } + } + ] + }, + "expectError": { + "isError": true, + "errorCode": 40234 + } + } + ] + }, + { + "description": "two accumulators in one field are refused", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "_id": "$g", + "v": { + "$sum": "$x", + "$max": "$x" + } + } + } + ] + }, + "expectError": { + "isError": true, + "errorCode": 40238 + } + } + ] + }, + { + "description": "a $group without _id is refused", + "operations": [ + { + "object": "collection0", + "name": "aggregate", + "arguments": { + "pipeline": [ + { + "$group": { + "n": { + "$sum": 1 + } + } + } + ] + }, + "expectError": { + "isError": true, + "errorCode": 15955 + } + } + ] + } + ] +} diff --git a/tests/spec/aggregate/record.js b/tests/spec/aggregate/record.js new file mode 100644 index 0000000..db1257c --- /dev/null +++ b/tests/spec/aggregate/record.js @@ -0,0 +1,128 @@ +// Record an aggregation corpus by asking a real mongod what the answer is. +// +// `mongodb/specifications` has no aggregation suite (see PLAN amendment A6), so +// M2.5 has to bring its own. The one thing a corpus we author must not do is +// encode our own bugs as expectations -- so the *inputs* are authored here and +// the *expectations* are measured, the same discipline that corrected three +// assumptions in M1's session work and every error code in M2. +// +// 1. author `sources/.json`: documents, and a list of pipelines +// 2. run this against a real mongod +// 3. it writes `.json` in the unified format, expectations filled in +// 4. `node tests/spec/run.js --suite-dir tests/spec/aggregate` runs it +// +// Generated files are committed: they are the corpus, and regenerating them is +// how a disagreement with mongod gets re-measured rather than argued about. +// +// node tests/spec/aggregate/record.js --mongod-port 27099 +// +// Options: +// --mongod-port a running mongod to measure against (default 27099) +// --only record just one source file +const fs = require('fs'); +const path = require('path'); +// The same pinned driver `run.js` uses, resolved the same way: there is one +// lockfile in this repo and it lives with the e2e suites. +const { MongoClient } = require(path.join(__dirname, '..', '..', 'e2e', 'node_modules', 'mongodb')); + +const argv = process.argv.slice(2); +function opt(name, dflt) { + const i = argv.indexOf('--' + name); + if (i < 0) return dflt; + const v = argv[i + 1]; + return v === undefined || v.startsWith('--') ? true : v; +} + +const PORT = parseInt(opt('mongod-port', '27099'), 10); +const ONLY = opt('only', null); +const SRC_DIR = path.join(__dirname, 'sources'); +const DB_NAME = 'aggregate-corpus'; + +// The corpus is run against a server that reports 4.4.0, so a case whose answer +// depends on a later server would be recorded from mongod 8.x and then judged +// against a 4.4 answer. Pinned here rather than per file: every case in this +// corpus is expected to be version-independent, and one that is not should be +// left out rather than annotated. +const SCHEMA_VERSION = '1.4'; + +async function main() { + if (!fs.existsSync(SRC_DIR)) { + console.error(`missing ${SRC_DIR}`); + process.exit(2); + } + const client = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 3000 }); + try { + await client.connect(); + } catch (e) { + console.error(`no mongod on :${PORT} -- start one first:\n` + + ` mongod --port ${PORT} --dbpath \n${e.message}`); + process.exit(2); + } + const build = await client.db('admin').command({ buildInfo: 1 }); + console.log(`recording against mongod ${build.version} on :${PORT}`); + + const sources = fs.readdirSync(SRC_DIR).filter((f) => f.endsWith('.json')) + .filter((f) => !ONLY || f === ONLY || f === ONLY + '.json') + .sort(); + if (!sources.length) { + console.error('no source files'); + process.exit(2); + } + + for (const file of sources) { + const src = JSON.parse(fs.readFileSync(path.join(SRC_DIR, file), 'utf8')); + const name = path.basename(file, '.json'); + const out = await record(client, name, src); + const target = path.join(__dirname, `${name}.json`); + fs.writeFileSync(target, JSON.stringify(out, null, 2) + '\n'); + const errs = out.tests.filter((t) => t.operations[0].expectError).length; + console.log(` ${name}: ${out.tests.length} cases, ${errs} of them errors`); + } + await client.close(); + console.log('RECORDED'); +} + +async function record(client, name, src) { + const db = client.db(DB_NAME); + const coll = db.collection('coll'); + const tests = []; + + for (const c of src.cases) { + await coll.drop().catch(() => {}); + await coll.insertMany(structuredClone(src.documents)); + + const op = { object: 'collection0', name: 'aggregate', arguments: { pipeline: c.pipeline } }; + try { + const got = await coll.aggregate(structuredClone(c.pipeline)).toArray(); + op.expectResult = got; + } catch (e) { + // The code, not the message: message text is mongod's to change + // between releases, and a corpus that pins it would fail for the + // wrong reason. `isError` plus the code is what the unified format + // asserts anyway. + op.expectError = { isError: true, errorCode: e.code }; + } + tests.push({ description: c.description, operations: [op] }); + } + await coll.drop().catch(() => {}); + + return { + description: name, + schemaVersion: SCHEMA_VERSION, + // Recorded, not authored. Regenerate with tests/spec/aggregate/record.js. + createEntities: [ + { client: { id: 'client0' } }, + { database: { id: 'database0', client: 'client0', databaseName: DB_NAME } }, + { collection: { id: 'collection0', database: 'database0', collectionName: 'coll' } }, + ], + initialData: [ + { collectionName: 'coll', databaseName: DB_NAME, documents: src.documents }, + ], + tests, + }; +} + +main().catch((e) => { + console.error('RECORD_FAIL', e); + process.exit(1); +}); diff --git a/tests/spec/aggregate/sources/group-accumulators.json b/tests/spec/aggregate/sources/group-accumulators.json new file mode 100644 index 0000000..2ac7453 --- /dev/null +++ b/tests/spec/aggregate/sources/group-accumulators.json @@ -0,0 +1,93 @@ +{ + "_comment": [ + "Inputs only. Expectations are measured -- see record.js.", + "Every pipeline ends in a $sort, because $group's output order is", + "unspecified and a corpus that depended on it would fail for the wrong", + "reason on either server." + ], + "documents": [ + { "_id": 1, "g": "a", "x": 10, "s": "p", "t": [1, 2] }, + { "_id": 2, "g": "a", "x": 20, "s": "q", "t": [2, 3] }, + { "_id": 3, "g": "b", "x": 30, "s": "p", "t": [] }, + { "_id": 4, "g": "b", "x": 7, "s": "r" }, + { "_id": 5, "g": "b", "x": "not a number", "s": "p", "t": [4] } + ], + "cases": [ + { + "description": "$sum over a field path", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$sum": "$x" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$sum counts with a constant", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$sum": 1 } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$avg ignores non-numeric values", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$avg": "$x" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$avg of a group with no numeric value at all", + "pipeline": [{ "$group": { "_id": "$s", "v": { "$avg": "$missing" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$min and $max compare across types", + "pipeline": [{ "$group": { "_id": "$g", "lo": { "$min": "$x" }, "hi": { "$max": "$x" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$min of a field no document has", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$min": "$missing" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$first and $last follow the input order", + "pipeline": [{ "$sort": { "_id": 1 } }, { "$group": { "_id": "$g", "f": { "$first": "$x" }, "l": { "$last": "$x" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$push keeps duplicates and order", + "pipeline": [{ "$sort": { "_id": 1 } }, { "$group": { "_id": "$s", "v": { "$push": "$g" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$push of a missing field skips it", + "pipeline": [{ "$sort": { "_id": 1 } }, { "$group": { "_id": "$g", "v": { "$push": "$t" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$addToSet drops duplicates", + "pipeline": [{ "$group": { "_id": "$s", "v": { "$addToSet": "$g" } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "$count is the number of documents in the group", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$count": {} } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "a compound _id groups on every field", + "pipeline": [{ "$group": { "_id": { "g": "$g", "s": "$s" }, "n": { "$sum": 1 } } }, { "$sort": { "_id.g": 1, "_id.s": 1 } }] + }, + { + "description": "a constant _id puts everything in one group", + "pipeline": [{ "$group": { "_id": null, "n": { "$sum": 1 } } }] + }, + { + "description": "an _id path that no document has", + "pipeline": [{ "$group": { "_id": "$missing", "n": { "$sum": 1 } } }] + }, + { + "description": "grouping on an array field", + "pipeline": [{ "$group": { "_id": "$t", "n": { "$sum": 1 } } }, { "$sort": { "_id": 1 } }] + }, + { + "description": "an unknown accumulator is refused", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$bogusAcc": "$x" } } }] + }, + { + "description": "an accumulator that is not a document is refused", + "pipeline": [{ "$group": { "_id": "$g", "v": "$x" } }] + }, + { + "description": "two accumulators in one field are refused", + "pipeline": [{ "$group": { "_id": "$g", "v": { "$sum": "$x", "$max": "$x" } } }] + }, + { + "description": "a $group without _id is refused", + "pipeline": [{ "$group": { "n": { "$sum": 1 } } }] + } + ] +} diff --git a/tests/spec/run.js b/tests/spec/run.js index 406267a..16510a5 100644 --- a/tests/spec/run.js +++ b/tests/spec/run.js @@ -27,7 +27,6 @@ const DRIVER = path.join(__dirname, '..', 'e2e', 'node_modules', 'mongodb'); const { MongoClient } = require(DRIVER); const { EJSON, Long, Int32, Double, Decimal128, ObjectId, Binary, Timestamp } = require(path.join(DRIVER, 'lib', 'bson.js')); -const SUITE_DIR = path.join(__dirname, 'specifications', 'source', 'crud', 'tests', 'unified'); const SCORECARD = path.join(__dirname, 'scorecard.txt'); const REPO = path.join(__dirname, '..', '..'); @@ -44,8 +43,25 @@ function opt(name, dflt) { return v === undefined || v.startsWith('--') ? true : v; } const VERBOSE = !!opt('verbose', false); +// The corpus. Defaults to the pinned crud suite; `--suite-dir` points the same +// runner at another one, which is how `tests/spec/aggregate/` is run. Sharing +// the runner rather than writing a second one is the point: the entity model, +// the matchers, the skip accounting and `expectEvents` all come for free, and a +// second runner would drift from this one exactly where it mattered. +const CUSTOM_SUITE = opt('suite-dir', null); +const SUITE_DIR = CUSTOM_SUITE + ? path.resolve(CUSTOM_SUITE) + : path.join(__dirname, 'specifications', 'source', 'crud', 'tests', 'unified'); const ONLY_FILE = opt('file', null); const WRITE_SCORECARD = !!opt('scorecard', false); +// `scorecard.txt` is the crud corpus's number and the milestones are compared +// against it. Writing it from a run over some other corpus would silently +// replace that record with an unrelated one, so the combination is refused +// rather than made to mean something. +if (WRITE_SCORECARD && CUSTOM_SUITE) { + console.error('--scorecard writes the crud corpus\'s record; it cannot be combined with --suite-dir'); + process.exit(2); +} const EXTERNAL_URL = opt('url', null); const PORT = parseInt(opt('port', '27222'), 10); const BIN = process.env.MFDB_BIN || path.join(REPO, 'zig-out', 'bin', 'multiforadb'); @@ -851,7 +867,9 @@ function checkError(exp, err, entities, where) { (async () => { if (!fs.existsSync(SUITE_DIR)) { - console.error(`missing ${SUITE_DIR}\nrun: bash tests/spec/fetch.sh`); + console.error(CUSTOM_SUITE + ? `missing ${SUITE_DIR}` + : `missing ${SUITE_DIR}\nrun: bash tests/spec/fetch.sh`); process.exit(2); } let files = fs.readdirSync(SUITE_DIR).filter((f) => f.endsWith('.json'))