tests/spec: a recorded corpus for partial and hashed indexes

M3's last row, and the first of the four corpora here whose subject nothing in
the repository tested at all: the pinned suite is crud and aggregate, and
`e2e5.js`/`e2e6.js` write neither a partial nor a hashed spec.

42 cases in two files, recorded red at 3/39. The three that pass are the reads
a partial index does not change -- this server indexes every document, so a
query still finds everything, which is exactly why the review called this a
smaller fire than `arrayFilters`.

A case here is a *sequence* rather than one operation: create an index, insert
against it, read back, list it. So the recorder walks a case's operations in
order and stops at the first that throws, which is what a client would see,
and drops the collection between cases because an index outlives a
`deleteMany`.

Two of the design review's own guesses were wrong, which is the argument for
recording rather than reasoning:

  - **`$in` in a partial filter is allowed.** The review grouped it with `$ne`
    and `$regex`, which are 67.
  - **Same key, different filter, no explicit name is IndexKeySpecsConflict
    (86)**, not the 67 the review assumed.

What it confirmed, and what the implementation now has to satisfy: a unique
partial index constrains only the documents its filter selects; a document
*leaving* the filter frees the value it held; a document *entering* it must
take a value nothing inside holds, or the update is E11000. `sparse` and
`partialFilterExpression` may not be combined (67). `expireAfterSeconds` and a
filter may, and `listIndexes` reports the filter before the expiry. Two hashed
components is 31303, `unique` on a hashed index is 16764, and an array at a
hashed path is 16766 *at insert time* rather than at creation.

tests/spec/indexes/record.js

@@ -0,0 +1,186 @@
+// Record an index corpus by asking a real mongod what the answer is.
+//
+// Partial and hashed indexes: M3's last row. `docs/M3_INDEX_TYPES_DESIGN_REVIEW.md`
+// found that `partialFilterExpression` was accepted and ignored here, which
+// made a *unique* partial index refuse inserts mongod accepts -- and that no
+// test in this repository covered the row, in any suite. The pinned corpus is
+// crud and aggregate; `e2e5.js` and `e2e6.js` test indexes but write neither
+// a partial nor a hashed spec.
+//
+// Unlike `tests/spec/operators/`, a case here is a *sequence*: create an
+// index, insert against it, read back, list it. So the source format carries
+// a list of operations rather than one, and the recorder walks them in order,
+// stopping at the first that throws -- which is what a client would see.
+//
+//   node tests/spec/indexes/record.js --mongod-port 27099
+//
+// Options:
+//   --mongod-port <n>  a running mongod to measure against (default 27099)
+//   --only <name>      record just one source file
+const fs = require('fs');
+const path = require('path');
+const DRIVER = path.join(__dirname, '..', '..', 'e2e', 'node_modules', 'mongodb');
+const { MongoClient } = require(DRIVER);
+const { EJSON, ObjectId } = require(path.join(DRIVER, 'lib', 'bson.js'));
+
+const argv = process.argv.slice(2);
+function opt(name, dflt) {
+    const i = argv.indexOf('--' + name);
+    if (i < 0) return dflt;
+    const v = argv[i + 1];
+    return v === undefined || v.startsWith('--') ? true : v;
+}
+
+const PORT = parseInt(opt('mongod-port', '27099'), 10);
+const ONLY = opt('only', null);
+const SRC_DIR = path.join(__dirname, 'sources');
+const DB_NAME = 'index-corpus';
+const COLL = 'coll';
+const SCHEMA_VERSION = '1.4';
+
+/// An ObjectId is the one value no source authors, so masking it is
+/// unambiguous. Nothing else here is unpredictable: an index name, a key
+/// specification and a document count are all decided by the input.
+function mask(value) {
+    if (value instanceof ObjectId) return { $$type: 'objectId' };
+    if (Array.isArray(value)) return value.map(mask);
+    if (value && typeof value === 'object' && value.constructor === Object) {
+        const out = {};
+        for (const [k, v] of Object.entries(value)) out[k] = mask(v);
+        return out;
+    }
+    return value;
+}
+
+async function main() {
+    const client = new MongoClient(`mongodb://127.0.0.1:${PORT}`, { serverSelectionTimeoutMS: 3000 });
+    try {
+        await client.connect();
+    } catch (e) {
+        console.error(`no mongod on :${PORT} -- start one first:\n` +
+            `  mongod --port ${PORT} --dbpath <dir>\n${e.message}`);
+        process.exit(2);
+    }
+    const build = await client.db('admin').command({ buildInfo: 1 });
+    console.log(`recording against mongod ${build.version} on :${PORT}`);
+
+    const sources = fs.readdirSync(SRC_DIR).filter((f) => f.endsWith('.json'))
+        .filter((f) => !ONLY || f === ONLY || f === ONLY + '.json')
+        .sort();
+    for (const file of sources) {
+        const src = JSON.parse(fs.readFileSync(path.join(SRC_DIR, file), 'utf8'));
+        const name = path.basename(file, '.json');
+        const out = await record(client, name, src);
+        fs.writeFileSync(path.join(__dirname, `${name}.json`), EJSON.stringify(out, { relaxed: false, indent: 2 }) + '\n');
+        const errs = out.tests.filter((t) => t.operations.some((o) => o.expectError)).length;
+        console.log(`  ${name}: ${out.tests.length} cases, ${errs} of them reaching an error`);
+    }
+    await client.close();
+    console.log('RECORDED');
+}
+
+async function record(client, name, src) {
+    const db = client.db(DB_NAME);
+    const coll = db.collection(COLL);
+    const tests = [];
+
+    for (const c of src.cases) {
+        const documents = c.documents === undefined ? src.documents : c.documents;
+        // Dropped rather than emptied: an index outlives a `deleteMany`, and
+        // every case here is about which indexes exist.
+        await coll.drop().catch(() => {});
+        if (documents.length) await coll.insertMany(structuredClone(documents));
+
+        const operations = [];
+        // A case that brings its own documents reseeds through operations,
+        // because `initialData` is per file and seeded once per test.
+        if (c.documents !== undefined) {
+            operations.push({ object: 'collection0', name: 'deleteMany', arguments: { filter: {} } });
+            if (documents.length) {
+                operations.push({
+                    object: 'collection0',
+                    name: 'insertMany',
+                    arguments: { documents: structuredClone(documents) },
+                });
+            }
+        }
+        for (const op of c.ops) {
+            const rec = { object: 'collection0', name: op.name, arguments: structuredClone(op.arguments) };
+            try {
+                rec.expectResult = mask(await invoke(coll, op.name, structuredClone(op.arguments)));
+            } catch (e) {
+                // The code, not the message. mongod's index-specification
+                // errors embed a rendering of the whole spec, which no
+                // formatter here produces.
+                delete rec.expectResult;
+                rec.expectError = { isError: true, errorCode: e.code };
+                operations.push(rec);
+                break; // a client would stop here too
+            }
+            operations.push(rec);
+        }
+
+        const after = await coll.find({}, { sort: { _id: 1 } }).toArray();
+        tests.push({
+            description: c.description,
+            operations,
+            outcome: [{ collectionName: COLL, databaseName: DB_NAME, documents: after.map(mask) }],
+        });
+    }
+    await coll.drop().catch(() => {});
+
+    return {
+        description: name,
+        schemaVersion: SCHEMA_VERSION,
+        // Recorded, not authored. Regenerate with tests/spec/indexes/record.js.
+        createEntities: [
+            { client: { id: 'client0' } },
+            { database: { id: 'database0', client: 'client0', databaseName: DB_NAME } },
+            { collection: { id: 'collection0', database: 'database0', collectionName: COLL } },
+        ],
+        initialData: [{ collectionName: COLL, databaseName: DB_NAME, documents: src.documents }],
+        tests,
+    };
+}
+
+async function invoke(coll, name, args) {
+    switch (name) {
+        case 'createIndex': {
+            const { keys, ...rest } = args;
+            return await coll.createIndex(keys, rest);
+        }
+        case 'listIndexes':
+            return await coll.listIndexes().toArray();
+        case 'dropIndex':
+            return plain(await coll.dropIndex(args.name));
+        case 'insertMany':
+            return plain(await coll.insertMany(args.documents));
+        case 'insertOne':
+            return plain(await coll.insertOne(args.document));
+        case 'find':
+            return await coll.find(args.filter || {}, { sort: args.sort }).toArray();
+        case 'countDocuments':
+            return await coll.countDocuments(args.filter || {});
+        case 'deleteMany':
+            return plain(await coll.deleteMany(args.filter || {}));
+        case 'updateOne':
+            return plain(await coll.updateOne(args.filter, args.update, args));
+        default:
+            throw new Error(`record.js does not know the operation '${name}'`);
+    }
+}
+
+/// Driver results carry `acknowledged` and driver-side types; the unified
+/// format's expectations are the plain counts and ids.
+function plain(result) {
+    const out = {};
+    for (const k of ['insertedCount', 'insertedId', 'insertedIds', 'deletedCount', 'matchedCount', 'modifiedCount', 'upsertedCount']) {
+        if (result[k] !== undefined) out[k] = result[k];
+    }
+    return out;
+}
+
+main().catch((e) => {
+    console.error('RECORD_FAIL', e);
+    process.exit(1);
+});