diff --git a/bin/corpora.js b/bin/corpora.js index a988b28..df973ba 100755 --- a/bin/corpora.js +++ b/bin/corpora.js @@ -6,6 +6,8 @@ * `run(argv) => Promise` returning a process exit code. */ +import { message } from '../src/message.js' + const USAGE = `corpora — a serverless AT Protocol lexicon authority Usage: @@ -48,7 +50,7 @@ try { const mod = await load() process.exit((await mod.run(argv)) ?? 0) } catch (err) { - process.stderr.write(`corpora ${cmd}: ${err instanceof Error ? err.message : String(err)}\n`) + process.stderr.write(`corpora ${cmd}: ${message(err)}\n`) if (process.env.CORPORA_DEBUG && err instanceof Error && err.stack) { process.stderr.write(`${err.stack}\n`) } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cc07f29..666bae9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -109,9 +109,10 @@ constraint is neither: it is the **1 second startup limit**, since `data.js` bas payloads in global scope. The measurement that matters is therefore gzipped, and the earlier "well under 1%" comparison was -against the wrong axis. Measured on a real 28-lexicon build: 172,800 bytes of artifacts, -219,543 bytes of `data.js`, **20,372 bytes gzipped — 0.68% of the free limit**. The claim was -conservative rather than wrong. Startup decoding at this size is microseconds. +against the wrong axis. Rebuilt from what `lexicon.atcr.io` actually serves on 2026-08-04: +231,214 bytes of `data.js`, **53,319 bytes gzipped — 1.78% of the free limit**. The earlier +0.68% came from synthetic lexicons smaller and more uniform than the real ones; the conclusion +survives the correction, the figure does not. Startup decoding at this size is microseconds. See [DEPLOYING](DEPLOYING.md) for the per-platform detail, including the CDN cache-key settings that break resolution. diff --git a/docs/DEPLOYING.md b/docs/DEPLOYING.md index 099b843..fd70b4c 100644 --- a/docs/DEPLOYING.md +++ b/docs/DEPLOYING.md @@ -68,13 +68,22 @@ the gzipped size without deploying. ### What it actually measures (verified) -A real `corpora build` over 28 synthetic lexicons of about 1.5 KB each — atcr.io's shape — -produced 172,800 bytes of served artifacts and a 219,543 byte `data.js`. Bundled with the serving -core and gzipped, that is **20,372 bytes, or 0.68% of the free-plan limit**. - -ARCHITECTURE's "well under 1% of the free limit" therefore holds, and holds against the limit -that binds. `test/worker/bundle-size.test.js` guards it, using deliberately incompressible -synthetic payloads that overstate a real build by roughly five times. +Measured against the live `lexicon.atcr.io` deployment on 2026-08-04, rebuilding its bundle from +what the authority serves: 28 lexicons, 42,013 bytes of record JSON, 71,268 bytes of proof CARs, +and a 36,835 byte repo CAR. That is a **231,214 byte `data.js`**, and bundled with the serving +core and gzipped, **53,319 bytes — 1.78% of the free-plan limit**. + +Real lexicons are larger and less uniform than the synthetic ones this document used to quote: +records range from 641 to 4,231 bytes against a 1,500 byte mean, and proofs average 2,562 rather +than the 2,100 the synthetic models. Expect a real build to exceed the synthetic estimate. + +ARCHITECTURE's "well under 1% of the free limit" does **not** hold at this shape; "comfortably +under 2%" does, with the binding limit being the gzipped one. +`test/worker/bundle-size.test.js` guards a quarter of the free limit, using deliberately +incompressible synthetic payloads. + +These numbers move with every deploy and are a snapshot, not a contract. What the tests actually +hold fixed is the budget, not the measurement. There is no plausible lexicon namespace that does not fit. That is what makes "embed in the script, not in static assets" a free choice rather than a trade. diff --git a/docs/INTERFACES.md b/docs/INTERFACES.md index f6c2ce1..113d9a7 100644 --- a/docs/INTERFACES.md +++ b/docs/INTERFACES.md @@ -12,6 +12,10 @@ each other. It describes internals and is not part of the public API surface doc Web-standard globals only (`Request`, `Response`, `URL`, `TextEncoder`, `atob`, `JSON`). - `src/node/**` may import `node:` builtins only. - `src/build/**` and `src/verify/**` may import anything; they run offline in Node. +- `src/message.js` belongs to no subsystem: it imports nothing, and `src/build/**`, + `src/verify/**`, and `bin/**` may import it without depending on each other. It exists because + `catch (err)` binds `unknown` everywhere and `verify` states in its own header that it does not + import the build. - Tests use `node:test` and `node:assert/strict`. ## Validating lexicon source @@ -74,7 +78,7 @@ export default { didDoc: { /* did.json, parsed */ }, commit: { cid: 'bafyrei...', rev: '3lxyz...' }, describeRepo: { /* describeRepo.json, parsed */ }, - list: [ { /* list/page-1.json, parsed */ } ], // pages in DESCENDING rkey order — see below + order: ['com.example.thing'], // rkeys in DESCENDING order — see below records: { 'com.example.thing': { uri, cid, value } }, proofs: { 'com.example.thing': Uint8Array }, repoCar: Uint8Array, @@ -111,16 +115,26 @@ No serving shell reads it. `manifest.json` must be byte-stable for identical inputs and key — no timestamps, no absolute paths, keys sorted. Reproducibility is an M1 acceptance criterion. -### `list` ordering is a contract between build and serve +### `order` is a contract between build and serve A reference PDS returns `listRecords` in **descending** rkey order by default — the opposite of MST order. Verified live against `jarrett.app` on 2026-08-03: 25 records, `io.atcr.tag` first and `io.atcr.authFullApp` last. -So `corpora build` must emit `list/page-N.json` in descending rkey order. The handler treats that -sequence as canonical `reverse=false` order and never sorts — it cannot, since it holds only -prebuilt pages. Emit them ascending and Corpora silently disagrees with a real PDS over the same -repo, which is exactly the class of near-miss this project is supposed to eliminate. +So `corpora build` must emit `order` — and `list/page-N.json` — in descending rkey order. The +handler treats that sequence as canonical `reverse=false` order and never sorts. Emit it ascending +and Corpora silently disagrees with a real PDS over the same repo, which is exactly the class of +near-miss this project is supposed to eliminate. + +`order` holds rkeys, not records. The bundle carries each record body exactly once, under +`records`, and the handler pages over `order` per request rather than serving a prebuilt page +shape. `list/page-N.json` still exists in the static tree for readers walking `dist/` directly; +the bundle does not embed it, because doing so meant a second copy of every record — about 27% of +`data.js` and 21% of the heap a Worker holds. `test/build/build.test.js` guards the invariant +structurally and `test/worker/bundle-size.test.js` guards it by size. + +`reverse=true` is served by reading `order` from the other end, not from a mirrored copy: a record +at index `i` is at `length - 1 - i` reversed, and cursors translate through the same arithmetic. ## `corpora/serve` diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 555d60e..e7b76a1 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -250,6 +250,22 @@ Run `goat lex check-dns` and `goat lex status` to confirm this independently bef ordering an input instead and both properties hold. Verified: `--rev "$(git log -1 --format=%cI -- lexicons/)"` yields strictly increasing revs across commits and byte-identical output when rebuilt from the same commit. See [BUILD](BUILD.md) and [RELAYS](RELAYS.md). +5. **Store CAR payloads as blocks rather than whole CARs?** **Not yet — but measured, so the + next person does not have to.** Verified against `lexicon.atcr.io`, 2026-08-04: the repo CAR + and the 28 proof CARs hold *exactly the same* 41 distinct blocks, so 108,103 embedded bytes + encode 36,776 bytes of distinct ones. All 29 share a header, and all 29 reassemble from a + block pool byte-for-byte. + + Cheaper than it sounds — CAR framing is fixed at build time, so the build emits pre-framed + blocks and the handler only concatenates byte ranges. No varints, no CIDs, no imports; the + zero-dependency rule is not the obstacle. Stacked on `order` it roughly halves what a Worker + holds again. + + **The cost is per-request allocation, where there is none today** — `car()` currently hands an + existing `Uint8Array` to `Response` uncopied. That is not worth it against a runtime floor + several times larger than the whole payload. **Revisit at ~10x the namespace**, where the + payload stops being a rounding error. It also assumes every proof roots at the commit, which + is what makes one shared header correct; the emitter would need to assert that. ### Answered during implementation diff --git a/docs/SERVING.md b/docs/SERVING.md index 601b8e8..56075e7 100644 --- a/docs/SERVING.md +++ b/docs/SERVING.md @@ -113,11 +113,12 @@ behaviour exists and can be matched. The full CAR is a valid superset of any diff a caller could request, so it is what they get. -## `listRecords` against static pages +## `listRecords` against a static order -The handler flattens `data.list` into one sequence and answers from it, rather than serving -the prebuilt pages verbatim. That is what makes `limit`, `cursor` and `reverse` work at all -for a caller who does not use the build's page size. +The handler answers from `data.order` — the rkeys in serving order — looking each record up in +`data.records` as it goes. There is no prebuilt page shape in the bundle to serve verbatim, which +is what makes `limit`, `cursor` and `reverse` work for a caller who does not use the build's page +size, and what keeps each record body embedded exactly once. An unknown `collection` here is an empty page rather than an error, which is what a reference PDS returns (verified) and cannot mislead for a repo holding one collection. @@ -128,10 +129,11 @@ Two more things follow, and the second is a contract on `corpora build`: also returns a cursor on the final page. A cursor naming no record in the repo is a 400 rather than a silent empty page, because for a fixed record set there is no ambiguity about whether it was ever valid. -- **`data.list` order is the `reverse=false` order.** The handler treats it as canonical and - reverses it for `reverse=true`; it never sorts. A reference PDS returns records in - *descending* rkey order by default (verified), which is the opposite of MST order, so the - build must emit its pages descending for Corpora to agree with a PDS over the same repo. +- **`data.order` is the `reverse=false` order.** The handler treats it as canonical and reads it + from the other end for `reverse=true`; it never sorts, and holds no reversed copy. A reference + PDS returns records in *descending* rkey order by default (verified), which is the opposite of + MST order, so the build must emit `order` descending for Corpora to agree with a PDS over the + same repo. ## `/.well-known/did.json` is the only DID document path diff --git a/src/build/cli.js b/src/build/cli.js index 094543d..956123f 100644 --- a/src/build/cli.js +++ b/src/build/cli.js @@ -7,6 +7,7 @@ import { readFile } from 'node:fs/promises' import { parseArgs } from 'node:util' +import { message } from '../message.js' import { build } from './index.js' import { isDevelopmentDid } from './diddoc.js' import { dnsCheckMode } from './dns.js' @@ -105,7 +106,7 @@ async function readKey(keyFile) { try { return (await readFile(keyFile, 'utf8')).trim() } catch (err) { - throw new BuildError(`cannot read --key-file ${keyFile}: ${err instanceof Error ? err.message : String(err)}`) + throw new BuildError(`cannot read --key-file ${keyFile}: ${message(err)}`) } } const fromEnv = process.env.CORPORA_SIGNING_KEY?.trim() @@ -128,10 +129,18 @@ function summarize(report, out) { '', ] + // `report.files` and `report.nsids` are built from the same `sources` array by the same + // build, and `planDist` emits a `records/` and a `proofs/` entry for every NSID in it. Every + // path looked up below therefore exists; indexing states that, where a `find` would invite a + // fallback for a report this build cannot produce. + /** @type {Record} */ + const size = {} + for (const file of report.files) size[file.path] = file.bytes + const nsidWidth = Math.max(...report.nsids.map((n) => n.length)) for (const nsid of report.nsids) { - const record = report.files.find((f) => f.path === `records/${nsid}.json`)?.bytes ?? 0 - const proof = report.files.find((f) => f.path === `proofs/${nsid}.car`)?.bytes ?? 0 + const record = size[`records/${nsid}.json`] + const proof = size[`proofs/${nsid}.car`] lines.push(`${nsid.padEnd(nsidWidth)} ${bytes(record)} record ${bytes(proof)} proof`) } lines.push('') diff --git a/src/build/diddoc.js b/src/build/diddoc.js index 52bcbb1..ffd2c3b 100644 --- a/src/build/diddoc.js +++ b/src/build/diddoc.js @@ -8,6 +8,7 @@ */ import { didDocument, parseToAtprotoDocument } from '@atproto/identity' +import { message } from '../message.js' import { BuildError } from './errors.js' const PLC_URL = 'https://plc.directory' @@ -128,7 +129,7 @@ export async function fetchPlcDocument(did, opts = {}) { redirect: 'error', headers: { accept: 'application/did+ld+json,application/json' }, }).catch((err) => { - throw new BuildError(`could not reach ${url}: ${err instanceof Error ? err.message : String(err)}`) + throw new BuildError(`could not reach ${url}: ${message(err)}`) }) if (!res.ok) throw new BuildError(`${url} returned HTTP ${res.status}; is ${did} registered?`) return /** @type {Record} */ (await res.json()) diff --git a/src/build/dns.js b/src/build/dns.js index 3ac6f5f..b0aa0df 100644 --- a/src/build/dns.js +++ b/src/build/dns.js @@ -13,6 +13,7 @@ */ import { resolveTxt as dnsResolveTxt } from 'node:dns/promises' +import { message } from '../message.js' import { BuildError } from './errors.js' /** @@ -69,7 +70,7 @@ export async function checkAuthorityDns(domains, did, opts = {}) { } catch (err) { const code = /** @type {NodeJS.ErrnoException} */ (err).code if (code === 'ENOTFOUND' || code === 'ENODATA') return { name, dids: [] } - throw new BuildError(`DNS lookup for ${name} failed: ${err instanceof Error ? err.message : String(err)}`) + throw new BuildError(`DNS lookup for ${name} failed: ${message(err)}`) } }), ) diff --git a/src/build/emit.js b/src/build/emit.js index 40a98a8..234c696 100644 --- a/src/build/emit.js +++ b/src/build/emit.js @@ -66,9 +66,13 @@ export function planDist(opts) { } // Descending rkey order, which is what a reference PDS returns from listRecords by default and // the opposite of MST order — verified live against jarrett.app on 2026-08-03, `io.atcr.tag` - // first and `io.atcr.authFullApp` last. The handler serves these pages exactly as emitted and - // cannot reorder them, so the order is decided here or not at all. - const list = pages([...nsids].reverse().map((nsid) => repo.records[nsid])) + // first and `io.atcr.authFullApp` last. The handler serves this order exactly as emitted and + // cannot reorder it, so the order is decided here or not at all. + const order = [...nsids].reverse() + // The paged form is written to `list/page-N.json` for readers of the static tree. The embedded + // bundle carries `order` instead: the same sequence as rkeys, without a second copy of every + // record body. See docs/INTERFACES.md. + const list = pages(order.map((nsid) => repo.records[nsid])) /** @type {DistFile[]} */ const files = [ @@ -112,7 +116,7 @@ export function planDist(opts) { didDoc: opts.didDoc, commit, describeRepo, - list, + order, records: Object.fromEntries(nsids.map((nsid) => [nsid, repo.records[nsid]])), proofs: Object.fromEntries(nsids.map((nsid) => [nsid, repo.proofs[nsid]])), repoCar: repo.repoCar, @@ -153,7 +157,7 @@ export function renderDataJs(data) { ['didDoc', json(data.didDoc)], ['commit', json(data.commit)], ['describeRepo', json(data.describeRepo)], - ['list', json(data.list)], + ['order', json(data.order)], ['records', json(data.records)], ['proofs', mapOfBytes(data.proofs)], ['repoCar', bytes(data.repoCar)], diff --git a/src/build/index.js b/src/build/index.js index a0badde..da87a52 100644 --- a/src/build/index.js +++ b/src/build/index.js @@ -12,6 +12,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { Secp256k1Keypair } from '@atproto/crypto' +import { message } from '../message.js' import { checkAuthorityDns, dnsCheckMode } from './dns.js' import { buildDidWebDocument, @@ -177,7 +178,7 @@ async function findUnchanged(opts, manifest) { const url = new URL('/xrpc/com.atproto.sync.getLatestCommit', opts.against) url.searchParams.set('did', manifest.did) const res = await fetch(url).catch((err) => { - throw new BuildError(`--against ${opts.against}: ${err instanceof Error ? err.message : String(err)}`) + throw new BuildError(`--against ${opts.against}: ${message(err)}`) }) if (!res.ok) throw new BuildError(`--against ${url} returned HTTP ${res.status}`) const commit = /** @type {{ cid?: string }} */ (await res.json()) diff --git a/src/build/keygen.js b/src/build/keygen.js index 7bde40d..9186604 100644 --- a/src/build/keygen.js +++ b/src/build/keygen.js @@ -9,6 +9,7 @@ import { writeFile } from 'node:fs/promises' import { parseArgs } from 'node:util' import { Secp256k1Keypair, formatMultikey } from '@atproto/crypto' +import { message } from '../message.js' import { BuildError } from './errors.js' const USAGE = `corpora keygen — generate a secp256k1 signing key @@ -56,7 +57,7 @@ export async function run(argv) { } catch (err) { const code = /** @type {NodeJS.ErrnoException} */ (err).code if (code === 'EEXIST') throw new BuildError(`${values.out} already exists; refusing to overwrite a signing key`) - throw new BuildError(`cannot write ${values.out}: ${err instanceof Error ? err.message : String(err)}`) + throw new BuildError(`cannot write ${values.out}: ${message(err)}`) } out.push(`private key written to ${values.out} (mode 0600)`) } else { diff --git a/src/build/lexicons.js b/src/build/lexicons.js index 2738ec6..911ffa2 100644 --- a/src/build/lexicons.js +++ b/src/build/lexicons.js @@ -8,8 +8,9 @@ import { createHash } from 'node:crypto' import { readFile, readdir } from 'node:fs/promises' -import { join, sep } from 'node:path' +import { join, relative, resolve, sep } from 'node:path' import { lexiconDocumentSchema } from '@atproto/lex-document' +import { message } from '../message.js' import { BuildError } from './errors.js' import { isValidNsid, nsidError } from './nsid.js' @@ -24,13 +25,19 @@ export const COLLECTION = 'com.atproto.lexicon.schema' */ /** + * `dir` is whatever the caller typed — `./lexicons`, `lexicons/`, an absolute path — and is + * resolved once, here, so that every path below this line is absolute and normalized. Comparing + * an un-normalized directory against the paths `readdir` hands back is what used to turn + * `--lexicons ./lexicons` into a doubled path and a raw ENOENT. + * * @param {string} dir * @returns {Promise<{ sources: LexiconSource[], warnings: string[] }>} * @throws {BuildError} listing every problem found across every file */ export async function loadLexicons(dir) { - const files = await listJsonFiles(dir) - if (files.length === 0) throw new BuildError(`no .json lexicon files under ${dir}`) + const root = resolve(dir) + const files = await listJsonFiles(root) + if (files.length === 0) throw new BuildError(`no .json lexicon files under ${root}`) /** @type {string[]} */ const problems = [] @@ -40,7 +47,7 @@ export async function loadLexicons(dir) { const byNsid = new Map() for (const path of files) { - const bytes = await readFile(join(dir, ...path.split('/'))) + const bytes = await readFile(join(root, ...path.split('/'))) /** @type {(msg: string) => number} */ const problem = (msg) => problems.push(`${path}: ${msg}`) @@ -49,7 +56,7 @@ export async function loadLexicons(dir) { try { doc = JSON.parse(bytes.toString('utf8')) } catch (err) { - problem(`invalid JSON: ${err instanceof Error ? err.message : String(err)}`) + problem(`invalid JSON: ${message(err)}`) continue } if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) { @@ -85,7 +92,7 @@ export async function loadLexicons(dir) { try { lexiconDocumentSchema.parse(record) } catch (err) { - problem(err instanceof Error ? err.message : String(err)) + problem(message(err)) continue } @@ -121,7 +128,7 @@ export async function loadLexicons(dir) { } if (problems.length) { - throw new BuildError(`${problems.length} problem${problems.length === 1 ? '' : 's'} in ${dir}`, problems) + throw new BuildError(`${problems.length} problem${problems.length === 1 ? '' : 's'} in ${root}`, problems) } const sources = [...byNsid.values()].sort((a, b) => (a.nsid < b.nsid ? -1 : 1)) @@ -129,36 +136,27 @@ export async function loadLexicons(dir) { } /** - * Every `.json` file under `dir`, recursively, as POSIX-separated relative paths, sorted. + * Every `.json` file under `root`, recursively, as POSIX-separated relative paths, sorted. * Dotfiles and dot-directories are skipped. * - * @param {string} dir + * @param {string} root absolute and normalized, as `loadLexicons` resolved it * @returns {Promise} */ -async function listJsonFiles(dir) { +async function listJsonFiles(root) { /** @type {import('node:fs').Dirent[]} */ let entries try { - entries = await readdir(dir, { recursive: true, withFileTypes: true }) + entries = await readdir(root, { recursive: true, withFileTypes: true }) } catch (err) { - throw new BuildError(`cannot read lexicon directory ${dir}: ${err instanceof Error ? err.message : String(err)}`) + throw new BuildError(`cannot read lexicon directory ${root}: ${message(err)}`) } return entries .filter((e) => e.isFile() && e.name.endsWith('.json') && !e.name.startsWith('.')) - .map((e) => relativeTo(dir, join(e.parentPath, e.name))) + .map((e) => relative(root, join(e.parentPath, e.name)).split(sep).join('/')) .filter((p) => !p.split('/').some((seg) => seg.startsWith('.'))) .sort() } -/** - * @param {string} dir - * @param {string} path - */ -function relativeTo(dir, path) { - const rel = path.startsWith(dir) ? path.slice(dir.length) : path - return rel.split(sep).filter(Boolean).join('/') -} - /** * How a file path relates to the id inside it. The id is authoritative either way; this only * decides whether to say something about the disagreement. diff --git a/src/message.js b/src/message.js new file mode 100644 index 0000000..917419c --- /dev/null +++ b/src/message.js @@ -0,0 +1,20 @@ +/** + * Rendering an unknown throw as text. + * + * `catch (err)` binds `unknown`: JavaScript lets any value be thrown, and the DNS resolver, a + * lexicon parser, or a stranger's `dist/data.js` may throw a string as readily as an `Error`. + * Every place that puts a caught value into a message needs the same two-line dance, so it + * lives here once. + * + * This module is a leaf that imports nothing, and it belongs to no subsystem: `src/build/**`, + * `src/verify/**`, and `bin/**` all reach it without any of them depending on each other. + * `src/serve/**`, `src/worker/**`, and `src/node/**` do not use it — see docs/INTERFACES.md. + */ + +/** + * @param {unknown} err + * @returns {string} `err.message` for an `Error`, and `String(err)` for anything else + */ +export function message(err) { + return err instanceof Error ? err.message : String(err) +} diff --git a/src/node/index.js b/src/node/index.js index 03856ef..02889e8 100644 --- a/src/node/index.js +++ b/src/node/index.js @@ -56,17 +56,29 @@ function toHeaders(incoming) { * @param {CorporaData} data * @param {HandlerOptions & { port?: number, host?: string }} [opts] * @returns {Promise} + * @throws {RangeError} when `port` is not a TCP port number */ export function serve(data, opts = {}) { const { port = 2583, host = '127.0.0.1' } = opts + + // `listen()` reads a non-numeric string as a pipe or unix-socket name, and `address()` then + // answers with that path instead of a record — leaving no port to report and a URL of + // `http://127.0.0.1://run/x.sock`, which is nobody's server. Sockets are not a supported + // input, so the declared type is enforced rather than half-honoured. Node itself rejects an + // out-of-range *number* this way (RangeError, before anything binds); the range is the one + // `corpora serve --port` documents, so the library and the flag agree on what a port is. + if (!Number.isInteger(port) || port < 0 || port > 65535) { + throw new RangeError(`serve: port must be 0-65535, got '${String(port)}'`) + } + const server = createServer(createRequestListener(createHandler(data, opts))) return new Promise((resolve, reject) => { server.once('error', reject) server.listen(port, host, () => { server.removeListener('error', reject) - const address = server.address() - const bound = typeof address === 'object' && address !== null ? address.port : port + // A TCP listener always reports an `AddressInfo`; the string form means a pipe, refused above. + const bound = /** @type {import('node:net').AddressInfo} */ (server.address()).port resolve({ url: `http://${bracket(host)}:${bound}`, port: bound, diff --git a/src/serve/index.js b/src/serve/index.js index 648212a..0b421fb 100644 --- a/src/serve/index.js +++ b/src/serve/index.js @@ -37,12 +37,13 @@ export function createHandler(data, opts = {}) { const landing = opts.landing === false ? null : utf8(renderLanding(data)) const favicon = utf8(FAVICON) - // Everything the list endpoint can answer, precomputed. `data.list` is the order the - // build chose, which is by definition the `reverse=false` order. - const forward = data.list.flatMap((page) => page.records) - const backward = [...forward].reverse() - const forwardPos = positions(forward) - const backwardPos = positions(backward) + // Everything the list endpoint can answer, precomputed. `data.order` is the order the + // build chose, as rkeys, which is by definition the `reverse=false` order. + // One array and one map, both directions. `reverse=true` is the same sequence read from the + // other end, so a record at index `i` sits at `length - 1 - i` reversed — a mirrored copy of + // either structure would only restate that. + const forward = data.order.map((rkey) => data.records[rkey]) + const position = positions(forward) return function handle(req) { if (req.method === 'OPTIONS') return preflight() @@ -72,11 +73,9 @@ export function createHandler(data, opts = {}) { // The critical path: this is the only endpoint the OAuth resolution chain calls. case '/xrpc/com.atproto.sync.getRecord': { const nsid = 'com.atproto.sync.getRecord' - const missing = requireParams(nsid, q, ['did', 'collection', 'rkey'], head) - if (missing) return missing - const did = str(q, 'did') - const collection = str(q, 'collection') - const rkey = str(q, 'rkey') + const params = requireParams(nsid, q, ['did', 'collection', 'rkey'], head) + if (params instanceof Response) return params + const [did, collection, rkey] = params if (did !== data.did) return repoNotFound(did, head) const proof = collection === data.collection ? data.proofs[rkey] : undefined // 404, where the repo.* endpoints below use 400. A PDS answers an absent rkey @@ -88,11 +87,9 @@ export function createHandler(data, opts = {}) { case '/xrpc/com.atproto.repo.getRecord': { const nsid = 'com.atproto.repo.getRecord' - const missing = requireParams(nsid, q, ['repo', 'collection', 'rkey'], head) - if (missing) return missing - const repo = str(q, 'repo') - const collection = str(q, 'collection') - const rkey = str(q, 'rkey') + const params = requireParams(nsid, q, ['repo', 'collection', 'rkey'], head) + if (params instanceof Response) return params + const [repo, collection, rkey] = params if (repo !== data.did) return repoNotResolvable(repo, head) const record = collection === data.collection ? data.records[rkey] : undefined const cid = q.get('cid') @@ -104,8 +101,9 @@ export function createHandler(data, opts = {}) { case '/xrpc/com.atproto.repo.listRecords': { const nsid = 'com.atproto.repo.listRecords' - const missing = requireParams(nsid, q, ['repo', 'collection'], head) - if (missing) return missing + const params = requireParams(nsid, q, ['repo', 'collection'], head) + if (params instanceof Response) return params + const [repo, collection] = params // Parameter types are checked before the repo is looked at, because a reference // PDS validates against the lexicon in a layer above the handler. const limit = parseLimit(nsid, q, head) @@ -113,27 +111,30 @@ export function createHandler(data, opts = {}) { const reverse = parseBool(nsid, q, 'reverse', head) if (reverse instanceof Response) return reverse - const repo = str(q, 'repo') if (repo !== data.did) return repoNotResolvable(repo, head) // An unknown collection is an empty page, not an error: that is what a reference // PDS returns, and for a repo with exactly one collection it cannot mislead. - if (str(q, 'collection') !== data.collection) { + if (collection !== data.collection) { return json({ records: [] }, { head, cacheControl: staticCache }) } - const ordered = reverse ? backward : forward + const total = forward.length let start = 0 const cursor = q.get('cursor') if (cursor !== null) { - const at = (reverse ? backwardPos : forwardPos).get(cursor) + const at = position.get(cursor) if (at === undefined) { return invalidRequest(`Invalid cursor: ${cursor}`, head) } - start = at + 1 + start = (reverse ? total - 1 - at : at) + 1 } - const records = ordered.slice(start, start + limit) + // `reverse` is an index transform, not a second array: walking `forward` backwards + // allocates exactly what a slice would, and costs no reversed copy to hold. + const end = Math.min(start + limit, total) + const records = [] + for (let i = start; i < end; i++) records.push(forward[reverse ? total - 1 - i : i]) // A reference PDS returns a cursor on the final page too, so a client always // makes one empty request to learn it is done. Matched deliberately. const last = records[records.length - 1] @@ -142,9 +143,9 @@ export function createHandler(data, opts = {}) { } case '/xrpc/com.atproto.sync.getRepo': { - const missing = requireParams('com.atproto.sync.getRepo', q, ['did'], head) - if (missing) return missing - const did = str(q, 'did') + const params = requireParams('com.atproto.sync.getRepo', q, ['did'], head) + if (params instanceof Response) return params + const [did] = params if (did !== data.did) return repoNotFound(did, head) // `since` is ignored: a static authority holds one revision, and the full CAR is // always a valid superset of any diff a caller could ask for. @@ -152,17 +153,17 @@ export function createHandler(data, opts = {}) { } case '/xrpc/com.atproto.repo.describeRepo': { - const missing = requireParams('com.atproto.repo.describeRepo', q, ['repo'], head) - if (missing) return missing - const repo = str(q, 'repo') + const params = requireParams('com.atproto.repo.describeRepo', q, ['repo'], head) + if (params instanceof Response) return params + const [repo] = params if (repo !== data.did) return repoNotFound(repo, head) return json(data.describeRepo, { head, cacheControl: staticCache }) } case '/xrpc/com.atproto.sync.getLatestCommit': { - const missing = requireParams('com.atproto.sync.getLatestCommit', q, ['did'], head) - if (missing) return missing - const did = str(q, 'did') + const params = requireParams('com.atproto.sync.getLatestCommit', q, ['did'], head) + if (params instanceof Response) return params + const [did] = params if (did !== data.did) return repoNotFound(did, head) return json(data.commit, { head, cacheControl: staticCache }) } @@ -171,9 +172,9 @@ export function createHandler(data, opts = {}) { // learns the repo is healthy and static, which is exactly what it is. Corpora serves no // firehose, so nothing here implies it can be tracked — see docs/RELAYS.md. case '/xrpc/com.atproto.sync.getRepoStatus': { - const missing = requireParams('com.atproto.sync.getRepoStatus', q, ['did'], head) - if (missing) return missing - const did = str(q, 'did') + const params = requireParams('com.atproto.sync.getRepoStatus', q, ['did'], head) + if (params instanceof Response) return params + const [did] = params if (did !== data.did) return repoNotFound(did, head) return json({ did: data.did, active: true, rev: data.rev }, { head, cacheControl: staticCache }) } @@ -195,9 +196,9 @@ export function createHandler(data, opts = {}) { // Lexicons are JSON; blobs are a non-goal. An empty list is the truthful answer and // stops a mirror treating a 404 here as the host being broken. case '/xrpc/com.atproto.sync.listBlobs': { - const missing = requireParams('com.atproto.sync.listBlobs', q, ['did'], head) - if (missing) return missing - const did = str(q, 'did') + const params = requireParams('com.atproto.sync.listBlobs', q, ['did'], head) + if (params instanceof Response) return params + const [did] = params if (did !== data.did) return repoNotFound(did, head) return json({ cids: [] }, { head, cacheControl: staticCache }) } @@ -228,33 +229,30 @@ function rkeyOf(uri) { return uri.slice(uri.lastIndexOf('/') + 1) } -/** - * Read a parameter already known to be present. - * @param {URLSearchParams} q - * @param {string} name - * @returns {string} - */ -function str(q, name) { - return q.get(name) ?? '' -} - /** * Required-parameter check, in the order the lexicon declares them, with the message a * reference PDS emits verbatim. * + * Returns the values in the order asked for, so a caller that got past the check holds strings + * rather than a second reason to ask `q` whether the parameter it just validated is there. + * * @param {string} nsid * @param {URLSearchParams} q * @param {string[]} names * @param {boolean} head - * @returns {Response | null} + * @returns {Response | string[]} */ function requireParams(nsid, q, names, head) { + /** @type {string[]} */ + const values = [] for (const name of names) { - if (q.get(name) === null) { + const value = q.get(name) + if (value === null) { return invalidRequest(`Invalid ${nsid} params: Missing required key "${name}"`, head) } + values.push(value) } - return null + return values } /** diff --git a/src/serve/landing.js b/src/serve/landing.js index f364e36..3121a36 100644 --- a/src/serve/landing.js +++ b/src/serve/landing.js @@ -75,7 +75,7 @@ footer { margin-top:3rem; padding-top:1rem; border-top:1px solid var(--rule); co * @returns {string} a complete HTML document */ export function renderLanding(data) { - const nsids = [...new Set(data.list.flatMap((page) => page.records.map((r) => rkeyOf(r.uri))))].sort() + const nsids = [...new Set(data.order)].sort() const proofOf = (/** @type {string} */ nsid) => xrpcPath('com.atproto.sync.getRecord', { did: data.did, collection: data.collection, rkey: nsid }) @@ -176,11 +176,6 @@ function xrpcPath(nsid, params) { return `/xrpc/${nsid}?${query}` } -/** @param {string} uri at://// */ -function rkeyOf(uri) { - return uri.slice(uri.lastIndexOf('/') + 1) -} - /** * Link text only. The href always carries the full query string. * @param {string} href diff --git a/src/serve/types.js b/src/serve/types.js index 918e5d6..961fcfe 100644 --- a/src/serve/types.js +++ b/src/serve/types.js @@ -34,6 +34,10 @@ /** * A `com.atproto.repo.listRecords` response body. + * + * Written to `list/page-N.json` in the static tree. The embedded bundle does not carry these: + * it carries `order` and `records`, and the handler pages over them per request. + * * @typedef {object} ListRecordsPage * @property {RecordResponse[]} records * @property {string} [cursor] @@ -66,7 +70,7 @@ * @property {Record} didDoc * @property {CommitResponse} commit * @property {DescribeRepoResponse} describeRepo - * @property {ListRecordsPage[]} list + * @property {string[]} order rkeys in `listRecords` `reverse=false` order * @property {Record} records * @property {Record} proofs * @property {Uint8Array} repoCar diff --git a/src/verify/index.js b/src/verify/index.js index d2e9add..68a8ef9 100644 --- a/src/verify/index.js +++ b/src/verify/index.js @@ -22,6 +22,8 @@ import { join } from 'node:path' import { parseToAtprotoDocument } from '@atproto/identity' import { def, MemoryBlockstore, MST, readCarWithRoot, verifyCommitSig } from '@atproto/repo' +import { message } from '../message.js' + const COLLECTION = 'com.atproto.lexicon.schema' const CAR_CONTENT_TYPE = 'application/vnd.ipld.car' @@ -507,13 +509,15 @@ function equalBytes(a, b) { return a.length === b.length && a.every((v, i) => v === b[i]) } -/** @param {string} endpoint @param {string} origin */ +/** + * Both arguments are known-parseable by the time this runs: `endpoint` is the `pds` that + * `@atproto/common-web`'s `validateUrl` already ran `URL.canParse` over, and `origin` served as + * the base of every `new URL(path, ...)` above. Neither `new URL` here can throw. + * + * @param {string} endpoint @param {string} origin + */ function sameOrigin(endpoint, origin) { - try { - return new URL(endpoint).origin === new URL(origin).origin - } catch { - return false - } + return new URL(endpoint).origin === new URL(origin).origin } /** @param {Response} res */ @@ -531,11 +535,6 @@ function short(cid) { return cid.length > 16 ? `${cid.slice(0, 10)}…${cid.slice(-4)}` : cid } -/** @param {unknown} err */ -function message(err) { - return err instanceof Error ? err.message : String(err) -} - /** @param {unknown} err */ function errCode(err) { const code = /** @type {{ code?: string }} */ (err)?.code diff --git a/test/build/build.test.js b/test/build/build.test.js index 326d2e4..e0a9842 100644 --- a/test/build/build.test.js +++ b/test/build/build.test.js @@ -1,6 +1,7 @@ import assert from 'node:assert/strict' import { createHash } from 'node:crypto' import { readFile, writeFile } from 'node:fs/promises' +import { createServer } from 'node:http' import { join } from 'node:path' import test from 'node:test' import { Secp256k1Keypair } from '@atproto/crypto' @@ -20,6 +21,25 @@ async function buildFixtures(t, opts = {}) { return { out, report } } +/** + * Every `{uri, cid, value}` reachable in a bundle, however deeply nested. Structural rather + * than textual, so it survives reformatting and catches a duplicate under any key. + * + * @param {unknown} value + * @param {Record[]} [found] + * @returns {Record[]} + */ +function recordShaped(value, found = []) { + if (Array.isArray(value)) { + for (const item of value) recordShaped(item, found) + } else if (value && typeof value === 'object') { + const v = /** @type {Record} */ (value) + if (typeof v.uri === 'string' && typeof v.cid === 'string' && typeof v.value === 'object') found.push(v) + for (const item of Object.values(v)) recordShaped(item, found) + } + return found +} + test('emits the dist tree docs/INTERFACES.md specifies', async (t) => { const { out, report } = await buildFixtures(t) const tree = [...(await readTree(out)).keys()].sort() @@ -148,7 +168,20 @@ test('data.js embeds every artifact and decodes without dependencies', async (t) assert.equal(data.commitCid, report.commitCid) assert.deepEqual(Object.keys(data.records), report.nsids) assert.deepEqual(data.manifest, report.manifest) - assert.equal(data.list.length, 1) + // `order` carries rkeys, not record bodies. Descending, which is the `reverse=false` order + // a reference PDS returns and the opposite of the ascending order the manifest lists. + assert.deepEqual(data.order, [...report.nsids].reverse()) + assert.equal(data.list, undefined) + + // The guard that keeps it that way: every `{uri, cid, value}` in the bundle lives under + // `records`. A second view of the same records — paged, reversed, or grouped — would show + // up here as a multiple of the namespace size. + const embedded = recordShaped(data) + assert.equal( + embedded.length, + report.nsids.length, + `bundle embeds ${embedded.length} record bodies for ${report.nsids.length} lexicons`, + ) for (const nsid of report.nsids) { assert.ok(data.proofs[nsid] instanceof Uint8Array) @@ -285,6 +318,47 @@ test('a did:plc document is checked rather than minted', async (t) => { assert.match(err.message, /names signing key .*but the build signed with/s) }) +test('a did:plc document is fetched from the directory when none is supplied', async (t) => { + // The document is PLC's to publish, so the build has to go and read it. Everything downstream — + // dist/did.json, the service endpoint, the key check — is then whatever came back over the wire. + const keypair = await Secp256k1Keypair.import(TEST_KEY) + const did = 'did:plc:wfj5kyialpmcv2fzk6uqwsln' + const published = { + id: did, + alsoKnownAs: [], + verificationMethod: [ + { + id: `${did}#atproto`, + type: 'Multikey', + controller: did, + publicKeyMultibase: keypair.did().slice('did:key:'.length), + }, + ], + service: [{ id: '#atproto_pds', type: 'AtprotoPersonalDataServer', serviceEndpoint: 'https://plc.example.com' }], + } + + /** @type {string[]} */ + const asked = [] + t.mock.method(globalThis, 'fetch', async (/** @type {URL | string} */ url) => { + asked.push(String(url)) + return Response.json(published) + }) + + const { out, report } = await buildFixtures(t, { did }) + assert.deepEqual(asked, [`https://plc.directory/${encodeURIComponent(did)}`]) + assert.deepEqual(JSON.parse(await readFile(join(out, 'did.json'), 'utf8')), published) + assert.equal(report.service, 'https://plc.example.com', 'the endpoint came from somewhere other than PLC') +}) + +test('--service is refused for a did:plc, whose document is not ours to write', async (t) => { + // plc.directory serves the document; an endpoint passed here would be written into dist/did.json + // and describeRepo while resolvers went on using the one PLC publishes. + const did = 'did:plc:wfj5kyialpmcv2fzk6uqwsln' + const err = await rejects(() => buildFixtures(t, { did, service: 'https://elsewhere.example.com' })) + assert.match(err.message, /--service applies to did:web only/) + assert.match(err.message, /plc\.directory/) +}) + test('a malformed DID document fails the build', async (t) => { const did = 'did:plc:wfj5kyialpmcv2fzk6uqwsln' const err = await rejects(() => @@ -448,6 +522,119 @@ test('a deleted lexicon disappears from dist', async (t) => { assert.equal(tree.includes(join('proofs', 'com.example.other.car')), false) }) +/** + * A stand-in for a deployed authority, answering `com.atproto.sync.getLatestCommit` with whatever + * the test wants and recording what was asked. One route is the whole surface `--against` uses. + * + * @param {import('node:test').TestContext} t + * @param {(url: URL) => { status?: number, body?: unknown }} respond + * @returns {Promise<{ origin: string, asked: URL[] }>} + */ +async function deployedAuthority(t, respond) { + /** @type {URL[]} */ + const asked = [] + const server = createServer((req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1') + asked.push(url) + const { status = 200, body = {} } = respond(url) + res.writeHead(status, { 'content-type': 'application/json' }) + res.end(JSON.stringify(body)) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(undefined))) + t.after(() => new Promise((resolve) => server.close(() => resolve(undefined)))) + const address = server.address() + assert.ok(address && typeof address !== 'string', 'the stand-in authority did not bind a port') + return { origin: `http://127.0.0.1:${address.port}`, asked } +} + +test('--against skips the build when the deployed authority already serves this commit', async (t) => { + const { report } = await buildFixtures(t) + const authority = await deployedAuthority(t, () => ({ body: { cid: report.commitCid, rev: report.rev } })) + + const out = await tempDir(t) + const skipped = await build({ + lexicons: FIXTURES, + out, + did: TEST_DID, + key: TEST_KEY, + dnsCheck: false, + against: authority.origin, + }) + assert.equal(skipped.written, false) + assert.match(/** @type {string} */ (skipped.unchanged), new RegExp(`${authority.origin} already serves commit`)) + assert.match(/** @type {string} */ (skipped.unchanged), new RegExp(report.commitCid)) + assert.deepEqual([...(await readTree(out)).keys()], [], 'the skipped build wrote to --out anyway') + + // The comparison is per-DID: an authority serving several repos is asked about ours. + assert.deepEqual( + authority.asked.map((u) => `${u.pathname}?${u.searchParams}`), + [`/xrpc/com.atproto.sync.getLatestCommit?did=${encodeURIComponent(TEST_DID)}`], + ) +}) + +test('--against writes when the deployed commit is a different one, and --force ignores it entirely', async (t) => { + const authority = await deployedAuthority(t, () => ({ body: { cid: 'bafysomethingelse', rev: '3lxaaaaaaaa22' } })) + const { report } = await buildFixtures(t, { against: authority.origin }) + assert.equal(report.written, true) + assert.equal(report.unchanged, undefined) + + const forced = await buildFixtures(t, { against: authority.origin, force: true }) + assert.equal(forced.report.written, true) + assert.equal(authority.asked.length, 1, '--force asked the deployed authority anyway') +}) + +test('--against reports an authority that answers with an error status', async (t) => { + const authority = await deployedAuthority(t, () => ({ status: 503, body: { error: 'ServiceUnavailable' } })) + const err = await rejects(() => buildFixtures(t, { against: authority.origin })) + assert.match(err.message, /returned HTTP 503/) + assert.match(err.message, /com\.atproto\.sync\.getLatestCommit/) +}) + +test('--against reports an authority it cannot reach, rather than rebuilding silently', async (t) => { + // Port 1 is privileged and bound by nothing, so the connection is refused immediately. Treating + // an unreachable authority as "changed" would be the wrong default: it hides an outage behind a + // successful-looking build. + const err = await rejects(() => buildFixtures(t, { against: 'http://127.0.0.1:1' })) + assert.match(err.message, /--against http:\/\/127\.0\.0\.1:1: /) + assert.match(err.message, /fetch failed|ECONNREFUSED/) +}) + +test('a rebuild after a reformat is still skipped, and says the records did not move', async (t) => { + // Reindenting a lexicon changes its bytes and nothing else. The source hashes in the manifest + // move, the MST root does not, and reporting only "unchanged" would leave an operator staring + // at a diff they can see and a build that claims there is nothing to do. + const out = await tempDir(t) + const dir = await lexiconDir(t, { 'com.example.thing.json': doc('com.example.thing') }) + /** @param {Partial} [opts] */ + const run = (opts) => build({ lexicons: dir, out, did: TEST_DID, key: TEST_KEY, dnsCheck: false, ...opts }) + + const first = await run() + assert.equal(first.written, true) + + const path = join(dir, 'com.example.thing.json') + await writeFile(path, JSON.stringify(JSON.parse(await readFile(path, 'utf8')))) + const second = await run() + + assert.equal(second.written, false) + assert.match(/** @type {string} */ (second.unchanged), /already holds MST root/) + assert.match( + /** @type {string} */ (second.unchanged), + /the source files changed but the records they encode to did not/, + ) + assert.equal(second.mstRoot, first.mstRoot) + assert.notDeepEqual(second.manifest.sources, first.manifest.sources, 'the reformat left the source hashes alone') +}) + +test('an already-imported keypair is accepted where the hex would go', async (t) => { + // The programmatic entry point is what a caller holding a key from somewhere else — a keystore, + // a hardware token, `corpora keygen` in the same process — has to reach for. + const keypair = await Secp256k1Keypair.import(TEST_KEY) + const { report } = await buildFixtures(t, { key: keypair }) + const fromHex = await buildFixtures(t) + assert.equal(report.signingKey, keypair.did()) + assert.equal(report.commitCid, fromHex.report.commitCid, 'the same key by another name signed a different repo') +}) + test('a key that is not 32 hex bytes is refused before anything is read', async (t) => { const err = await rejects(() => buildFixtures(t, { key: 'not-a-key' })) assert.match(err.message, /64 hex characters/) @@ -457,3 +644,12 @@ test('a DID that is neither did:web nor did:plc is refused', async (t) => { const err = await rejects(() => buildFixtures(t, { did: 'did:key:zQ3sh' })) assert.match(err.message, /did:web or a did:plc/) }) + +test('no DID at all is refused as nothing, rather than quoted as an empty string', async (t) => { + // The field a caller forgot to fill in, which `Partial` is happy to leave out. + const missing = await rejects(() => buildFixtures(t, { did: undefined })) + assert.match(missing.message, /--did must be a did:web or a did:plc, got nothing/) + + const blank = await rejects(() => buildFixtures(t, { did: '' })) + assert.match(blank.message, /got nothing/) +}) diff --git a/test/build/cli.test.js b/test/build/cli.test.js index 116025f..1270595 100644 --- a/test/build/cli.test.js +++ b/test/build/cli.test.js @@ -1,11 +1,12 @@ import assert from 'node:assert/strict' import { readFile, stat } from 'node:fs/promises' -import { join } from 'node:path' +import { join, relative } from 'node:path' import test from 'node:test' import { Secp256k1Keypair } from '@atproto/crypto' import { run as buildCli } from '../../src/build/cli.js' +import { BuildError } from '../../src/build/errors.js' import { run as keygenCli } from '../../src/build/keygen.js' -import { FIXTURES, TEST_DID, TEST_KEY, tempDir } from './helpers.js' +import { FIXTURES, TEST_DID, TEST_KEY, rejects, tempDir } from './helpers.js' /** * Runs a subcommand with stdout and stderr captured. @@ -49,6 +50,32 @@ test('build prints a summary and writes dist', async (t) => { assert.equal(manifest.did, TEST_DID) }) +test('build accepts the --lexicons path a shell would hand it', async (t) => { + // `corpora build --lexicons ./lexicons` is what tab completion produces and what most people + // type. It reached the loader un-normalized once, which doubled the directory onto every file + // and surfaced as a bare ENOENT with no file named and no problem list. + const dist = await tempDir(t) + process.env.CORPORA_SIGNING_KEY = TEST_KEY + t.after(() => delete process.env.CORPORA_SIGNING_KEY) + + // Relative to wherever the runner was started, so this does not pin a cwd. + const dotted = `./${relative(process.cwd(), FIXTURES)}` + const { code, out, err } = await capture(t, buildCli, [ + '--lexicons', + dotted, + '--out', + dist, + '--did', + TEST_DID, + '--no-dns-check', + ]) + assert.equal(code, 0, err) + assert.match(out, /com\.example\.thing .* record .* proof/) + + const manifest = JSON.parse(await readFile(join(dist, 'manifest.json'), 'utf8')) + assert.equal(manifest.did, TEST_DID) +}) + test('build stays quiet with --quiet and reads --key-file', async (t) => { const dir = await tempDir(t) const keyFile = join(dir, 'signing.key') @@ -101,6 +128,128 @@ test('build reports missing arguments as an exit code, not a stack trace', async assert.match(err, /--lexicons is required/) }) +test('build names the other required argument too, once --lexicons is there', async (t) => { + // The required-argument checks are separate lines: a run that satisfies the first must still be + // told about the second, rather than falling through to a DID-shape error from the builder. + const { code, err } = await capture(t, buildCli, ['--lexicons', FIXTURES]) + assert.equal(code, 1) + assert.match(err, /--did is required/) +}) + +test('a localhost DID is built, but the summary and stderr both say it goes nowhere', async (t) => { + // did:web:localhost%3A2583 is the shape the dev server uses, and the only one the spec lets + // carry a port. It builds, and every place that could mislead someone into deploying it says so. + const dist = await tempDir(t) + process.env.CORPORA_SIGNING_KEY = TEST_KEY + t.after(() => delete process.env.CORPORA_SIGNING_KEY) + + const { code, out, err } = await capture(t, buildCli, [ + '--lexicons', + FIXTURES, + '--out', + dist, + '--did', + 'did:web:localhost%3A2583', + '--no-dns-check', + ]) + assert.equal(code, 0) + assert.match(out, /^service {6}http:\/\/localhost:2583 {2}\(development only, not resolvable elsewhere\)$/m) + assert.match(err, /^warning: did:web:localhost%3A2583 is the local development form/m) +}) + +test('a second identical build writes nothing and says what it matched', async (t) => { + // Republishing an unchanged repo churns a commit for nothing. The second run has to make the + // reason legible, and point at the flag that overrides it. + const dist = await tempDir(t) + process.env.CORPORA_SIGNING_KEY = TEST_KEY + t.after(() => delete process.env.CORPORA_SIGNING_KEY) + + const argv = ['--lexicons', FIXTURES, '--out', dist, '--did', TEST_DID, '--no-dns-check'] + const first = await capture(t, buildCli, argv) + assert.equal(first.code, 0) + assert.match(first.out, /wrote \d+ files/) + + const second = await capture(t, buildCli, argv) + assert.equal(second.code, 0) + assert.match(second.out, /^unchanged: .*already holds MST root /m) + assert.match(second.out, /nothing written; pass --force to rebuild anyway/) + assert.equal(second.out.includes('wrote '), false, 'the unchanged build claimed to have written') +}) + +test('a failure that is not a BuildError is not swallowed into exit 1', async (t) => { + // The CLI reports its own BuildErrors and rethrows everything else, so a bug surfaces as a bug + // instead of an exit code with no explanation. `--against notaurl` is the shortest way there: + // the origin is fed straight to `new URL`, which throws a plain TypeError. + process.env.CORPORA_SIGNING_KEY = TEST_KEY + t.after(() => delete process.env.CORPORA_SIGNING_KEY) + + const out = await tempDir(t) + const err = await rejects(() => + capture(t, buildCli, [ + '--lexicons', + FIXTURES, + '--out', + out, + '--did', + TEST_DID, + '--no-dns-check', + '--against', + 'notaurl', + ]), + ) + assert.equal(err instanceof BuildError, false, 'a TypeError was reported as if the input were understood') + assert.match(err.message, /Invalid URL/) +}) + +test('build --help prints options to stdout and builds nothing', async (t) => { + const { code, out, err } = await capture(t, buildCli, ['--help']) + assert.equal(code, 0) + assert.match(out, /^corpora build — sign a lexicon directory into dist\//) + assert.match(out, /--dns-check /) + assert.equal(err, '') +}) + +test('build says which key file it could not read, and does not fall back to the environment', async (t) => { + const missing = join(await tempDir(t), 'absent.key') + process.env.CORPORA_SIGNING_KEY = TEST_KEY + t.after(() => delete process.env.CORPORA_SIGNING_KEY) + + const { code, err } = await capture(t, buildCli, [ + '--lexicons', + FIXTURES, + '--out', + await tempDir(t), + '--did', + TEST_DID, + '--no-dns-check', + '--key-file', + missing, + ]) + assert.equal(code, 1) + assert.match(err, new RegExp(`cannot read --key-file ${missing}`)) + assert.match(err, /ENOENT/) +}) + +test('build with no key at all says where a key comes from', async (t) => { + const previous = process.env.CORPORA_SIGNING_KEY + delete process.env.CORPORA_SIGNING_KEY + t.after(() => { + if (previous !== undefined) process.env.CORPORA_SIGNING_KEY = previous + }) + + const { code, err } = await capture(t, buildCli, [ + '--lexicons', + FIXTURES, + '--out', + await tempDir(t), + '--did', + TEST_DID, + '--no-dns-check', + ]) + assert.equal(code, 1) + assert.match(err, /no signing key: pass --key-file or set CORPORA_SIGNING_KEY/) +}) + test('keygen writes a key file that build can import, and does not print it', async (t) => { const dir = await tempDir(t) const keyFile = join(dir, 'signing.key') diff --git a/test/build/diddoc.test.js b/test/build/diddoc.test.js new file mode 100644 index 0000000..91f056f --- /dev/null +++ b/test/build/diddoc.test.js @@ -0,0 +1,213 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + buildDidWebDocument, + checkDidDocument, + defaultServiceEndpoint, + didWebHost, + fetchPlcDocument, + isDevelopmentDid, +} from '../../src/build/diddoc.js' +import { rejects } from './helpers.js' + +const DID = 'did:web:lexicon.example.com' +const SIGNING_KEY = 'did:key:zQ3shokFTS3brHcDQrn82RUDfCZESWL1ZdCEJwekUDPQiYBme' + +/** + * A stand-in for `fetch` that records what the build asked for. + * + * @param {(url: URL, init: RequestInit) => Response | Promise} handler + */ +function fakeFetch(handler) { + // Wrapped so it returns a promise even when the handler throws, exactly as `fetch` does. + return /** @type {typeof globalThis.fetch} */ ( + /** @type {unknown} */ (async (/** @type {URL} */ url, /** @type {RequestInit} */ init) => handler(url, init)) + ) +} + +/** + * @param {() => unknown} fn + * @returns {Error & { details?: string[] }} + */ +function throws(fn) { + try { + fn() + } catch (err) { + return /** @type {Error & { details?: string[] }} */ (err) + } + throw new Error('expected a throw, got none') +} + +test('didWebHost decodes a port and rejects the path form', () => { + assert.equal(didWebHost(DID), 'lexicon.example.com') + assert.equal(didWebHost('did:web:localhost%3A2583'), 'localhost:2583') + + assert.match(throws(() => didWebHost('did:web:example.com:lexicons')).message, /supports only did:web:/) +}) + +test('didWebHost rejects a host it cannot parse', () => { + // `[` survives decodeURIComponent and carries no colon or slash, so it reaches the URL parser + // and fails there — the last line of defence before a garbage host is minted into a document. + const err = throws(() => didWebHost('did:web:%5B')) + assert.equal(err.message, 'did:web:%5B does not name a host') + + // The cheaper checks that never reach the parser. + assert.match(throws(() => didWebHost('did:web:')).message, /does not name a host/) + assert.match(throws(() => didWebHost('did:web:.example.com')).message, /does not name a host/) + assert.match(throws(() => didWebHost('did:web:example.com%2Flexicons')).message, /does not name a host/) +}) + +test('didWebHost allows a port only on localhost', () => { + const err = throws(() => didWebHost('did:web:example.com%3A8080')) + assert.match(err.message, /names a port, which is allowed only for localhost/) + assert.ok( + err.details?.some((d) => d.includes('goat rejects any port at all')), + 'the port error should explain why the two halves of the ecosystem disagree', + ) +}) + +test('isDevelopmentDid is true only for localhost did:web', () => { + assert.equal(isDevelopmentDid('did:web:localhost%3A2583'), true) + assert.equal(isDevelopmentDid('did:web:LOCALHOST'), true) + assert.equal(isDevelopmentDid(DID), false) + assert.equal(isDevelopmentDid('did:plc:ewvi7nxzyoun6zhxrhs64oiz'), false) +}) + +test('defaultServiceEndpoint downgrades localhost to http', () => { + assert.equal(defaultServiceEndpoint(DID), 'https://lexicon.example.com') + assert.equal(defaultServiceEndpoint('did:web:localhost%3A2583'), 'http://localhost:2583') +}) + +test('a minted did:web document round-trips through the resolver parser', () => { + const doc = buildDidWebDocument({ did: DID, signingKey: SIGNING_KEY }) + assert.equal(doc.id, DID) + assert.deepEqual(doc.alsoKnownAs, []) + assert.equal(checkDidDocument(doc, { did: DID, signingKey: SIGNING_KEY }), 'https://lexicon.example.com') + + const withService = buildDidWebDocument({ did: DID, signingKey: SIGNING_KEY, service: 'https://pds.example.com' }) + assert.equal( + checkDidDocument(withService, { did: DID, signingKey: SIGNING_KEY, service: 'https://pds.example.com' }), + 'https://pds.example.com', + ) +}) + +test('checkDidDocument reports schema issues rather than a parser stack trace', () => { + const err = throws(() => checkDidDocument({}, { did: DID, signingKey: SIGNING_KEY })) + assert.equal(err.message.split('\n')[0], 'the DID document is malformed') + assert.ok(err.details?.length, 'a malformed document should list its issues') + assert.ok( + err.details?.some((d) => d.startsWith('id:')), + `expected an issue about the missing id, got ${JSON.stringify(err.details)}`, + ) +}) + +test('checkDidDocument catches a document that names the wrong things', () => { + const doc = buildDidWebDocument({ did: DID, signingKey: SIGNING_KEY }) + const err = throws(() => checkDidDocument(doc, { did: 'did:web:other.example.com', signingKey: 'did:key:zOther' })) + assert.equal(err.message.split('\n')[0], 'the DID document would not resolve as an atproto identity') + assert.deepEqual(err.details, [ + `id is ${DID}, expected did:web:other.example.com`, + `names signing key ${SIGNING_KEY}, but the build signed with did:key:zOther`, + ]) +}) + +test('checkDidDocument catches a document with no usable key or service', () => { + const doc = /** @type {any} */ (buildDidWebDocument({ did: DID, signingKey: SIGNING_KEY })) + // Ids a resolver does not look for: same shape, no atproto identity. + doc.verificationMethod = [{ ...doc.verificationMethod[0], id: `${DID}#signing` }] + doc.service = [{ ...doc.service[0], id: '#atproto_labeler' }] + + const err = throws(() => checkDidDocument(doc, { did: DID, signingKey: SIGNING_KEY })) + assert.ok( + err.details?.some((d) => d.includes('no usable verificationMethod')), + `expected a verificationMethod complaint, got ${JSON.stringify(err.details)}`, + ) + assert.ok( + err.details?.some((d) => d.includes('no usable service')), + `expected a service complaint, got ${JSON.stringify(err.details)}`, + ) +}) + +test('checkDidDocument catches a service endpoint other than the one asked for', () => { + const doc = buildDidWebDocument({ did: DID, signingKey: SIGNING_KEY, service: 'https://old.example.com' }) + const err = throws(() => + checkDidDocument(doc, { did: DID, signingKey: SIGNING_KEY, service: 'https://new.example.com' }), + ) + assert.deepEqual(err.details, [ + 'publishes service endpoint https://old.example.com, expected https://new.example.com', + ]) +}) + +test('fetchPlcDocument asks the directory for one DID and returns its document', async () => { + const did = 'did:plc:ewvi7nxzyoun6zhxrhs64oiz' + /** @type {any[]} */ + const calls = [] + const doc = await fetchPlcDocument(did, { + fetch: fakeFetch((url, init) => { + calls.push({ url, init }) + return new Response(JSON.stringify({ id: did }), { headers: { 'content-type': 'application/json' } }) + }), + }) + assert.deepEqual(doc, { id: did }) + assert.equal(calls.length, 1) + // Percent-encoded exactly as @atproto/identity's own PlcResolver does it, so the build asks + // the directory the same question a resolver would. + assert.equal(String(calls[0].url), `https://plc.directory/${encodeURIComponent(did)}`) + // A redirect would silently move the identity's source of truth; refuse rather than follow. + assert.equal(calls[0].init.redirect, 'error') + assert.match(String(calls[0].init.headers?.accept), /application\/did\+ld\+json/) +}) + +test('fetchPlcDocument honours plcUrl and percent-encodes the DID into the path', async () => { + /** @type {string[]} */ + const seen = [] + await fetchPlcDocument('did:plc:abc', { + plcUrl: 'http://localhost:2582/', + fetch: fakeFetch((url) => { + seen.push(String(url)) + return new Response('{}') + }), + }) + assert.deepEqual(seen, ['http://localhost:2582/did%3Aplc%3Aabc']) +}) + +test('fetchPlcDocument falls back to the ambient fetch when given none', async (t) => { + t.mock.method(globalThis, 'fetch', async (/** @type {any} */ url) => + String(url) === 'https://plc.directory/did%3Aplc%3Aabc' + ? new Response('{"id":"did:plc:abc"}') + : new Response('wrong url', { status: 500 }), + ) + assert.deepEqual(await fetchPlcDocument('did:plc:abc'), { id: 'did:plc:abc' }) +}) + +test('fetchPlcDocument turns an unreachable directory into a build error', async () => { + const err = await rejects(() => + fetchPlcDocument('did:plc:abc', { + plcUrl: 'https://plc.example.com', + fetch: fakeFetch(() => { + throw new TypeError('fetch failed') + }), + }), + ) + assert.equal(err.name, 'BuildError') + assert.match(err.message, /could not reach https:\/\/plc\.example\.com\/did%3Aplc%3Aabc: fetch failed/) +}) + +test('fetchPlcDocument reports a non-Error rejection without losing it', async () => { + const err = await rejects(() => + fetchPlcDocument('did:plc:abc', { + fetch: fakeFetch(() => { + throw 'socket hang up' + }), + }), + ) + assert.match(err.message, /could not reach .*: socket hang up/) +}) + +test('fetchPlcDocument names the DID when the directory has never heard of it', async () => { + const err = await rejects(() => + fetchPlcDocument('did:plc:missing', { fetch: fakeFetch(() => new Response('not found', { status: 404 })) }), + ) + assert.equal(err.name, 'BuildError') + assert.match(err.message, /returned HTTP 404; is did:plc:missing registered\?/) +}) diff --git a/test/build/dns.test.js b/test/build/dns.test.js new file mode 100644 index 0000000..2daca09 --- /dev/null +++ b/test/build/dns.test.js @@ -0,0 +1,530 @@ +/** + * The authority-domain DNS check, branch by branch. + * + * `checkAuthorityDns` is mostly a decision table — three modes crossed with "record missing" and + * "record names someone else", each arm producing prose an operator has to act on — so the tests + * here are organised by mode and assert the wording, not just the throw. The singular/plural forks + * get their own cases because the plural arm is what a real multi-lexicon build hits and the + * singular arm is what the smallest repro hits; getting either wrong is the kind of thing nobody + * notices until it is in a bug report. + * + * Most cases inject `resolveTxt`, which is a documented option. One block deliberately does not: it + * points this process's resolver at the local DNS server so the default `node:dns/promises` path, + * and the real `ENOTFOUND`/`ENODATA` codes c-ares produces, are exercised for real rather than + * asserted from memory. + */ + +import assert from 'node:assert/strict' +import { after, before, describe, it } from 'node:test' + +import { checkAuthorityDns, dnsCheckMode } from '../../src/build/dns.js' +import { BuildError } from '../../src/build/errors.js' +import { startDnsServer, useLocalDns } from '../helpers/dns.js' + +const DID = 'did:web:corpora.example' +const OTHER = 'did:plc:o7hqxx2xhz3vqcpmrqjkw3ba' + +/** + * @param {string} message + * @param {string} code + */ +function dnsError(message, code) { + const err = /** @type {NodeJS.ErrnoException} */ (new Error(message)) + err.code = code + return err +} + +/** + * A `resolveTxt` stand-in over a fixed table, in the shape `node:dns` hands back: one array of + * character-string chunks per record. A name missing from the table answers `ENOTFOUND`, which is + * what a resolver says about a domain that does not exist. + * + * @param {Record} table + * @returns {(name: string) => Promise} + */ +function resolverOver(table) { + return async (name) => { + const records = table[name] + if (!records) throw dnsError(`queryTxt ENOTFOUND ${name}`, 'ENOTFOUND') + return records.map((record) => (Array.isArray(record) ? record : [record])) + } +} + +/** Fails the test if the check looks anything up at all. */ +const neverResolves = async (/** @type {string} */ name) => { + assert.fail(`resolveTxt was called for ${name}`) +} + +describe('dnsCheckMode', () => { + it('defaults to strict when nothing was passed', () => { + assert.equal(dnsCheckMode(undefined), 'strict') + }) + + it('reads a bare --dns-check as strict', () => { + assert.equal(dnsCheckMode(true), 'strict') + }) + + it('reads --no-dns-check as off', () => { + assert.equal(dnsCheckMode(false), 'off') + }) + + for (const mode of ['strict', 'cutover', 'off']) { + it(`passes '${mode}' through`, () => { + assert.equal(dnsCheckMode(mode), mode) + }) + } + + it('names the modes it accepts when given something else', () => { + assert.throws(() => dnsCheckMode('lenient'), (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /must be one of strict, cutover, off; got 'lenient'/) + return true + }) + }) + + it('rejects the empty string rather than treating it as absent', () => { + assert.throws(() => dnsCheckMode(''), BuildError) + }) +}) + +describe('checkAuthorityDns, off', () => { + it('returns no warnings and looks nothing up', async () => { + const warnings = await checkAuthorityDns( + { '_lexicon.example.com': ['com.example.thing'] }, + DID, + { mode: 'off', resolveTxt: neverResolves }, + ) + assert.deepEqual(warnings, []) + }) + + it('stays quiet even about a domain that does not exist', async () => { + const warnings = await checkAuthorityDns({ '_lexicon.nowhere.invalid': ['invalid.nowhere.x'] }, DID, { + mode: 'off', + resolveTxt: resolverOver({}), + }) + assert.deepEqual(warnings, []) + }) +}) + +describe('checkAuthorityDns, strict', () => { + it('passes when every domain names this DID', async () => { + const warnings = await checkAuthorityDns( + { + '_lexicon.example.com': ['com.example.thing'], + '_lexicon.nested.example.com': ['com.example.nested.widget'], + }, + DID, + { + mode: 'strict', + resolveTxt: resolverOver({ + '_lexicon.example.com': [`did=${DID}`], + '_lexicon.nested.example.com': [`did=${DID}`], + }), + }, + ) + assert.deepEqual(warnings, []) + }) + + it('joins a value split across character-strings before reading it', async () => { + // A TXT record longer than 255 bytes arrives in chunks; concatenating them is the difference + // between a match and a spurious mismatch. + const long = `did:web:${'a'.repeat(300)}.example` + const warnings = await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, long, { + resolveTxt: resolverOver({ + '_lexicon.example.com': [[`did=${long.slice(0, 200)}`, long.slice(200)]], + }), + }) + assert.deepEqual(warnings, []) + }) + + it('ignores TXT records that are not did=', async () => { + const warnings = await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({ + '_lexicon.example.com': ['v=spf1 -all', `did=${DID}`, 'google-site-verification=xyz'], + }), + }) + assert.deepEqual(warnings, []) + }) + + it('accepts a domain where one of several did= records names this DID', async () => { + // Two did= records is the state a half-finished cutover leaves. The build check only asks + // whether this DID is among them. + const warnings = await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({ '_lexicon.example.com': [`did=${OTHER}`, `did=${DID}`] }), + }) + assert.deepEqual(warnings, []) + }) + + it('trims whitespace around the DID', async () => { + const warnings = await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({ '_lexicon.example.com': [`did= ${DID}\t`] }), + }) + assert.deepEqual(warnings, []) + }) + + it('rejects a domain with no TXT record at all, in the singular', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({}), + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /^1 authority domain not pointing at did:web:corpora\.example$/m) + assert.deepEqual(err.details[0], '_lexicon.example.com has no `did=` TXT record (needed by com.example.thing)') + assert.ok(err.details.some((d) => d.includes(`IN TXT "did=${DID}"`))) + assert.ok(err.details.some((d) => d.includes('--dns-check cutover'))) + assert.ok(err.details.some((d) => d.includes('--no-dns-check'))) + return true + }, + ) + }) + + it('treats ENODATA — the name exists, the record does not — as missing', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: async () => { + throw dnsError('queryTxt ENODATA _lexicon.example.com', 'ENODATA') + }, + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.ok(err.details[0].includes('has no `did=` TXT record')) + return true + }, + ) + }) + + it('treats a domain with only unrelated TXT records as missing', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({ '_lexicon.example.com': ['v=spf1 -all'] }), + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.ok(err.details[0].includes('has no `did=` TXT record')) + return true + }, + ) + }) + + it('rejects a domain that names another DID, and says which', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({ '_lexicon.example.com': [`did=${OTHER}`] }), + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.equal(err.details[0], `_lexicon.example.com names ${OTHER}, not ${DID} (needed by com.example.thing)`) + return true + }, + ) + }) + + it('lists every did= a mismatched domain names', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: resolverOver({ '_lexicon.example.com': [`did=${OTHER}`, 'did=did:web:third.example'] }), + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.ok(err.details[0].startsWith(`_lexicon.example.com names ${OTHER}, did:web:third.example, not ${DID}`)) + return true + }, + ) + }) + + it('reports missing and mismatched together, in the plural, missing first', async () => { + await assert.rejects( + () => + checkAuthorityDns( + { + '_lexicon.gone.example.com': ['com.example.gone.thing'], + '_lexicon.example.com': ['com.example.thing', 'com.example.other'], + }, + DID, + { + resolveTxt: resolverOver({ '_lexicon.example.com': [`did=${OTHER}`] }), + }, + ), + (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /^2 authority domains not pointing at did:web:corpora\.example$/m) + assert.ok(err.details[0].includes('_lexicon.gone.example.com has no `did=` TXT record')) + assert.ok(err.details[1].includes(`_lexicon.example.com names ${OTHER}`)) + // Every NSID that needed the domain is named, so the operator knows what breaks. + assert.ok(err.details[1].includes('(needed by com.example.thing, com.example.other)')) + return true + }, + ) + }) + + it('fails the build outright on a lookup error that is not a missing record', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: async () => { + throw dnsError('queryTxt ESERVFAIL _lexicon.example.com', 'ESERVFAIL') + }, + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.equal( + err.message, + 'DNS lookup for _lexicon.example.com failed: queryTxt ESERVFAIL _lexicon.example.com', + ) + return true + }, + ) + }) + + it('fails the build on an error with no code at all', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: async () => { + throw new Error('socket hang up') + }, + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.equal(err.message, 'DNS lookup for _lexicon.example.com failed: socket hang up') + return true + }, + ) + }) + + it('stringifies a resolver that throws something other than an Error', async () => { + // A custom `resolveTxt` is a public option, so the thrown value is not ours to assume. + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + resolveTxt: async () => { + throw 'resolver exploded' + }, + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.equal(err.message, 'DNS lookup for _lexicon.example.com failed: resolver exploded') + return true + }, + ) + }) + + it('does nothing when there are no domains to check', async () => { + assert.deepEqual(await checkAuthorityDns({}, DID, { resolveTxt: neverResolves }), []) + }) +}) + +describe('checkAuthorityDns, cutover', () => { + it('passes silently when every domain already names this DID', async () => { + const warnings = await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + mode: 'cutover', + resolveTxt: resolverOver({ '_lexicon.example.com': [`did=${DID}`] }), + }) + assert.deepEqual(warnings, []) + }) + + it('still refuses a missing record, in the singular', async () => { + await assert.rejects( + () => + checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + mode: 'cutover', + resolveTxt: resolverOver({}), + }), + (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /^1 authority domain with no `did=` TXT record$/m) + assert.ok(err.details.some((d) => d.includes('--dns-check cutover will not skip this'))) + return true + }, + ) + }) + + it('still refuses missing records, in the plural', async () => { + await assert.rejects( + () => + checkAuthorityDns( + { + '_lexicon.example.com': ['com.example.thing'], + '_lexicon.nested.example.com': ['com.example.nested.widget'], + }, + DID, + { mode: 'cutover', resolveTxt: resolverOver({}) }, + ), + (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /^2 authority domains with no `did=` TXT record$/m) + assert.equal(err.details.filter((d) => d.includes('has no `did=` TXT record')).length, 2) + return true + }, + ) + }) + + it('reports the missing record and not the mismatch when both are present', async () => { + // The missing one is never an expected cutover state, so it is the failure worth surfacing. + await assert.rejects( + () => + checkAuthorityDns( + { + '_lexicon.example.com': ['com.example.thing'], + '_lexicon.nested.example.com': ['com.example.nested.widget'], + }, + DID, + { + mode: 'cutover', + resolveTxt: resolverOver({ '_lexicon.nested.example.com': [`did=${OTHER}`] }), + }, + ), + (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /^1 authority domain with no `did=` TXT record$/m) + assert.ok(!err.message.includes('_lexicon.nested.example.com names')) + return true + }, + ) + }) + + it('downgrades a single mismatch to a warning, phrased in the singular', async () => { + const warnings = await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID, { + mode: 'cutover', + resolveTxt: resolverOver({ '_lexicon.example.com': [`did=${OTHER}`] }), + }) + assert.equal(warnings.length, 2) + assert.equal( + warnings[0], + `DNS not yet flipped: _lexicon.example.com names ${OTHER}, not ${DID} (needed by com.example.thing)`, + ) + assert.ok(warnings[1].includes('the authority domain above still names another DID')) + assert.ok(warnings[1].includes('Until that record says did=did:web:corpora.example')) + assert.ok(warnings[1].includes('Flip it once the new authority verifies')) + assert.ok(!warnings[1].includes('domains')) + }) + + it('downgrades several mismatches to warnings, phrased in the plural and counted', async () => { + const warnings = await checkAuthorityDns( + { + '_lexicon.example.com': ['com.example.thing'], + '_lexicon.nested.example.com': ['com.example.nested.widget'], + }, + DID, + { + mode: 'cutover', + resolveTxt: resolverOver({ + '_lexicon.example.com': [`did=${OTHER}`], + '_lexicon.nested.example.com': [`did=${OTHER}`], + }), + }, + ) + assert.equal(warnings.length, 3) + assert.ok(warnings[0].startsWith('DNS not yet flipped: _lexicon.example.com names')) + assert.ok(warnings[1].startsWith('DNS not yet flipped: _lexicon.nested.example.com names')) + assert.ok(warnings[2].includes('the 2 authority domains above still name another DID')) + assert.ok(warnings[2].includes('Until those records say did=did:web:corpora.example')) + assert.ok(warnings[2].includes('Flip them once the new authority verifies')) + }) + + it('warns only about the domains that have not flipped', async () => { + const warnings = await checkAuthorityDns( + { + '_lexicon.example.com': ['com.example.thing'], + '_lexicon.done.example.com': ['com.example.done.widget'], + }, + DID, + { + mode: 'cutover', + resolveTxt: resolverOver({ + '_lexicon.example.com': [`did=${OTHER}`], + '_lexicon.done.example.com': [`did=${DID}`], + }), + }, + ) + assert.equal(warnings.length, 2) + assert.ok(!warnings.join('\n').includes('_lexicon.done.example.com')) + }) +}) + +describe('checkAuthorityDns, through the real node:dns resolver', () => { + // No `resolveTxt` and no `mode` below: this block is here to prove the defaults — the imported + // `node:dns/promises` resolver and `strict` — are wired up, and that the error codes the missing + // branch keys off of are the codes c-ares actually produces. + /** @type {Awaited>} */ + let dns + /** @type {() => void} */ + let restore + + before(async () => { + dns = await startDnsServer({ records: { '_lexicon.example.com': `did=${DID}` } }) + restore = useLocalDns(dns.port) + }) + after(async () => { + restore?.() + await dns?.close() + }) + + it('resolves the TXT record itself when given no resolveTxt', async () => { + dns.setTxt('_lexicon.example.com', `did=${DID}`) + dns.queries.length = 0 + assert.deepEqual(await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID), []) + assert.ok(dns.queried('_lexicon.example.com')) + }) + + it('joins a real 300-byte record split across character-strings', async () => { + const long = `did:web:${'a'.repeat(300)}.example` + dns.setTxt('_lexicon.example.com', `did=${long}`) + assert.deepEqual(await checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, long), []) + }) + + it('reads NXDOMAIN as a missing record rather than a lookup failure', async () => { + dns.setFailure('_lexicon.example.com', 'nxdomain') + await assert.rejects( + () => checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID), + (err) => { + assert.ok(err instanceof BuildError) + assert.ok(err.details[0].includes('has no `did=` TXT record')) + return true + }, + ) + }) + + it('reads an empty NOERROR answer as a missing record too', async () => { + dns.setFailure('_lexicon.example.com', 'empty') + await assert.rejects( + () => checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID), + (err) => { + assert.ok(err instanceof BuildError) + assert.ok(err.details[0].includes('has no `did=` TXT record')) + return true + }, + ) + }) + + it('fails the build when the server answers SERVFAIL', async () => { + dns.setFailure('_lexicon.example.com', 'servfail') + await assert.rejects( + () => checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID), + (err) => { + assert.ok(err instanceof BuildError) + assert.match(err.message, /^DNS lookup for _lexicon\.example\.com failed: /) + assert.ok(!err.message.includes('has no `did=` TXT record')) + return true + }, + ) + }) + + it('reports a real mismatch from a real answer', async () => { + dns.setTxt('_lexicon.example.com', `did=${OTHER}`) + await assert.rejects( + () => checkAuthorityDns({ '_lexicon.example.com': ['com.example.thing'] }, DID), + (err) => { + assert.ok(err instanceof BuildError) + assert.ok(err.details[0].includes(`names ${OTHER}, not ${DID}`)) + return true + }, + ) + }) +}) diff --git a/test/build/emit.test.js b/test/build/emit.test.js new file mode 100644 index 0000000..1e6c7af --- /dev/null +++ b/test/build/emit.test.js @@ -0,0 +1,104 @@ +/** + * `planDist` is pure, so the paging it decides can be checked without signing anything. The + * namespaces that reach a hundred records are the ones nobody builds by hand, and the cursor is + * the only field in `dist/` a reader has to send back — getting it wrong strands page two. + */ + +import assert from 'node:assert/strict' +import test from 'node:test' +import { planDist } from '../../src/build/emit.js' +import { COLLECTION } from '../../src/build/lexicons.js' +import { TEST_DID } from './helpers.js' + +/** @param {number} count */ +function nsids(count) { + return Array.from({ length: count }, (_, i) => `com.example.n${String(i).padStart(4, '0')}`) +} + +/** + * A signed repo's shape, with none of its cost: `planDist` copies these through and never looks + * inside a CAR. + * + * @param {string[]} ids ascending, as `loadLexicons` returns them + */ +function plan(ids) { + return planDist({ + corporaVersion: '0.0.0-test', + did: TEST_DID, + collection: COLLECTION, + signingKey: 'did:key:zTestSigningKey', + didDoc: { id: TEST_DID }, + repo: { + rev: '3lxaaaaaaaa22', + commitCid: 'bafycommit', + mstRoot: 'bafyroot', + repoCar: new Uint8Array([1, 2, 3]), + records: Object.fromEntries( + ids.map((nsid, i) => { + const uri = `at://${TEST_DID}/${COLLECTION}/${nsid}` + return [nsid, { uri, cid: `bafyrecord${i}`, value: { $type: COLLECTION, id: nsid } }] + }), + ), + proofs: Object.fromEntries(ids.map((nsid) => [nsid, new Uint8Array([0])])), + }, + sources: ids.map((nsid) => ({ nsid, path: `${nsid}.json`, record: { id: nsid }, sourceHash: `sha256-${nsid}` })), + authorityDomains: { '_lexicon.example.com': ids }, + }) +} + +/** + * @param {import('../../src/build/emit.js').DistFile[]} files + * @returns {import('../../src/serve/types.js').ListRecordsPage[]} + */ +function listPages(files) { + return files + .filter((f) => f.path.startsWith('list/')) + .map((f) => JSON.parse(new TextDecoder().decode(f.bytes))) +} + +test('a namespace larger than one page carries a cursor on every page but the last', async () => { + const ids = nsids(250) + const { files } = plan(ids) + const pages = listPages(files) + assert.deepEqual( + files.filter((f) => f.path.startsWith('list/')).map((f) => f.path), + ['list/page-1.json', 'list/page-2.json', 'list/page-3.json'], + ) + assert.deepEqual(pages.map((p) => p.records.length), [100, 100, 50]) + + // The cursor is an rkey, which is what listRecords takes back. A full at:// URI here would be + // accepted by nothing and would strand every reader on page one. + const descending = [...ids].reverse() + assert.equal(pages[0].cursor, descending[99]) + assert.equal(pages[1].cursor, descending[199]) + assert.equal(pages[2].cursor, undefined, 'the last page invites a request for a page that is not there') + + // Every page's cursor names its own last record, and the pages concatenate back to the order. + for (const page of pages.slice(0, -1)) { + const last = page.records[page.records.length - 1] + assert.equal(page.cursor, last.uri.slice(last.uri.lastIndexOf('/') + 1)) + } + assert.deepEqual( + pages.flatMap((p) => p.records.map((r) => r.value.id)), + descending, + ) +}) + +test('exactly one page of records is one page, with nothing to follow', async () => { + // The off-by-one worth guarding: 100 records fill a page exactly, and a cursor here would + // promise a page 2 that `planDist` never wrote. + const full = listPages(plan(nsids(100)).files) + assert.equal(full.length, 1) + assert.equal(full[0].records.length, 100) + assert.equal(full[0].cursor, undefined) + + const over = listPages(plan(nsids(101)).files) + assert.deepEqual(over.map((p) => p.records.length), [100, 1]) + assert.equal(over[0].cursor, 'com.example.n0001', 'the 100th record in descending order') + assert.equal(over[1].cursor, undefined) +}) + +test('an empty namespace still emits a page, rather than no list at all', async () => { + const pages = listPages(plan([]).files) + assert.deepEqual(pages, [{ records: [] }]) +}) diff --git a/test/build/entrypoint.test.js b/test/build/entrypoint.test.js new file mode 100644 index 0000000..2a132ad --- /dev/null +++ b/test/build/entrypoint.test.js @@ -0,0 +1,104 @@ +/** + * `bin/corpora.js` — the argv dispatcher, exercised the only way it can be: as a process. + * + * Everything here is about what a shell sees, so the assertions are exit codes and which stream the + * text landed on. A user who typed a command wrong gets stderr and a non-zero code; a user who asked + * for help gets stdout. + */ + +import assert from 'node:assert/strict' +import { readFile, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import test from 'node:test' +import { REPO_ROOT, runCli } from '../helpers/build.js' +import { tempDir } from './helpers.js' + +/** Never inherit a debug flag from whoever ran the suite: it changes what stderr contains. */ +const NO_DEBUG = { CORPORA_DEBUG: '' } + +test('corpora with no command prints usage and fails, so a bare invocation is not a success', async () => { + const { code, stdout, stderr } = await runCli([], { env: NO_DEBUG }) + assert.equal(code, 1) + assert.match(stdout, /^corpora — a serverless AT Protocol lexicon authority/) + assert.match(stdout, /corpora build --lexicons /) + assert.equal(stderr, '') +}) + +test('every spelling of asking for help exits 0 with the same usage', async () => { + for (const argv of [['--help'], ['-h'], ['help']]) { + const { code, stdout, stderr } = await runCli(argv, { env: NO_DEBUG }) + assert.equal(code, 0, `corpora ${argv[0]}`) + assert.match(stdout, /^corpora — a serverless AT Protocol lexicon authority/) + assert.equal(stderr, '') + } +}) + +test('--version prints the package version and nothing else', async () => { + const { version } = JSON.parse(await readFile(new URL('package.json', `file://${REPO_ROOT}`), 'utf8')) + for (const flag of ['--version', '-v']) { + const { code, stdout, stderr } = await runCli([flag], { env: NO_DEBUG }) + assert.equal(code, 0, `corpora ${flag}`) + assert.equal(stdout, `${version}\n`) + assert.equal(stderr, '') + } +}) + +test('an unknown command names itself on stderr and still shows what is on offer', async () => { + const { code, stdout, stderr } = await runCli(['bulid'], { env: NO_DEBUG }) + assert.equal(code, 1) + assert.equal(stdout, '') + assert.match(stderr, /^corpora: unknown command 'bulid'\n/) + assert.match(stderr, /corpora keygen \[--out \]/) +}) + +test('a known command is dispatched and its own exit code is the process exit code', async () => { + const { code, stdout } = await runCli(['build', '--help'], { env: NO_DEBUG }) + assert.equal(code, 0) + assert.match(stdout, /^corpora build — sign a lexicon directory into dist\//) +}) + +test('every command in the usage text actually resolves to a module', async () => { + // The table of subcommands is four lazy imports, and a typo in any path would only show up the + // first time someone ran that command. Ask each one for its own help: the module has to load and + // answer for the dispatcher to exit 0. + for (const [cmd, banner] of [ + ['build', /^corpora build — /], + ['verify', /^corpora verify — /], + ['serve', /^corpora serve — /], + ['keygen', /^corpora keygen — /], + ]) { + const { code, stdout, stderr } = await runCli([/** @type {string} */ (cmd), '--help'], { env: NO_DEBUG }) + assert.equal(code, 0, `corpora ${cmd} --help exited ${code}: ${stderr}`) + assert.match(stdout, /** @type {RegExp} */ (banner)) + assert.equal(stderr, '') + } +}) + +test('a subcommand that throws something that is not an Error still gets one readable line', async (t) => { + // `--dist` points at a directory whose data.js throws a bare string, which is what a hand-edited + // or half-written bundle can do. The dispatcher has to render it without assuming `.message`. + const dist = await tempDir(t) + await writeFile(join(dist, 'data.js'), "throw 'data.js is not a module'\n") + + const { code, stdout, stderr } = await runCli(['serve', '--dist', dist], { env: NO_DEBUG }) + assert.equal(code, 1) + assert.equal(stdout, '') + assert.equal(stderr, 'corpora serve: data.js is not a module\n') +}) + +test('a subcommand that throws becomes a one-line message, not a stack trace', async () => { + // `--bogus` fails in parseArgs, before the subcommand's own try/catch, so this is the dispatcher's + // last-resort handler rather than a BuildError the build CLI reported itself. + const { code, stdout, stderr } = await runCli(['build', '--bogus'], { env: NO_DEBUG }) + assert.equal(code, 1) + assert.equal(stdout, '') + assert.match(stderr, /^corpora build: .*--bogus/) + assert.equal(stderr.trim().split('\n').length, 1, 'a stack trace leaked without CORPORA_DEBUG') +}) + +test('CORPORA_DEBUG adds the stack the plain message deliberately withholds', async () => { + const { code, stderr } = await runCli(['build', '--bogus'], { env: { CORPORA_DEBUG: '1' } }) + assert.equal(code, 1) + assert.match(stderr, /^corpora build: .*--bogus/) + assert.match(stderr, /\n\s+at /) +}) diff --git a/test/build/keygen.test.js b/test/build/keygen.test.js new file mode 100644 index 0000000..8bbcbdb --- /dev/null +++ b/test/build/keygen.test.js @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict' +import { readFile, stat } from 'node:fs/promises' +import { join } from 'node:path' +import test from 'node:test' +import { Secp256k1Keypair } from '@atproto/crypto' +import { run as keygen } from '../../src/build/keygen.js' +import { rejects, tempDir } from './helpers.js' + +/** + * Runs keygen with stdout captured, so the one secret in the system never reaches the reporter. + * + * @param {import('node:test').TestContext} t + * @param {string[]} argv + */ +async function capture(t, argv) { + let out = '' + t.mock.method(process.stdout, 'write', (/** @type {string} */ chunk) => void (out += chunk)) + try { + return { code: await keygen(argv), out } + } finally { + t.mock.restoreAll() + } +} + +test('keygen --help prints usage and generates nothing', async (t) => { + // A key is worth something only if nobody made it by accident: --help must not mint one. + const created = t.mock.method(Secp256k1Keypair, 'create') + const { code, out } = await capture(t, ['--help']) + + assert.equal(code, 0) + assert.match(out, /^corpora keygen — generate a secp256k1 signing key\n/) + assert.match(out, /--out \s+write the private key to with mode 0600/) + assert.equal(created.mock.callCount(), 0, '--help generated a key') +}) + +test('keygen prints a key it has just proved importable', async (t) => { + const { code, out } = await capture(t, []) + assert.equal(code, 0) + + const hex = /** @type {RegExpMatchArray} */ (out.match(/^[0-9a-f]{64}$/m))[0] + const keypair = await Secp256k1Keypair.import(hex) + assert.match(out, new RegExp(`public key ${keypair.did()}`)) + assert.match(out, /^multikey z/m) + assert.match(out, /CORPORA_SIGNING_KEY/) + assert.match(out, /Never deploy it/) +}) + +test('keygen --out writes 0600 and keeps the key off stdout', async (t) => { + const file = join(await tempDir(t), 'signing.key') + const { code, out } = await capture(t, ['--out', file]) + + assert.equal(code, 0) + const hex = (await readFile(file, 'utf8')).trim() + assert.match(hex, /^[0-9a-f]{64}$/) + assert.equal((await stat(file)).mode & 0o777, 0o600) + assert.equal(out.includes(hex), false, 'keygen echoed the key it had just written to a file') + assert.match(out, new RegExp(`private key written to ${file} \\(mode 0600\\)`)) +}) + +test('keygen refuses to overwrite an existing key file', async (t) => { + const file = join(await tempDir(t), 'signing.key') + await capture(t, ['--out', file]) + const before = await readFile(file, 'utf8') + + const err = await rejects(() => capture(t, ['--out', file])) + assert.equal(err.name, 'BuildError') + assert.match(err.message, /already exists; refusing to overwrite a signing key/) + assert.equal(await readFile(file, 'utf8'), before, 'the existing key was disturbed') +}) + +test('keygen reports any other write failure by path', async (t) => { + const file = join(await tempDir(t), 'no-such-dir', 'signing.key') + const err = await rejects(() => capture(t, ['--out', file])) + assert.equal(err.name, 'BuildError') + assert.match(err.message, new RegExp(`^cannot write ${file}: .*ENOENT`)) +}) + +test('keygen refuses to print a key that does not survive a round trip', async (t) => { + // The whole point of the format is that `corpora build` can import it again; if the import + // came back as some other identity, printing the key would hand the operator a dud. + t.mock.method(Secp256k1Keypair, 'import', async () => ({ did: () => 'did:key:zSomethingElse' })) + const err = await rejects(() => capture(t, [])) + + assert.equal(err.name, 'BuildError') + assert.match(err.message, /did not survive a round trip; refusing to print it/) +}) + +test('a round-trip failure with --out leaves no key file behind', async (t) => { + const file = join(await tempDir(t), 'signing.key') + t.mock.method(Secp256k1Keypair, 'import', async () => ({ did: () => 'did:key:zSomethingElse' })) + await rejects(() => capture(t, ['--out', file])) + + await assert.rejects(() => stat(file), /ENOENT/, 'a key that failed its own check was written anyway') +}) diff --git a/test/build/lexicons.test.js b/test/build/lexicons.test.js index 68221a2..ae0d57e 100644 --- a/test/build/lexicons.test.js +++ b/test/build/lexicons.test.js @@ -1,7 +1,8 @@ import assert from 'node:assert/strict' +import { basename, dirname, join } from 'node:path' import test from 'node:test' import { COLLECTION, loadLexicons } from '../../src/build/lexicons.js' -import { FIXTURES, doc, lexiconDir, rejects } from './helpers.js' +import { FIXTURES, doc, lexiconDir, rejects, tempDir } from './helpers.js' test('reads the fixture directory, keyed by id and sorted', async () => { const { sources, warnings } = await loadLexicons(FIXTURES) @@ -110,6 +111,154 @@ test('rejects invalid JSON, a missing lexicon version, and a bad id', async (t) assert.match(err.message, /is not a valid NSID/) }) +test('rejects valid JSON that is not a document at all', async (t) => { + // A file that parses is not a file that means anything: a top-level array is what a hand-rolled + // export of several lexicons looks like, and reading `.lexicon` off it would quietly say + // `undefined` rather than name the file. + const dir = await lexiconDir(t, { + 'list.json': '[{ "lexicon": 1, "id": "com.example.thing" }]', + 'nothing.json': 'null', + 'number.json': '7', + }) + const err = await rejects(() => loadLexicons(dir)) + assert.match(err.message, /3 problems/) + for (const path of ['list.json', 'nothing.json', 'number.json']) { + assert.match(err.message, new RegExp(`${path}: is not a JSON object`)) + } +}) + +test('rejects a document whose id is missing or is not a string', async (t) => { + // The id is the rkey the document is published under, so there is nothing to publish without + // one; a number that looks like an NSID is not one either. + const dir = await lexiconDir(t, { + 'com.example.thing.json': { lexicon: 1, defs: {} }, + 'com.example.two.json': { lexicon: 1, id: 42, defs: {} }, + }) + const err = await rejects(() => loadLexicons(dir)) + assert.match(err.message, /2 problems/) + assert.match(err.message, /com\.example\.thing\.json: missing `id`, which is the NSID/) + assert.match(err.message, /com\.example\.two\.json: missing `id`/) +}) + +test('rejects a $type naming anything but the lexicon collection', async (t) => { + // `$type` is optional in the source and injected on the way out. A document that spells it + // itself, wrongly, would be signed into the repo under a collection nothing resolves through — + // and `lexiconDocumentSchema` has no opinion about the field at all. + const dir = await lexiconDir(t, { + 'com.example.thing.json': { ...doc('com.example.thing'), $type: 'app.bsky.feed.post' }, + }) + const err = await rejects(() => loadLexicons(dir)) + assert.match(err.message, /com\.example\.thing\.json: has `\$type: "app\.bsky\.feed\.post"`/) + assert.match(err.message, new RegExp(`lexicon records are '${COLLECTION}'`)) + + // Spelling it correctly is redundant rather than wrong, and survives to the emitted record. + const ok = await lexiconDir(t, { + 'com.example.thing.json': { ...doc('com.example.thing'), $type: COLLECTION }, + }) + const { sources } = await loadLexicons(ok) + assert.equal(sources[0].record.$type, COLLECTION) +}) + +test('rejects an integer too large to survive a JSON round trip', async (t) => { + // DRISL will encode this one, and JSON.parse on the other side will hand back a different + // number. Written as source text because the literal cannot be typed exactly in JS either. + const dir = await lexiconDir(t, { + 'com.example.thing.json': JSON.stringify({ + lexicon: 1, + id: 'com.example.thing', + defs: { + main: { + type: 'record', + key: 'tid', + record: { type: 'object', properties: { count: { type: 'integer', maximum: 9007199254740994 } } }, + }, + }, + }), + }) + const err = await rejects(() => loadLexicons(dir)) + assert.match(err.message, /integer 9007199254740994 at defs\.main\.record\.properties\.count\.maximum/) + assert.match(err.message, /outside the safe integer range/) +}) + +test('one directory spelled every way a shell spells it loads identically', async (t) => { + // `--lexicons ./lexicons` is the form people type by reflex and the form tab completion + // produces; `lexicons/`, `a//b`, and a path with a `..` in it all arrive the same way. The + // loader resolves the directory once and reports paths relative to it, so a source path — what + // every later message and the emitted manifest names a file by — cannot depend on the spelling. + const dir = await lexiconDir(t, { + 'com.example.thing.json': doc('com.example.thing'), + 'nested/com.example.other.json': doc('com.example.other'), + }) + const expected = ['nested/com.example.other.json', 'com.example.thing.json'] + const name = basename(dir) + + const cwd = process.cwd() + process.chdir(dirname(dir)) + try { + const spellings = [name, `./${name}`, `${name}/.`, `${name}//nested/..`, `${name}/../${name}`, `${name}/`, dir] + for (const spelling of spellings) { + const { sources, warnings } = await loadLexicons(spelling) + assert.deepEqual( + sources.map((s) => s.path), + expected, + `--lexicons ${spelling}`, + ) + assert.deepEqual(warnings, [], `--lexicons ${spelling}`) + } + } finally { + process.chdir(cwd) + } +}) + +test('a lexicon directory named relative to itself still reports paths relative to itself', async (t) => { + // `corpora build --lexicons .` from inside the directory: the resolved root is the cwd, and + // nothing of it survives into the paths reported back. + const dir = await lexiconDir(t, { + 'com.example.thing.json': doc('com.example.thing'), + 'nested/com.example.other.json': doc('com.example.other'), + }) + const cwd = process.cwd() + process.chdir(dir) + try { + const { sources } = await loadLexicons('.') + assert.deepEqual( + sources.map((s) => s.path), + ['nested/com.example.other.json', 'com.example.thing.json'], + ) + } finally { + process.chdir(cwd) + } +}) + +test('a defs section of any shape is reported rather than crashed on', async (t) => { + // The permission walk reads `defs` before the official parser has vouched for it, so it meets + // whatever a hand-written file put there. Each of these is a TypeError inside the walk if the + // guards go — and a TypeError names no file, which is the whole reason this module exists. + const dir = await lexiconDir(t, { + 'com.example.one.json': { lexicon: 1, id: 'com.example.one', defs: null }, + 'com.example.two.json': { lexicon: 1, id: 'com.example.two', defs: 7 }, + 'com.example.three.json': { lexicon: 1, id: 'com.example.three', defs: { main: null, other: 7 } }, + 'com.example.four.json': { + lexicon: 1, + id: 'com.example.four', + defs: { main: { type: 'permission-set', title: 'x', permissions: [null, 'nope', 7] } }, + }, + }) + const err = await rejects(() => loadLexicons(dir)) + assert.equal(err.name, 'BuildError') + assert.match(err.message, /4 problems/) + for (const name of ['one', 'two', 'three', 'four']) { + assert.match(err.message, new RegExp(`com\\.example\\.${name}\\.json: `)) + } +}) + +test('a lexicon directory that cannot be read says so, naming it', async (t) => { + const missing = join(await tempDir(t), 'lexicons') + const err = await rejects(() => loadLexicons(missing)) + assert.match(err.message, new RegExp(`cannot read lexicon directory ${missing}`)) + assert.match(err.message, /ENOENT/) +}) + test('rejects a document the official parser rejects, naming the file', async (t) => { const dir = await lexiconDir(t, { 'com.example.thing.json': { lexicon: 1, id: 'com.example.thing', defs: { main: { type: 'nonsense' } } }, diff --git a/test/build/nsid.test.js b/test/build/nsid.test.js index 1b4ad87..aef6e6c 100644 --- a/test/build/nsid.test.js +++ b/test/build/nsid.test.js @@ -35,6 +35,43 @@ test('NSID validation explains itself', () => { assert.equal(isValidNsid('com.example.thing'), true) }) +test('an NSID longer than the DNS name it implies is refused on length alone', () => { + // 253 for the authority, a dot, 63 for the name: nothing longer can be a DNS name at all, so + // the check fires before any of the per-segment rules get a chance to explain something else. + const longest = ['com', ...Array.from({ length: 4 }, () => 'a'.repeat(63)), 'a'.repeat(51), 'thing'].join('.') + assert.equal(longest.length, 317) + assert.equal(nsidError(longest), null) + assert.match(/** @type {string} */ (nsidError(`${longest}x`)), /longer than 317 characters/) +}) + +test('a segment longer than a DNS label is refused, quoting the segment', () => { + // A label is 63 octets, and an NSID authority is spelled in labels. The message names the + // offender because a 300-character id has no other way of saying which part of it is wrong. + const overlong = 'a'.repeat(64) + const err = /** @type {string} */ (nsidError(`com.${overlong}.thing`)) + assert.match(err, /has a segment longer than 63 characters/) + assert.ok(err.includes(`('${overlong}')`), 'the message did not quote the segment') + assert.equal(nsidError(`com.${'a'.repeat(63)}.thing`), null) +}) + +test('domains come out sorted even when their NSIDs sorted the other way', () => { + // Reversing an authority moves the deepest segment to the front, so NSID order and domain order + // disagree: `io.atcr.hold.stats` sorts first and `_lexicon.hold.atcr.io` sorts second. + const domains = authorityDomains(['io.atcr.hold.stats', 'io.atcr.zebra']) + assert.deepEqual(Object.keys(domains), ['_lexicon.atcr.io', '_lexicon.hold.atcr.io']) + assert.deepEqual(domains['_lexicon.atcr.io'], ['io.atcr.zebra']) + assert.deepEqual(domains['_lexicon.hold.atcr.io'], ['io.atcr.hold.stats']) +}) + +test('a hyphen inside an authority segment is fine; one on either end is not', () => { + // Hyphens are legal in a domain label, which is what the authority segments are — but a label + // may not begin or end with one, and that is the rule a hand-written NSID trips over. + assert.equal(nsidError('com.my-example.thing'), null) + assert.match(/** @type {string} */ (nsidError('com.-example.thing')), /starting or ending with a hyphen/) + assert.match(/** @type {string} */ (nsidError('com.example-.thing')), /\('example-'\)/) + assert.equal(isValidNsid('com.-example.thing'), false) +}) + test('a file path agrees with an id, contradicts it, or says nothing', () => { assert.equal(pathVsId('com.example.thing.json', 'com.example.thing'), 'agrees') assert.equal(pathVsId('com/example/thing.json', 'com.example.thing'), 'agrees') diff --git a/test/build/repo.test.js b/test/build/repo.test.js new file mode 100644 index 0000000..b5c0a7d --- /dev/null +++ b/test/build/repo.test.js @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { Secp256k1Keypair } from '@atproto/crypto' +import { verifyRepoCar } from '@atproto/repo' +import { parseRev, revFromRoot, signRepo } from '../../src/build/repo.js' +import { TEST_DID, TEST_KEY, doc } from './helpers.js' + +/** + * The rev `signRepo` builds its throwaway probe under: `formatTid(REV_ANCHOR_MICROS, 0)`, the + * anchor instant with a zero clock id. Spelled out rather than imported because the constant is + * private, and a change to the anchor should fail this file loudly rather than silently stop + * exercising the shortcut. + */ +const PROBE_REV = '3gv352zok2222' + +/** + * Lexicon sources as `loadLexicons` would hand them over, without going near a filesystem. + * + * @param {...string} nsids + * @returns {import('../../src/build/lexicons.js').LexiconSource[]} + */ +function sources(...nsids) { + return nsids.map((nsid) => ({ + nsid, + path: `${nsid}.json`, + record: { $type: 'com.atproto.lexicon.schema', ...doc(nsid) }, + sourceHash: `sha256-${'0'.repeat(64)}`, + })) +} + +test('a rev that lands on the probe rev reuses the probe rather than signing a second time', async () => { + // `signRepo` builds the tree twice — once to learn the MST root, once under the rev derived from + // it — and skips the second build when the two revs coincide. An operator reaches that shortcut + // by passing the probe rev explicitly, and what comes back has to be a complete repo, not the + // half-built scaffolding the probe was meant to be. + const keypair = await Secp256k1Keypair.import(TEST_KEY) + const src = sources('com.example.thing', 'com.example.other') + + const derived = await signRepo({ did: TEST_DID, keypair, sources: src }) + const probed = await signRepo({ did: TEST_DID, keypair, sources: src, rev: PROBE_REV }) + + assert.equal(probed.rev, PROBE_REV) + assert.equal(parseRev(PROBE_REV), PROBE_REV, 'the probe rev is not a TID a user could pass to --rev') + assert.notEqual(derived.rev, PROBE_REV, 'these lexicons derive the probe rev; the shortcut is no longer the odd path') + + // The rev lives in the commit and nowhere else, so reusing the probe must move the commit and + // leave the tree of records exactly where it was. + assert.equal(probed.mstRoot, derived.mstRoot) + assert.notEqual(probed.commitCid, derived.commitCid) + assert.deepEqual(Object.keys(probed.records), Object.keys(derived.records)) + assert.deepEqual(probed.records['com.example.thing'], derived.records['com.example.thing']) + + const verified = await verifyRepoCar(probed.repoCar, TEST_DID, keypair.did()) + assert.equal(verified.commit.rev, PROBE_REV) + assert.deepEqual( + verified.creates.map((c) => c.rkey).sort(), + ['com.example.other', 'com.example.thing'], + ) + for (const nsid of ['com.example.other', 'com.example.thing']) { + assert.ok(probed.proofs[nsid].byteLength > 0, `${nsid} came back without a proof`) + } +}) + +test('the rev is a pure function of the MST root', async () => { + // The one thing standing between a rebuild and byte-identical output: same root in, same TID + // out, and no clock anywhere in it. + const root = 'bafyreidfuzeuwtqhdxbrbnbrjpvvtcqxgbbdvbmtqcaqcmb4hbjqmw3ora' + assert.equal(revFromRoot(root), revFromRoot(root)) + assert.match(revFromRoot(root), /^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$/) + assert.notEqual(revFromRoot(root), revFromRoot(`${root}x`)) +}) diff --git a/test/helpers/server.js b/test/helpers/server.js index 4a87cc3..87cd0ee 100644 --- a/test/helpers/server.js +++ b/test/helpers/server.js @@ -175,7 +175,7 @@ function fixtureHandler(data) { return json(record) } case '/xrpc/com.atproto.repo.listRecords': - return json(data.list[0]) + return json({ records: data.order.map((rkey) => data.records[rkey]) }) case '/xrpc/com.atproto.repo.describeRepo': return json(data.describeRepo) case '/xrpc/com.atproto.sync.getLatestCommit': diff --git a/test/message.test.js b/test/message.test.js new file mode 100644 index 0000000..b348a43 --- /dev/null +++ b/test/message.test.js @@ -0,0 +1,13 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { message } from '../src/message.js' + +test('an Error renders as its message', () => { + assert.equal(message(new Error('x')), 'x') +}) + +test('anything else renders as String(), because anything can be thrown', () => { + // `test/build/dns.test.js` injects a resolver that throws this exact string, and + // `test/build/entrypoint.test.js` a `dist/data.js` that throws one: the arm is live. + assert.equal(message('boom'), 'boom') +}) diff --git a/test/serve/cli.test.js b/test/serve/cli.test.js index 571eb12..679ba6f 100644 --- a/test/serve/cli.test.js +++ b/test/serve/cli.test.js @@ -6,10 +6,12 @@ import assert from 'node:assert/strict' import { spawn } from 'node:child_process' import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' import { tmpdir } from 'node:os' import { join } from 'node:path' import { after, before, describe, it } from 'node:test' import { fileURLToPath } from 'node:url' +import { run as serveCli } from '../../src/node/cli.js' const BIN = fileURLToPath(new URL('../../bin/corpora.js', import.meta.url)) const FIXTURE = new URL('./fixture.js', import.meta.url).href @@ -89,6 +91,193 @@ describe('corpora serve', () => { }) }) +describe('corpora serve arguments', () => { + it('prints usage for --help and starts nothing', async (t) => { + const { code, out, err } = await capture(t, ['--help']) + assert.equal(code, 0) + assert.equal(err, '') + assert.match(out, /^corpora serve — serve a built dist\/ over HTTP/) + assert.match(out, /--dist /) + assert.match(out, /--port {5}port to bind, 0 for an ephemeral one \(default 2583\)/) + }) + + it('refuses to start without --dist, and says what to type instead', async (t) => { + const { code, out, err } = await capture(t, []) + assert.equal(code, 1) + assert.equal(out, '') + assert.match(err, /^corpora serve: --dist is required\n\n/) + assert.match(err, /corpora serve --dist /, 'the refusal carries the usage') + }) + + it('refuses a --port that is not a port, without binding anything', async (t) => { + const dist = makeDist(t) + for (const port of ['abc', '-1', '65536', '2583.5']) { + const { code, err } = await capture(t, ['--dist', dist, `--port=${port}`]) + assert.equal(code, 1, `--port=${port} was accepted`) + assert.equal(err, `corpora serve: --port must be 0-65535, got '${port}'\n`) + } + // The edges are ports, not errors: these get past the check and fail later, on the dist. + const empty = mkdtempSync(join(tmpdir(), 'corpora-empty-')) + t.after(() => rmSync(empty, { recursive: true, force: true })) + for (const port of ['0', '65535']) { + const { err } = await capture(t, ['--dist', empty, `--port=${port}`]) + assert.match(err, /no data\.js/, `--port=${port} was rejected`) + } + }) + + it('names the missing data.js and the build that would produce it', async (t) => { + const dist = mkdtempSync(join(tmpdir(), 'corpora-empty-')) + t.after(() => rmSync(dist, { recursive: true, force: true })) + const { code, out, err } = await capture(t, ['--dist', dist]) + assert.equal(code, 1) + assert.equal(out, '') + assert.match(err, new RegExp(`^corpora serve: no data\\.js in ${dist}\n`)) + assert.match(err, new RegExp(`run \`corpora build --lexicons --out ${dist} --did \` first`)) + }) + + it('resolves a relative --dist before reporting it missing', async (t) => { + // The path in the error is resolved, so it names the directory that was actually looked in + // rather than echoing back what the user typed from a different working directory. + const { code, err } = await capture(t, ['--dist', 'no/such/dist']) + assert.equal(code, 1) + assert.match(err, new RegExp(`no data\\.js in ${join(process.cwd(), 'no/such/dist')}\n`)) + }) +}) + +describe('corpora serve --quiet', () => { + it('prints nothing, serves anyway, and stops on SIGTERM', async (t) => { + const dist = makeDist(t) + const port = await freePort() + const { child, out } = startServe(t, ['--dist', dist, '--port', String(port), '--quiet']) + + const url = `http://127.0.0.1:${port}` + await waitForServer(url, out) + assert.equal(out(), '', 'a --quiet server announced itself') + + assert.equal(child.exitCode, null, 'serve returned while it was meant to be serving') + const exit = new Promise((resolve) => child.once('exit', resolve)) + child.kill('SIGTERM') + assert.equal(await exit, 0, 'SIGTERM must stop it as cleanly as Ctrl-C') + assert.equal(out(), '', '--quiet leaked the shutdown notice') + }) +}) + +describe('the serve banner', () => { + it('counts one lexicon in the singular', async (t) => { + const dist = makeDist( + t, + `data.order = ['com.example.thing']\n` + + `data.records = { 'com.example.thing': data.records['com.example.thing'] }`, + ) + const { out, stop } = startServe(t, ['--dist', dist, '--port', '0']) + await waitFor(() => /Ctrl-C to stop/.test(out()), out) + assert.match(out(), /1 lexicon at rev 3lxyzrevrevrev/) + assert.equal(/\d lexicons/.test(out()), false) + await stop() + }) + + it('still prints a copy-pasteable getRecord when nothing is published', async (t) => { + // An empty dist/ is a real state — a first build with no lexicons — and the banner has to + // put *some* rkey in the commands rather than printing `rkey=undefined`. + const dist = makeDist(t, 'data.order = []\ndata.records = {}') + const { out, stop } = startServe(t, ['--dist', dist, '--port', '0']) + await waitFor(() => /Ctrl-C to stop/.test(out()), out) + assert.match(out(), /0 lexicons at rev/) + + const recordUrl = /curl -s '([^']+repo\.getRecord[^']*)'/.exec(out())?.[1] + assert.ok(recordUrl) + assert.equal(new URL(recordUrl).searchParams.get('rkey'), 'com.example.thing') + + // It is a placeholder, so the server answers it the way it answers any absent record. + const res = await fetch(recordUrl) + assert.equal(res.status, 400) + assert.equal((await res.json()).error, 'RecordNotFound') + await stop() + }) +}) + +/** + * Runs `corpora serve` in this process with stdout and stderr captured. Only usable for argv + * that returns before the server starts: `run` does not resolve while it is serving. + * + * @param {import('node:test').TestContext} t + * @param {string[]} argv + */ +async function capture(t, argv) { + let out = '' + let err = '' + t.mock.method(process.stdout, 'write', (/** @type {string} */ chunk) => void (out += chunk)) + t.mock.method(process.stderr, 'write', (/** @type {string} */ chunk) => void (err += chunk)) + const code = await serveCli(argv) + t.mock.restoreAll() + return { code, out, err } +} + +/** + * A dist/ directory holding the synthetic fixture data. + * + * @param {import('node:test').TestContext} t + * @param {string} [transform] statements run against a `data` binding before it is exported + * @returns {string} + */ +function makeDist(t, transform = '') { + const dir = mkdtempSync(join(tmpdir(), 'corpora-serve-')) + t.after(() => rmSync(dir, { recursive: true, force: true })) + writeFileSync( + join(dir, 'data.js'), + `import { makeData } from ${JSON.stringify(FIXTURE)}\n` + + `const data = makeData()\n${transform}\nexport default data\n`, + ) + return dir +} + +/** + * @param {import('node:test').TestContext} t + * @param {string[]} args + */ +function startServe(t, args) { + const child = spawn(process.execPath, [BIN, 'serve', ...args], { stdio: ['ignore', 'pipe', 'pipe'] }) + let output = '' + child.stdout?.on('data', (chunk) => (output += chunk)) + child.stderr?.on('data', (chunk) => (output += chunk)) + t.after(() => void (child.exitCode === null && child.kill('SIGKILL'))) + + /** Ctrl-C, and wait for it: a killed server never runs its shutdown path. */ + const stop = async () => { + const exit = new Promise((resolve) => child.once('exit', resolve)) + child.kill('SIGINT') + assert.equal(await exit, 0) + } + return { child, out: () => output, stop } +} + +/** A port nothing is using, released before it is handed back. @returns {Promise} */ +function freePort() { + return new Promise((resolve, reject) => { + const probe = createServer() + probe.once('error', reject) + probe.listen(0, '127.0.0.1', () => { + const address = probe.address() + if (address === null || typeof address === 'string') return reject(new Error('no port')) + probe.close(() => resolve(address.port)) + }) + }) +} + +/** + * @param {string} url + * @param {() => string} describe + * @returns {Promise} + */ +async function waitForServer(url, describe) { + for (let i = 0; i < 100; i++) { + const ok = await fetch(`${url}/.well-known/did.json`).then((r) => r.ok, () => false) + if (ok) return + await new Promise((r) => setTimeout(r, 50)) + } + throw new Error(`timed out waiting for the server to answer:\n${describe()}`) +} + /** * @param {() => boolean} predicate * @param {() => string} describe diff --git a/test/serve/fixture.js b/test/serve/fixture.js index b9e6225..c086a5f 100644 --- a/test/serve/fixture.js +++ b/test/serve/fixture.js @@ -39,14 +39,13 @@ function record(nsid, did) { } /** - * @param {{ pageSize?: number, did?: string }} [opts] + * @param {{ did?: string }} [opts] * `did` exists so a test can use a port-bearing `did:web`, whose `%3A` makes it the * only shape that catches double-encoding in a query string. * @returns {import('../../src/serve/types.js').CorporaData} */ export function makeData(opts = {}) { const did = opts.did ?? DID - const pageSize = opts.pageSize ?? NSIDS.length /** @type {Record} */ const records = {} /** @type {Record} */ @@ -56,13 +55,6 @@ export function makeData(opts = {}) { proofs[nsid] = fakeCar(nsid) } - /** @type {import('../../src/serve/types.js').ListRecordsPage[]} */ - const list = [] - for (let i = 0; i < NSIDS.length; i += pageSize) { - const page = NSIDS.slice(i, i + pageSize) - list.push({ records: page.map((n) => records[n]), cursor: page[page.length - 1] }) - } - return { did, rev: '3lxyzrevrevrev', @@ -90,7 +82,7 @@ export function makeData(opts = {}) { }, commit: { cid: 'bafyreicommitcommit', rev: '3lxyzrevrevrev' }, describeRepo: { did, collections: [COLLECTION], handleIsCorrect: false }, - list, + order: [...NSIDS], records, proofs, repoCar: fakeCar('full-repo'), diff --git a/test/serve/handler.test.js b/test/serve/handler.test.js index dccb2df..3646520 100644 --- a/test/serve/handler.test.js +++ b/test/serve/handler.test.js @@ -208,10 +208,22 @@ describe('repo.listRecords', () => { assert.deepEqual(rkeys(next), [...NSIDS].reverse().slice(2)) }) - it('flattens prebuilt pages rather than serving them verbatim', async () => { - const paged = createHandler(makeData({ pageSize: 2 })) - const res = paged(get(xrpc('com.atproto.repo.listRecords', { repo: DID, collection: COLLECTION }))) - assert.deepEqual(rkeys(await body(res)), NSIDS) + it('pages on limit alone, from one copy of each record', async () => { + // The bundle carries `order` (rkeys) and `records`, so there is no prebuilt page shape the + // handler could accidentally serve verbatim. Paging is a function of `limit` and nothing else. + const data = makeData() + const handle = createHandler(data) + const call = (/** @type {string} */ limit) => + handle(get(xrpc('com.atproto.repo.listRecords', { repo: DID, collection: COLLECTION, limit }))) + for (const limit of ['1', '2', '3', '100']) { + assert.deepEqual(rkeys(await body(call(limit))), NSIDS.slice(0, Number(limit)), `limit=${limit}`) + } + // listRecords and getRecord read the same `records` map, so they cannot disagree. + const listed = await body(call('100')) + for (const record of listed.records) { + const rkey = record.uri.slice(record.uri.lastIndexOf('/') + 1) + assert.deepEqual(record, data.records[rkey], `${rkey} disagrees with getRecord`) + } }) it('returns an empty page for an unknown collection, as a reference PDS does', async () => { @@ -364,6 +376,29 @@ describe('method and path handling', () => { assert.equal((await head.arrayBuffer()).byteLength, 0) }) + it('answers HEAD on an error with the error headers and no body', async () => { + // Errors take a different construction path from successes, and a HEAD that leaked a + // body — or lost the content-length describing it — would only show up here. + for (const path of [ + '/robots.txt', + '/xrpc/com.atproto.sync.subscribeRepos', + xrpc('com.atproto.sync.getRecord', { did: DID, collection: COLLECTION }), + xrpc('com.atproto.sync.getRecord', { did: DID, collection: COLLECTION, rkey: 'com.example.nope' }), + xrpc('com.atproto.repo.listRecords', { repo: DID, collection: COLLECTION, limit: 'lots' }), + ]) { + const head = handle(req(path, 'HEAD')) + const full = handle(get(path)) + assert.ok(head.status >= 400, path) + assert.equal(head.status, full.status, path) + assert.equal(head.headers.get('content-type'), JSON_TYPE, path) + assert.equal(head.headers.get('cache-control'), 'no-store', path) + const length = Number(head.headers.get('content-length')) + assert.equal(length, (await full.arrayBuffer()).byteLength, path) + assert.ok(length > 0, path) + assert.equal((await head.arrayBuffer()).byteLength, 0, path) + } + }) + it('sets content-length to the byte length, not the character count', async () => { // A lexicon description with non-ASCII characters would otherwise truncate. const wide = makeData() diff --git a/test/serve/landing.test.js b/test/serve/landing.test.js index 5a55222..5aa7e56 100644 --- a/test/serve/landing.test.js +++ b/test/serve/landing.test.js @@ -66,7 +66,7 @@ describe('the landing page', () => { // The page reads the same prebuilt listRecords pages the XRPC endpoint serves, so a // namespace that gains or loses a lexicon needs no change here. const one = makeData() - one.list = [{ records: [one.records[NSIDS[0]]] }] + one.order = [NSIDS[0]] const html = await createHandler(one)(get('/')).text() assert.ok(html.includes(`rkey=${NSIDS[0]}`)) assert.ok(!html.includes(`rkey=${NSIDS[1]}`), 'listed a lexicon the bundle does not serve') diff --git a/test/serve/node.test.js b/test/serve/node.test.js index 0c592c5..6a1cb00 100644 --- a/test/serve/node.test.js +++ b/test/serve/node.test.js @@ -6,8 +6,13 @@ */ import assert from 'node:assert/strict' +import { mkdtemp, rm } from 'node:fs/promises' +import { request } from 'node:http' +import { createServer, connect } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { after, before, describe, it } from 'node:test' -import { serve } from '../../src/node/index.js' +import { createRequestListener, serve } from '../../src/node/index.js' import { COLLECTION, DID, makeData, xrpc } from './fixture.js' const data = makeData() @@ -95,6 +100,50 @@ describe('corpora/node over HTTP', () => { assert.equal(res.status, 404) assert.equal((await res.json()).error, 'NotFound') }) + + it('answers an HTTP/1.0 request that carries no Host header', async () => { + // Host is optional before HTTP/1.1, and `new URL()` needs an authority regardless. A + // hostname is invented rather than throwing, because the handler routes on path and query. + const raw = await rawRequest(server.port, ['GET /.well-known/did.json HTTP/1.0']) + assert.match(raw, /^HTTP\/1\.1 200 OK/) + assert.equal(JSON.parse(raw.slice(raw.indexOf('\r\n\r\n') + 4)).id, DID) + }) + + it('does not trip over a header sent more than once', async () => { + const raw = await rawRequest(server.port, [ + 'GET /.well-known/did.json HTTP/1.1', + `Host: 127.0.0.1:${server.port}`, + 'Connection: close', + 'Set-Cookie: a=1', + 'Set-Cookie: b=2', + ]) + assert.match(raw, /^HTTP\/1\.1 200 OK/) + assert.equal(JSON.parse(raw.slice(raw.indexOf('\r\n\r\n') + 4)).id, DID) + }) +}) + +describe('createRequestListener', () => { + it('invents an origin when there is neither a url nor a host', async () => { + // Both are typed optional on IncomingMessage. Neither should turn into a `new URL()` + // TypeError that Node would surface as a dropped socket rather than a response. + const listener = createRequestListener( + (req) => + new Response( + JSON.stringify({ url: req.url, repeat: req.headers.get('x-repeat'), host: req.headers.get('host') }), + ), + ) + const sink = responseSink() + await listener( + /** @type {any} */ ({ method: 'GET', headers: { host: undefined, 'x-repeat': ['a', 'b'] } }), + /** @type {any} */ (sink), + ) + + assert.equal(sink.statusCode, 200) + const body = JSON.parse(new TextDecoder().decode(sink.body)) + assert.equal(body.url, 'http://localhost/') + assert.equal(body.host, null, 'a header with no value is not forwarded as one') + assert.equal(body.repeat, 'a, b', 'a repeated header keeps both values') + }) }) describe('serve() options', () => { @@ -120,4 +169,125 @@ describe('serve() options', () => { await s.close() } }) + + it('defaults to 127.0.0.1:2583, the port the CLI documents', async (t) => { + /** @type {Awaited>} */ + let s + try { + s = await serve(data) + } catch (err) { + // A developer's own `corpora serve` on the default port would otherwise fail this. + if (/** @type {NodeJS.ErrnoException} */ (err).code === 'EADDRINUSE') return t.skip('2583 is busy') + throw err + } + try { + assert.equal(s.port, 2583) + assert.equal(s.url, 'http://127.0.0.1:2583') + assert.equal((await fetch(`${s.url}/.well-known/did.json`)).status, 200) + } finally { + await s.close() + } + }) + + it('rejects instead of hanging when the port is already taken', async () => { + const squatter = createServer() + await new Promise((done) => squatter.listen(0, '127.0.0.1', () => done(undefined))) + const address = squatter.address() + assert.ok(address !== null && typeof address === 'object') + try { + await assert.rejects(() => serve(data, { port: address.port }), { code: 'EADDRINUSE' }) + } finally { + await new Promise((done) => squatter.close(() => done(undefined))) + } + }) + + it('refuses anything that is not a port, including a unix socket path', async () => { + // `node:http` would read a non-numeric string as a pipe name and `server.address()` would + // answer with that path, so `port` and `url` would come back as `/tmp/…/s` and + // `http://127.0.0.1://tmp/…/s`. Sockets are not a supported input; the declared type is + // enforced instead, over the range `corpora serve --port` documents. + const dir = await mkdtemp(join(tmpdir(), 'corpora-sock-')) + const path = join(dir, 's') + try { + assert.throws(() => serve(data, /** @type {any} */ ({ port: path })), { + name: 'RangeError', + message: `serve: port must be 0-65535, got '${path}'`, + }) + await assert.rejects(() => overSocket(path, '/.well-known/did.json'), { code: 'ENOENT' }, 'nothing bound') + + for (const bad of ['2583', -1, 65536, 2583.5, NaN, null, {}]) { + assert.throws( + () => serve(data, /** @type {any} */ ({ port: bad })), + { name: 'RangeError', message: `serve: port must be 0-65535, got '${String(bad)}'` }, + `port ${String(bad)} was accepted`, + ) + } + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + + it('surfaces an error from close() rather than resolving a second time', async () => { + const s = await serve(data, { port: 0 }) + await s.close() + await assert.rejects(() => s.close(), { code: 'ERR_SERVER_NOT_RUNNING' }) + }) }) + +/** + * Speaks HTTP by hand, for the requests `fetch` will not send: no Host header, or the same + * header twice. + * + * @param {number} port + * @param {string[]} lines request line and headers, without the terminating blank line + * @returns {Promise} + */ +function rawRequest(port, lines) { + return new Promise((resolve, reject) => { + let out = '' + const socket = connect(port, '127.0.0.1', () => socket.end(`${lines.join('\r\n')}\r\n\r\n`)) + socket.setEncoding('utf8') + socket.on('data', (chunk) => (out += chunk)) + socket.on('error', reject) + socket.on('close', () => resolve(out)) + }) +} + +/** + * A GET over a unix socket, which `fetch` cannot address. + * + * @param {string} socketPath + * @param {string} path + * @returns {Promise<{ status: number, body: string }>} + */ +function overSocket(socketPath, path) { + return new Promise((resolve, reject) => { + const req = request({ socketPath, path, method: 'GET' }, (res) => { + let body = '' + res.setEncoding('utf8') + res.on('data', (chunk) => (body += chunk)) + res.on('end', () => resolve({ status: res.statusCode ?? 0, body })) + }) + req.on('error', reject) + req.end() + }) +} + +/** A ServerResponse stand-in that records what the listener wrote to it. */ +function responseSink() { + return { + statusCode: 0, + /** @type {Record} */ + headers: {}, + /** @type {Uint8Array | undefined} */ + body: undefined, + /** @param {string} name @param {string} value */ + setHeader(name, value) { + this.headers[name] = value + }, + /** @param {Uint8Array} [body] */ + end(body) { + this.body = body + }, + } +} diff --git a/test/verify/cli.test.js b/test/verify/cli.test.js new file mode 100644 index 0000000..ee4aef9 --- /dev/null +++ b/test/verify/cli.test.js @@ -0,0 +1,219 @@ +/** + * `corpora verify` as an argument parser and a report writer. + * + * The report is the product: it is read once, under stress, during a cutover. So the passing + * shape, the failing shape, and the colour decision are asserted here directly, against reports + * built by hand — no authority, no DNS, no network. What needs a live authority is in + * test/e2e/verify.test.js. + */ + +import assert from 'node:assert/strict' +import test from 'node:test' + +import { format, run as verifyCli } from '../../src/verify/cli.js' + +/** + * Runs the subcommand with stdout and stderr captured. + * + * @param {import('node:test').TestContext} t + * @param {string[]} argv + */ +async function capture(t, argv) { + let out = '' + let err = '' + t.mock.method(process.stdout, 'write', (/** @type {string} */ chunk) => void (out += chunk)) + t.mock.method(process.stderr, 'write', (/** @type {string} */ chunk) => void (err += chunk)) + const code = await verifyCli(argv) + t.mock.restoreAll() + return { code, out, err } +} + +/** + * An origin nothing is listening on: the connection is refused before any check can run, which + * is the cheapest way to reach the reporting half of `run` without a server. + */ +const DEAD_ORIGIN = 'http://127.0.0.1:1' + +/** + * @param {Partial[]} checks + * @param {Partial} [rest] + * @returns {import('../../src/verify/index.js').VerifyReport} + */ +function report(checks, rest = {}) { + const full = checks.map((c) => ({ name: 'a check', ok: true, detail: 'fine', ...c })) + return { ok: full.every((c) => c.ok), checks: full, nsids: [], ...rest } +} + +test('--help prints usage and exits 0 without touching the network', async (t) => { + const { code, out, err } = await capture(t, ['--help']) + assert.equal(code, 0) + assert.equal(err, '') + assert.match(out, /^corpora verify — check a built or deployed authority over HTTP/) + assert.match(out, /--origin /) + assert.match(out, /Exits non-zero if any check fails/) +}) + +test('-h is the same as --help, and beats a missing --origin', async (t) => { + // `-h` is checked before parseArgs, so it must win even when the rest of argv is unusable. + const { code, out, err } = await capture(t, ['-h']) + assert.equal(code, 0) + assert.equal(err, '') + assert.match(out, /^corpora verify — /) +}) + +test('--origin is required, and the refusal repeats the usage', async (t) => { + const { code, out, err } = await capture(t, ['--did', 'did:web:example.com']) + assert.equal(code, 1) + assert.equal(out, '') + assert.match(err, /^corpora verify: --origin is required\n\n/) + assert.match(err, /--origin /) +}) + +test('an unreachable origin fails the first check and exits 1', async (t) => { + const { code, out } = await capture(t, ['--origin', DEAD_ORIGIN]) + assert.equal(code, 1) + assert.match(out, /FAIL/) + assert.match(out, /did\.json reachable/) + assert.match(out, /1 of 1 check failed/) +}) + +test('--json emits the report as JSON instead of the human report', async (t) => { + const { code, out } = await capture(t, ['--origin', DEAD_ORIGIN, '--json']) + assert.equal(code, 1) + const parsed = JSON.parse(out) + assert.equal(parsed.ok, false) + assert.equal(parsed.origin, DEAD_ORIGIN) + assert.equal(parsed.checks[0].ok, false) + assert.equal(out.includes('FAIL'), false, '--json must not carry the human report') +}) + +test('resolvers come from --dns-server, or from CORPORA_DNS_SERVERS, or from the system', async (t) => { + // The origin is dead, so no lookup happens either way; what is asserted is that each spelling + // is accepted and reaches the verifier rather than being rejected as an unknown option. + const previous = process.env.CORPORA_DNS_SERVERS + t.after(() => { + if (previous === undefined) delete process.env.CORPORA_DNS_SERVERS + else process.env.CORPORA_DNS_SERVERS = previous + }) + + delete process.env.CORPORA_DNS_SERVERS + const flagged = await capture(t, [ + '--origin', + DEAD_ORIGIN, + '--dns-server', + '127.0.0.1:5333', + '--dns-server', + '127.0.0.1:5334', + '--json', + ]) + assert.equal(flagged.code, 1) + + process.env.CORPORA_DNS_SERVERS = ' 127.0.0.1:5333 , ,127.0.0.1:5334 ' + const fromEnv = await capture(t, ['--origin', DEAD_ORIGIN, '--json']) + assert.equal(fromEnv.code, 1) + + process.env.CORPORA_DNS_SERVERS = '' + const empty = await capture(t, ['--origin', DEAD_ORIGIN, '--json']) + assert.equal(empty.code, 1) +}) + +test('the also-accepted flags are accepted', async (t) => { + const { code } = await capture(t, [ + '--origin', + DEAD_ORIGIN, + '--source', + 'test/fixtures/lexicons', + '--expect-did-doc', + `sha256-${'0'.repeat(64)}`, + '--no-dns', + '--json', + ]) + assert.equal(code, 1) +}) + +test('a clean report says passed, and says what it passed over', () => { + const text = format( + report([ + { name: 'did.json responds', ok: true, detail: 'HTTP 200', group: 'identity' }, + { name: 'com.example.thing', ok: true, detail: 'proof verifies', group: 'namespace' }, + ], { did: 'did:web:lexicon.example.com', origin: 'https://lexicon.example.com', nsids: ['a', 'b'] }), + ) + assert.match(text, /corpora verify {2}https:\/\/lexicon\.example\.com/) + assert.match(text, /^ {4}ok {4}did\.json responds {2}HTTP 200$/m) + assert.match(text, /passed {2}2 checks, 2 lexicons under did:web:lexicon\.example\.com/) + assert.equal(text.includes('FAIL'), false) + assert.equal(/\x1b\[/.test(text), false, 'no escape codes when stdout is not a TTY') +}) + +test('one lexicon and one check are said in the singular', () => { + const text = format(report([{ name: 'DID', ok: true }], { did: 'did:web:x.example', nsids: ['a'] })) + assert.match(text, /passed {2}1 check, 1 lexicon under did:web:x\.example/) +}) + +test('failures are repeated at the end with the detail that says what to do', () => { + const text = format( + report( + [ + { name: 'did.json responds', ok: true, detail: 'HTTP 200', group: 'identity' }, + { name: '_lexicon.example.com', ok: false, detail: 'no TXT record', group: 'dns' }, + { name: 'unknown rkey is refused', ok: false, detail: 'HTTP 200, expected 404', group: 'namespace' }, + ], + { did: 'did:web:lexicon.example.com' }, + ), + ) + // The group headings are printed once, in the order the checks arrive. + assert.deepEqual( + text.split('\n').filter((l) => /^ {2}\S/.test(l) && !l.startsWith(' corpora')), + [' identity', ' dns', ' namespace', ' 2 of 3 checks failed, did:web:lexicon.example.com'], + ) + assert.match(text, /^ {4}FAIL {2}_lexicon\.example\.com/m) + assert.match(text, /^ {4}· _lexicon\.example\.com: no TXT record$/m) + assert.match(text, /^ {4}· unknown rkey is refused: HTTP 200, expected 404$/m) +}) + +test('a check that belongs to no group opens a nameless section', () => { + // `group` is optional on a check, so a grouped run followed by an ungrouped one must not + // print the word `undefined` where a heading goes. + const text = format( + report([ + { name: 'did.json responds', ok: true, group: 'identity' }, + { name: 'a loose check', ok: true }, + ]), + ) + assert.deepEqual( + text.split('\n').filter((l) => /^ {2}\S?/.test(l) && !l.startsWith(' ')), + [' identity', ' ', ' passed 2 checks'], + ) + assert.equal(text.includes('undefined'), false) +}) + +test('a report with no origin, no did and no groups still formats', () => { + // Everything but `checks` is optional on the way out of a verifier that bailed early. + const text = format({ ok: false, checks: [{ name: 'did.json reachable', ok: false, detail: 'ECONNREFUSED' }], nsids: [] }) + assert.match(text, /corpora verify\s*$/m) + assert.match(text, /^ {2}1 of 1 check failed$/m, 'no scope means no trailing comma') + assert.match(text, /^ {4}FAIL {2}did\.json reachable {2}ECONNREFUSED$/m) +}) + +test('colour is used when stdout is a TTY, and suppressed by NO_COLOR', (t) => { + const wasTTY = process.stdout.isTTY + const wasNoColor = process.env.NO_COLOR + t.after(() => { + process.stdout.isTTY = wasTTY + if (wasNoColor === undefined) delete process.env.NO_COLOR + else process.env.NO_COLOR = wasNoColor + }) + + const both = report([{ name: 'ok check', ok: true }, { name: 'bad check', ok: false }], { did: 'did:web:x.example' }) + + process.stdout.isTTY = true + delete process.env.NO_COLOR + const coloured = format(both) + assert.match(coloured, /\x1b\[32mok {2}\x1b\[0m/, 'a passing mark is green') + assert.match(coloured, /\x1b\[31mFAIL\x1b\[0m/, 'a failing mark is red') + assert.match(coloured, /\x1b\[1mcorpora verify\x1b\[0m/, 'the heading is bold') + assert.match(format(report([{ ok: true }])), /\x1b\[32mpassed\x1b\[0m/) + + process.env.NO_COLOR = '1' + assert.equal(/\x1b\[/.test(format(both)), false, 'NO_COLOR wins over a TTY') +}) diff --git a/test/verify/index.test.js b/test/verify/index.test.js new file mode 100644 index 0000000..d0bf787 --- /dev/null +++ b/test/verify/index.test.js @@ -0,0 +1,1116 @@ +/** + * `verifyAuthority` against a synthetic authority, one failure at a time. + * + * The end-to-end suite already proves the command works against a real local authority. What it + * cannot do cheaply is serve a *broken* one: a 500 on one record and a 200 on the next, a proof CAR + * signed by the right key over the wrong commit, a `listRecords` that pages forever. Those are the + * paths a verifier exists for, so they are exercised here against an in-memory authority built from + * `@atproto/repo` and handed to `verifyAuthority` through its `fetch` seam. + * + * Nothing is mocked below the wire: every proof is a real CAR signed by a real secp256k1 key, and + * every failure is produced by serving genuinely wrong bytes rather than by stubbing a checker. + */ + +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { after, before, describe, it } from 'node:test' + +import { Secp256k1Keypair } from '@atproto/crypto' +import { + blocksToCarFile, + cborToLexRecord, + concatBytesAsync, + getRecords, + MemoryBlockstore, + MST, + readCar, + Repo, + WriteOpAction, +} from '@atproto/repo' + +import { authorityDomain, authorityDomains, verifyAuthority, verifyProof } from '../../src/verify/index.js' +import { LEXICON_DIR } from '../helpers/build.js' +import { startDnsServer, useLocalDns } from '../helpers/dns.js' + +const COLLECTION = 'com.atproto.lexicon.schema' +const CAR_CONTENT_TYPE = 'application/vnd.ipld.car' +const ORIGIN = 'https://lexicon.example.com' +const DID = 'did:web:lexicon.example.com' +const REV = '3lbb2ekhjyk2t' +const OTHER_REV = '3lbb2ekhjyk3t' + +// ------------------------------------------------------------------ fixtures + +/** + * @typedef {object} World + * @property {string} did + * @property {import('@atproto/crypto').Keypair} keypair + * @property {string} signingKey did:key of the key that signed the commit + * @property {Record} didDoc + * @property {string} didDocBody the exact bytes served at /.well-known/did.json + * @property {string[]} nsids + * @property {Record} proofs nsid -> sync.getRecord CAR + * @property {Record} values nsid -> repo.getRecord body + * @property {Record} cids nsid -> record CID object + * @property {string} root commit CID + */ + +/** @param {string} dir @returns {Promise[]>} */ +async function readLexicons(dir) { + /** @type {Record[]} */ + const docs = [] + for (const name of (await readdir(dir)).sort()) { + if (name.endsWith('.json')) docs.push(JSON.parse(await readFile(join(dir, name), 'utf8'))) + } + return docs +} + +/** + * A signed repository plus everything an authority serves over it. Deterministic: the rev is fixed, + * so two worlds built from the same documents and key are byte-identical. + * + * @param {object} opts + * @param {import('@atproto/crypto').Keypair} opts.keypair + * @param {{ rkey: string, record: Record }[]} opts.writes + * @param {string} [opts.did] + * @param {string} [opts.rev] + * @param {string} [opts.endpoint] + * @param {string} [opts.publishKey] the did:key to put in did.json, when it is not the signer's + * @returns {Promise} + */ +async function makeWorld(opts) { + const did = opts.did ?? DID + const storage = new MemoryBlockstore() + const commitData = await Repo.formatInitCommit( + storage, + did, + opts.keypair, + opts.writes.map((w) => ({ + action: WriteOpAction.Create, + collection: /** @type {`${string}.${string}.${string}`} */ (COLLECTION), + rkey: w.rkey, + record: /** @type {any} */ (w.record), + })), + opts.rev ?? REV, + ) + const repo = await Repo.createFromCommit(storage, commitData) + const mst = MST.load(storage, repo.commit.data) + + /** @type {Record} */ + const proofs = {} + /** @type {Record} */ + const values = {} + /** @type {Record} */ + const cids = {} + for (const { rkey } of opts.writes) { + const cid = await mst.get(`${COLLECTION}/${rkey}`) + if (!cid) throw new Error(`fixture repo is missing ${rkey}`) + cids[rkey] = cid + proofs[rkey] = await concatBytesAsync(getRecords(storage, repo.cid, [{ collection: COLLECTION, rkey }])) + const bytes = await storage.getBytes(cid) + values[rkey] = { + uri: `at://${did}/${COLLECTION}/${rkey}`, + cid: cid.toString(), + value: cborToLexRecord(/** @type {Uint8Array} */ (bytes)), + } + } + + const signingKey = opts.keypair.did() + const didDoc = didDocument({ did, key: opts.publishKey ?? signingKey, endpoint: opts.endpoint ?? ORIGIN }) + return { + did, + keypair: opts.keypair, + signingKey, + didDoc, + didDocBody: JSON.stringify(didDoc, null, 2), + nsids: opts.writes.map((w) => w.rkey).sort(), + proofs, + values, + cids, + root: repo.cid.toString(), + } +} + +/** @param {{ did: string, key: string, endpoint: string }} opts */ +function didDocument({ did, key, endpoint }) { + return { + '@context': ['https://www.w3.org/ns/did/v1'], + id: did, + alsoKnownAs: [], + verificationMethod: [ + { + id: `${did}#atproto`, + type: 'Multikey', + controller: did, + publicKeyMultibase: key.slice('did:key:'.length), + }, + ], + service: [{ id: '#atproto_pds', type: 'AtprotoPersonalDataServer', serviceEndpoint: endpoint }], + } +} + +/** @param {Record} doc @param {any} [id] */ +const lexiconWrite = (doc, id = doc.id) => ({ rkey: id, record: { ...doc, $type: COLLECTION } }) + +// -------------------------------------------------------------------- server + +/** @param {unknown} body @param {number} [status] */ +const json = (body, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json; charset=utf-8' } }) + +const notFound = () => json({ error: 'RecordNotFound', message: 'record not found' }, 404) + +/** + * The minimum surface docs/RESOLUTION.md requires, answered from a signed in-memory repo. + * @param {World} world + * @param {URL} url + * @returns {Response} + */ +function route(world, url) { + const q = url.searchParams + switch (url.pathname) { + case '/.well-known/did.json': + return new Response(world.didDocBody, { status: 200, headers: { 'content-type': 'application/json' } }) + case '/xrpc/com.atproto.repo.listRecords': { + if (q.get('repo') !== world.did) return notFound() + return json({ records: world.nsids.map((n) => ({ uri: world.values[n].uri, cid: world.values[n].cid })) }) + } + case '/xrpc/com.atproto.sync.getRecord': { + if (q.get('did') !== world.did || q.get('collection') !== COLLECTION) return notFound() + const proof = world.proofs[q.get('rkey') ?? ''] + if (!proof) return notFound() + return new Response(/** @type {BodyInit} */ (proof), { headers: { 'content-type': CAR_CONTENT_TYPE } }) + } + case '/xrpc/com.atproto.repo.getRecord': { + if (q.get('repo') !== world.did) return notFound() + const value = world.values[q.get('rkey') ?? ''] + return value ? json(value) : notFound() + } + default: + return notFound() + } +} + +/** @typedef {(url: URL, next: (url: URL) => Response) => Response | undefined | Promise} Intercept */ + +/** + * @param {World} world + * @param {Intercept} [intercept] + * @returns {typeof globalThis.fetch} + */ +function makeFetch(world, intercept) { + const next = (/** @type {URL} */ url) => route(world, url) + return /** @type {any} */ ( + async (/** @type {any} */ input) => { + const url = input instanceof URL ? input : new URL(String(input)) + return (await intercept?.(url, next)) ?? next(url) + } + ) +} + +// ---------------------------------------------------------------------- suite + +/** @type {World} */ +let base +/** @type {World} */ +let otherRev +/** @type {World} */ +let single +/** @type {Record[]} */ +let docs +/** @type {Awaited>} */ +let dns +/** @type {string} */ +let sourceExtra +/** @type {string} */ +let sourceSubset +/** @type {string} */ +let workdir + +before(async () => { + docs = await readLexicons(LEXICON_DIR) + const keypair = await Secp256k1Keypair.create() + const writes = docs.map((d) => lexiconWrite(d)) + base = await makeWorld({ keypair, writes }) + // Same records, same key, a different rev — so a different commit root, which is what a + // deployment serving a mix of builds looks like from outside. + otherRev = await makeWorld({ keypair, writes, rev: OTHER_REV }) + single = await makeWorld({ keypair, writes: [lexiconWrite(docs[3])] }) + + dns = await startDnsServer({ + records: { + '_lexicon.example.com': `did=${base.did}`, + '_lexicon.nested.example.com': `did=${base.did}`, + }, + }) + + workdir = await mkdtemp(join(tmpdir(), 'corpora-verify-')) + + // A source tree with one lexicon that is not published, plus the files readSources must ignore. + sourceExtra = join(workdir, 'extra') + await mkdir(join(sourceExtra, 'nested'), { recursive: true }) + for (const doc of docs) await writeFile(join(sourceExtra, `${doc.id}.json`), JSON.stringify(doc)) + await writeFile( + join(sourceExtra, 'nested', 'com.example.unpublished.json'), + JSON.stringify({ lexicon: 1, id: 'com.example.unpublished', defs: {} }), + ) + await writeFile(join(sourceExtra, 'README.txt'), 'not a lexicon') + await writeFile(join(sourceExtra, 'package-ish.json'), JSON.stringify({ name: 'no id here' })) + + // A source tree missing two of the published lexicons. + sourceSubset = join(workdir, 'subset') + await mkdir(sourceSubset, { recursive: true }) + for (const doc of docs.slice(0, 2)) await writeFile(join(sourceSubset, `${doc.id}.json`), JSON.stringify(doc)) +}) + +after(async () => { + await dns?.close() + await rm(workdir, { recursive: true, force: true }) +}) + +/** + * @param {Partial & { world?: World, intercept?: Intercept }} [opts] + */ +const verify = (opts = {}) => { + const { world = base, intercept, ...rest } = opts + return verifyAuthority({ origin: ORIGIN, skipDns: true, fetch: makeFetch(world, intercept), ...rest }) +} + +/** + * @param {import('../../src/verify/index.js').VerifyReport} report + * @param {string} name + * @param {string} [group] + */ +const check = (report, name, group) => report.checks.find((c) => c.name === name && (!group || c.group === group)) + +/** @param {import('../../src/verify/index.js').VerifyReport} report @param {string} name @param {string} [group] */ +const detail = (report, name, group) => check(report, name, group)?.detail ?? '' + +/** @param {import('../../src/verify/index.js').VerifyReport} report */ +const failures = (report) => report.checks.filter((c) => !c.ok).map((c) => c.name) + +/** Serve a hand-made did.json in place of the real one. @param {unknown} body */ +const serveDidDoc = (body) => /** @type {Intercept} */ ( + (url) => + url.pathname === '/.well-known/did.json' + ? new Response(typeof body === 'string' ? body : JSON.stringify(body), { + status: 200, + headers: { 'content-type': 'application/json' }, + }) + : undefined +) + +/** Serve a did.json derived from the healthy one. @param {(doc: any) => void} patch */ +const patchDidDoc = (patch) => { + const doc = JSON.parse(base.didDocBody) + patch(doc) + return serveDidDoc(doc) +} + +describe('verifyAuthority — identity', () => { + it('passes against an authority that serves everything correctly', async () => { + const report = await verify({ did: DID, sourceDir: LEXICON_DIR }) + assert.deepEqual(failures(report), [], JSON.stringify(report.checks.filter((c) => !c.ok), null, 2)) + assert.equal(report.ok, true) + assert.equal(report.did, DID) + assert.equal(report.origin, ORIGIN) + assert.deepEqual(report.nsids, base.nsids) + assert.equal(detail(report, 'DID'), DID) + assert.equal(detail(report, 'signing key'), base.signingKey) + assert.equal(detail(report, 'service endpoint'), ORIGIN) + assert.equal(detail(report, 'listRecords'), `4 records in ${COLLECTION}`) + assert.equal(detail(report, 'one commit behind every proof'), base.root) + assert.equal(detail(report, 'com.example.thing', 'records'), 'repo.getRecord matches the signed record') + assert.equal(detail(report, 'com.example.thing', 'source'), 'matches source') + assert.match(detail(report, 'sync.getRecord varies by rkey'), /return different proofs/) + assert.match(detail(report, 'unknown rkey is refused'), /^HTTP 404$/) + }) + + it('strips trailing slashes from the origin before building request URLs', async () => { + /** @type {string[]} */ + const seen = [] + const report = await verifyAuthority({ + origin: `${ORIGIN}///`, + skipDns: true, + fetch: makeFetch(base, (url) => { + seen.push(url.href) + return undefined + }), + }) + assert.equal(report.origin, ORIGIN) + assert.equal(seen[0], `${ORIGIN}/.well-known/did.json`) + assert.ok(seen.every((u) => !u.includes('//.well-known'))) + }) + + it('stops at the first request when did.json cannot be fetched at all', async () => { + const report = await verify({ + intercept: (url) => { + if (url.pathname.endsWith('did.json')) throw new Error('ECONNREFUSED') + return undefined + }, + }) + assert.equal(report.ok, false) + assert.deepEqual(report.checks.map((c) => c.name), ['did.json reachable']) + assert.equal(detail(report, 'did.json reachable'), `${ORIGIN}/.well-known/did.json: ECONNREFUSED`) + assert.deepEqual(report.nsids, []) + assert.equal(report.did, undefined) + }) + + it('falls back to the global fetch when none is supplied', async () => { + // Port 1 refuses immediately, so this exercises the default fetch without touching a network. + const report = await verifyAuthority({ origin: 'http://127.0.0.1:1', skipDns: true }) + assert.equal(report.ok, false) + assert.deepEqual(report.checks.map((c) => c.name), ['did.json reachable']) + assert.match(detail(report, 'did.json reachable'), /^http:\/\/127\.0\.0\.1:1\/\.well-known\/did\.json: /) + }) + + it('reports a non-Error rejection as its string form', async () => { + const report = await verify({ + intercept: () => { + throw 'socket hang up' + }, + }) + assert.match(detail(report, 'did.json reachable'), /socket hang up$/) + }) + + it('treats a redirect on did.json as a hard failure, and says why', async () => { + const report = await verify({ + intercept: (url) => + url.pathname.endsWith('did.json') + ? new Response(null, { status: 301, headers: { location: 'https://www.example.com/.well-known/did.json' } }) + : undefined, + }) + assert.equal(report.ok, false) + assert.equal(check(report, 'did.json responds')?.ok, false) + assert.match(detail(report, 'did.json responds'), /HTTP 301 to https:\/\/www\.example\.com/) + assert.match(detail(report, 'did.json responds'), /redirect:'error'/) + assert.deepEqual(report.checks.map((c) => c.name), ['did.json responds'], 'nothing else can be checked') + }) + + it('reports the content type of a did.json that is not a 200, and its absence', async () => { + const withType = await verify({ + intercept: (url) => (url.pathname.endsWith('did.json') ? json({ error: 'NotFound' }, 404) : undefined), + }) + assert.equal(detail(withType, 'did.json responds'), 'HTTP 404, application/json; charset=utf-8') + + const without = await verify({ + intercept: (url) => (url.pathname.endsWith('did.json') ? new Response(null, { status: 502 }) : undefined), + }) + assert.equal(detail(without, 'did.json responds'), 'HTTP 502, no content-type') + }) + + it('stops when did.json is not JSON', async () => { + const report = await verify({ intercept: serveDidDoc('

404

') }) + assert.equal(report.ok, false) + assert.equal(check(report, 'did.json parses')?.ok, false) + assert.match(detail(report, 'did.json parses'), /JSON/i) + assert.deepEqual(report.checks.map((c) => c.name), ['did.json responds', 'did.json parses']) + assert.deepEqual(report.nsids, []) + }) + + const shapes = [ + { what: 'no id', doc: { verificationMethod: [], service: [] }, detail: 'no `id`' }, + { what: 'a non-string id', doc: { id: 42 }, detail: 'no `id`' }, + { + what: 'no #atproto verification method', + doc: { id: DID, service: [] }, + detail: 'no verificationMethod with id `#atproto` (or `#atproto`)', + }, + { + what: 'a verification method with no key material', + doc: { id: DID, verificationMethod: [{ id: '#atproto', type: 'Multikey' }] }, + detail: 'the #atproto verificationMethod has no publicKeyMultibase', + }, + { + what: 'no #atproto_pds service', + doc: { id: DID, verificationMethod: [{ id: '#atproto', publicKeyMultibase: 'z1' }] }, + detail: 'no service with id `#atproto_pds`', + }, + { + what: 'a service of the wrong type', + doc: { + id: DID, + verificationMethod: [{ id: '#atproto', publicKeyMultibase: 'z1' }], + service: [{ id: '#atproto_pds', type: 'LinkedDomains', serviceEndpoint: ORIGIN }], + }, + detail: 'service #atproto_pds has type LinkedDomains, expected AtprotoPersonalDataServer', + }, + { + what: 'a service with no endpoint', + doc: { + id: DID, + verificationMethod: [{ id: '#atproto', publicKeyMultibase: 'z1' }], + service: [{ id: `${DID}#atproto_pds`, type: 'AtprotoPersonalDataServer' }], + }, + detail: 'service #atproto_pds has no serviceEndpoint', + }, + ] + + for (const shape of shapes) { + it(`stops on a did.json with ${shape.what}`, async () => { + const report = await verify({ intercept: serveDidDoc(shape.doc) }) + assert.equal(report.ok, false) + assert.equal(check(report, 'did.json shape')?.ok, false) + assert.equal(detail(report, 'did.json shape'), shape.detail) + assert.equal(report.checks.at(-1)?.name, 'did.json shape', 'nothing is checked past a malformed document') + }) + } + + it('accepts the relative form of the #atproto ids', async () => { + const report = await verify({ + intercept: patchDidDoc((doc) => { + doc.verificationMethod[0].id = '#atproto' + doc.service[0].id = `${DID}#atproto_pds` + }), + }) + assert.equal(check(report, 'did.json shape')?.ok, true) + assert.equal(detail(report, 'did.json shape'), 'verificationMethod #atproto and service #atproto_pds present') + assert.equal(check(report, 'signing key')?.ok, true) + }) + + it('fails the DID check when the document claims a different DID than the one expected', async () => { + const report = await verify({ did: 'did:web:someone-else.example' }) + assert.equal(report.ok, false) + assert.equal(check(report, 'DID')?.ok, false) + assert.equal(detail(report, 'DID'), `document says ${DID}, expected did:web:someone-else.example`) + assert.equal(report.did, DID, 'the report still carries the DID that was actually served') + }) + + it('stops when the #atproto key is unusable', async () => { + const report = await verify({ + // A well-formed multibase value under a verification method type atproto does not know. + intercept: patchDidDoc((doc) => (doc.verificationMethod[0].type = 'JsonWebKey2020')), + }) + assert.equal(report.ok, false) + assert.equal(check(report, 'signing key')?.ok, false) + assert.equal(detail(report, 'signing key'), 'no usable #atproto verification method') + assert.equal(report.checks.some((c) => c.group === 'namespace'), false, 'no proof can be checked without a key') + assert.equal(report.did, DID) + assert.deepEqual(report.nsids, []) + }) + + it('stops when the service endpoint is not a usable URL', async () => { + const report = await verify({ + intercept: patchDidDoc((doc) => (doc.service[0].serviceEndpoint = 'ftp://lexicon.example.com')), + }) + assert.equal(report.ok, false) + assert.equal(check(report, 'service endpoint')?.ok, false) + assert.equal(detail(report, 'service endpoint'), 'missing') + assert.equal(report.checks.some((c) => c.group === 'namespace'), false) + }) + + it('notes, without failing, a service endpoint on a different origin', async () => { + const report = await verify({ + intercept: patchDidDoc((doc) => (doc.service[0].serviceEndpoint = 'https://pds.example.net')), + }) + assert.equal(check(report, 'service endpoint')?.ok, true) + assert.equal(detail(report, 'service endpoint'), 'https://pds.example.net (a different origin from the one being verified)') + }) + + it('checks the did.json fingerprint against the bytes actually served', async () => { + const digest = createHash('sha256').update(Buffer.from(base.didDocBody, 'utf8')).digest('hex') + + const bare = await verify({ expectDidDoc: digest }) + assert.equal(check(bare, 'did.json fingerprint')?.ok, true) + assert.equal(detail(bare, 'did.json fingerprint'), `sha256-${digest}`) + + const prefixed = await verify({ expectDidDoc: `sha256-${digest.toUpperCase()}` }) + assert.equal(check(prefixed, 'did.json fingerprint')?.ok, true, 'the prefix is optional and hex is case-insensitive') + + const wrong = await verify({ expectDidDoc: `sha256-${'0'.repeat(64)}` }) + assert.equal(check(wrong, 'did.json fingerprint')?.ok, false) + assert.equal(detail(wrong, 'did.json fingerprint'), `sha256-${digest}, expected sha256-${'0'.repeat(64)}`) + }) +}) + +describe('verifyAuthority — enumerating the namespace', () => { + /** @param {(page: number, url: URL) => Response | undefined} handler */ + const pages = (handler) => { + let page = 0 + return /** @type {Intercept} */ ( + (url) => (url.pathname.endsWith('repo.listRecords') ? handler(page++, url) : undefined) + ) + } + + it('accepts an unserved listRecords when --source supplies the namespace', async () => { + const report = await verify({ sourceDir: LEXICON_DIR, intercept: pages(() => notFound()) }) + assert.equal(check(report, 'listRecords')?.ok, true, 'enumeration is optional') + assert.equal(detail(report, 'listRecords'), 'not served (optional); enumerating 4 NSIDs from --source instead') + assert.deepEqual(report.nsids, base.nsids) + assert.equal(report.ok, true, JSON.stringify(failures(report))) + }) + + it('will not let --source launder a listRecords that is served badly', async () => { + const report = await verify({ sourceDir: LEXICON_DIR, intercept: pages(() => json({ error: 'Oops' }, 500)) }) + assert.equal(check(report, 'listRecords')?.ok, false) + assert.equal(detail(report, 'listRecords'), 'repo.listRecords returned HTTP 500. Continuing with 4 NSIDs from --source') + assert.deepEqual(report.nsids, base.nsids) + assert.equal(check(report, 'com.example.thing', 'namespace')?.ok, true, 'the run continues past the failure') + assert.equal(report.ok, false) + }) + + it('gives up when listRecords fails and there is no --source to fall back on', async () => { + const report = await verify({ intercept: pages(() => json({ error: 'Oops' }, 503)) }) + assert.equal(check(report, 'listRecords')?.ok, false) + assert.match(detail(report, 'listRecords'), /^repo\.listRecords returned HTTP 503\./) + assert.match(detail(report, 'listRecords'), /pass --source to supply the list/) + assert.deepEqual(report.nsids, []) + assert.equal(report.checks.at(-1)?.name, 'listRecords') + }) + + it('reports a listRecords body that is not JSON', async () => { + const report = await verify({ intercept: pages(() => new Response('nope')) }) + assert.equal(check(report, 'listRecords')?.ok, false) + assert.match(detail(report, 'listRecords'), /^repo\.listRecords body is not JSON: /) + assert.deepEqual(report.nsids, []) + }) + + it('treats a 404 partway through a page walk as a real failure, not an unserved endpoint', async () => { + const report = await verify({ + sourceDir: LEXICON_DIR, + intercept: pages((page) => + page === 0 + ? json({ records: [{ uri: base.values[base.nsids[0]].uri, cid: 'x' }], cursor: 'page-2' }) + : notFound(), + ), + }) + assert.equal(check(report, 'listRecords')?.ok, false, 'optional only applies to the very first page') + assert.match(detail(report, 'listRecords'), /HTTP 404\. Continuing with 4 NSIDs from --source/) + }) + + it('follows the cursor across pages and merges what it finds', async () => { + /** @type {(string | null)[]} */ + const cursors = [] + const report = await verify({ + intercept: pages((page, url) => { + cursors.push(url.searchParams.get('cursor')) + const record = (/** @type {string} */ nsid) => ({ uri: base.values[nsid].uri, cid: base.values[nsid].cid }) + if (page === 0) return json({ records: [record(base.nsids[0]), record(base.nsids[1])], cursor: 'c1' }) + if (page === 1) return json({ records: [record(base.nsids[2]), record(base.nsids[3])], cursor: 'c2' }) + return json({ records: [], cursor: 'c3' }) + }), + }) + assert.deepEqual(cursors, [null, 'c1', 'c2'], 'the final empty page is what ends the walk, not a missing cursor') + assert.equal(detail(report, 'listRecords'), `4 records in ${COLLECTION}`) + assert.deepEqual(report.nsids, base.nsids) + assert.equal(report.ok, true, JSON.stringify(failures(report))) + }) + + it('catches paging that never advances because the cursor was dropped', async () => { + const record = { uri: base.values[base.nsids[0]].uri, cid: base.values[base.nsids[0]].cid } + const report = await verify({ intercept: pages(() => json({ records: [record], cursor: 'stuck' })) }) + assert.equal(check(report, 'listRecords')?.ok, false) + assert.match(detail(report, 'listRecords'), /returned cursor stuck twice/) + assert.match(detail(report, 'listRecords'), /caching on path alone and dropping the query string/) + }) + + it('stops paging after a bounded number of pages', async () => { + let served = 0 + const report = await verify({ + intercept: pages((page) => { + served++ + return json({ + records: [{ uri: `at://${DID}/${COLLECTION}/com.example.p${page}`, cid: `cid-${page}` }], + cursor: `cursor-${page}`, + }) + }), + }) + assert.equal(served, 1000, 'the walk is bounded, so a hostile server cannot hang the verifier') + assert.equal(check(report, 'listRecords')?.ok, false) + assert.equal(detail(report, 'listRecords'), 'repo.listRecords paged without end. Without it there is no way to enumerate the namespace; pass --source to supply the list') + }) + + it('fails an authority that publishes nothing', async () => { + const report = await verify({ intercept: pages(() => json({ records: [] })) }) + assert.equal(report.ok, false) + assert.equal(check(report, 'namespace is not empty')?.ok, false) + assert.equal(detail(report, 'namespace is not empty'), 'the authority publishes no lexicon records') + assert.deepEqual(report.nsids, []) + assert.equal(report.checks.at(-1)?.name, 'namespace is not empty') + }) + + it('fails an authority whose listRecords body has no records field at all', async () => { + const report = await verify({ intercept: pages(() => json({ cursor: 'anything' })) }) + assert.equal(check(report, 'namespace is not empty')?.ok, false) + }) + + it('counts a one-record namespace in the singular', async () => { + const report = await verify({ world: single }) + assert.equal(detail(report, 'listRecords'), `1 record in ${COLLECTION}`) + }) +}) + +describe('verifyAuthority — proofs', () => { + /** @param {(url: URL, next: (url: URL) => Response) => Response | undefined} handler */ + const onProof = (handler) => /** @type {Intercept} */ ( + (url, next) => (url.pathname.endsWith('sync.getRecord') ? handler(url, next) : undefined) + ) + + it('reports the status when sync.getRecord will not serve a proof', async () => { + const report = await verify({ + intercept: onProof((url, next) => + url.searchParams.get('rkey') === 'com.example.thing' ? json({ error: 'Oops' }, 500) : next(url), + ), + }) + assert.equal(report.ok, false) + assert.equal(detail(report, 'com.example.thing', 'namespace'), 'sync.getRecord returned HTTP 500') + assert.equal(check(report, 'com.example.getThings', 'namespace')?.ok, true, 'one bad record does not stop the walk') + assert.equal(report.checks.some((c) => c.group === 'records' && c.name === 'com.example.thing'), false) + }) + + it('says so when no proof verified at all', async () => { + const report = await verify({ intercept: onProof(() => json({ error: 'Oops' }, 502)) }) + assert.equal(detail(report, 'one commit behind every proof'), 'no proofs verified') + assert.equal(check(report, 'one commit behind every proof')?.ok, true, 'zero roots is not a mix of builds') + assert.equal(detail(report, 'repo.getRecord'), 'not served (optional); resolvers do not use it') + }) + + it('refuses a proof served under the wrong content type', async () => { + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.thing' + ? new Response(/** @type {BodyInit} */ (base.proofs['com.example.thing']), { + headers: { 'content-type': 'application/json; charset=utf-8' }, + }) + : undefined, + ), + }) + assert.equal( + detail(report, 'com.example.thing', 'namespace'), + `sync.getRecord served content-type application/json, expected ${CAR_CONTENT_TYPE}`, + ) + }) + + it('refuses a proof served with an empty content type', async () => { + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.thing' + ? new Response(/** @type {BodyInit} */ (base.proofs['com.example.thing']), { + headers: { 'content-type': '' }, + }) + : undefined, + ), + }) + assert.equal( + detail(report, 'com.example.thing', 'namespace'), + `sync.getRecord served content-type (none), expected ${CAR_CONTENT_TYPE}`, + ) + }) + + it('refuses a proof served with no content type header at all', async () => { + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.thing' + ? new Response(/** @type {BodyInit} */ (base.proofs['com.example.thing'])) + : undefined, + ), + }) + assert.equal( + detail(report, 'com.example.thing', 'namespace'), + `sync.getRecord served content-type (none), expected ${CAR_CONTENT_TYPE}`, + ) + }) + + it('reports a proof that is not a readable CAR', async () => { + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.thing' + ? new Response(new Uint8Array([0, 1, 2, 3]), { headers: { 'content-type': CAR_CONTENT_TYPE } }) + : undefined, + ), + }) + assert.match(detail(report, 'com.example.thing', 'namespace'), /^proof is not a readable CAR: /) + }) + + it('reports a proof that does not contain the record it was asked for', async () => { + // A validly signed commit from a repo that never held this NSID: the signature and the DID are + // both fine, and the MST simply has nothing at that key. + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.getThings' + ? new Response(/** @type {BodyInit} */ (single.proofs['com.example.thing']), { + headers: { 'content-type': CAR_CONTENT_TYPE }, + }) + : undefined, + ), + }) + assert.equal( + detail(report, 'com.example.getThings', 'namespace'), + `no ${COLLECTION}/com.example.getThings under the signed MST root`, + ) + }) + + it('reports a proof whose CAR is missing the record block itself', async () => { + // The proof for a *different* rkey: its MST path can reach this key's leaf, but the block the + // leaf points at was never included, so the walk fails at the last step. + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.thing' + ? new Response(/** @type {BodyInit} */ (base.proofs['com.example.getThings']), { + headers: { 'content-type': CAR_CONTENT_TYPE }, + }) + : undefined, + ), + }) + assert.equal(check(report, 'com.example.thing', 'namespace')?.ok, false) + assert.equal( + detail(report, 'com.example.thing', 'namespace'), + `block not found: ${base.values['com.example.thing'].cid}`, + ) + }) + + it('catches a deployment serving proofs from two different builds', async () => { + const report = await verify({ + intercept: onProof((url) => + url.searchParams.get('rkey') === 'com.example.thing' + ? new Response(/** @type {BodyInit} */ (otherRev.proofs['com.example.thing']), { + headers: { 'content-type': CAR_CONTENT_TYPE }, + }) + : undefined, + ), + }) + assert.notEqual(base.root, otherRev.root) + assert.equal(check(report, 'com.example.thing', 'namespace')?.ok, true, 'each proof verifies on its own') + assert.equal(check(report, 'one commit behind every proof')?.ok, false) + assert.equal( + detail(report, 'one commit behind every proof'), + '2 distinct commit roots — the deployment is a mix of builds', + ) + }) + + it('describes a verified proof by size and record CID', async () => { + const report = await verify() + const size = base.proofs['com.example.thing'].length.toLocaleString('en-US') + const cid = base.values['com.example.thing'].cid + assert.equal( + detail(report, 'com.example.thing', 'namespace'), + `proof verifies, ${size} bytes, record ${cid.slice(0, 10)}…${cid.slice(-4)}`, + ) + }) +}) + +describe('verifyAuthority — repo.getRecord', () => { + /** @param {(url: URL, next: (url: URL) => Response) => Response | undefined} handler */ + const onPlain = (handler) => /** @type {Intercept} */ ( + (url, next) => (url.pathname.endsWith('repo.getRecord') ? handler(url, next) : undefined) + ) + + it('accepts an authority that does not serve it', async () => { + const report = await verify({ intercept: onPlain(() => notFound()) }) + assert.equal(report.ok, true, JSON.stringify(failures(report))) + assert.equal(detail(report, 'repo.getRecord'), 'not served (optional); resolvers do not use it') + assert.equal(report.checks.filter((c) => c.group === 'records').length, 1) + }) + + it('reports a status that is neither 200 nor 404', async () => { + const report = await verify({ + intercept: onPlain((url, next) => + url.searchParams.get('rkey') === 'com.example.thing' ? json({ error: 'Oops' }, 500) : next(url), + ), + }) + assert.equal(report.ok, false) + assert.equal(detail(report, 'com.example.thing', 'records'), 'repo.getRecord returned HTTP 500') + }) + + it('reports a body that is not JSON', async () => { + const report = await verify({ + intercept: onPlain((url, next) => + url.searchParams.get('rkey') === 'com.example.thing' ? new Response('not json') : next(url), + ), + }) + assert.equal(report.ok, false) + assert.match(detail(report, 'com.example.thing', 'records'), /^repo\.getRecord body is not JSON: /) + }) + + it('catches a plain record that disagrees with the signed proof', async () => { + const report = await verify({ + intercept: onPlain((url, next) => { + if (url.searchParams.get('rkey') !== 'com.example.thing') return next(url) + const body = structuredClone(base.values['com.example.thing']) + body.value.defs.main.description = 'not what was signed' + return json(body) + }), + }) + assert.equal(report.ok, false) + assert.equal(detail(report, 'com.example.thing', 'records'), 'repo.getRecord disagrees with the signed proof') + }) + + it('catches a body with no value at all', async () => { + const report = await verify({ + intercept: onPlain((url, next) => + url.searchParams.get('rkey') === 'com.example.thing' ? json({ uri: 'at://x', cid: 'y' }) : next(url), + ), + }) + assert.equal(detail(report, 'com.example.thing', 'records'), 'repo.getRecord disagrees with the signed proof') + }) + + it('is not asked for records whose proof never verified', async () => { + /** @type {string[]} */ + const asked = [] + await verify({ + intercept: (url, next) => { + if (url.pathname.endsWith('repo.getRecord')) asked.push(url.searchParams.get('rkey') ?? '') + if (url.pathname.endsWith('sync.getRecord') && url.searchParams.get('rkey') === 'com.example.thing') { + return json({ error: 'Oops' }, 500) + } + return next(url) + }, + }) + assert.equal(asked.includes('com.example.thing'), false) + assert.equal(asked.length, 3) + }) +}) + +describe('verifyAuthority — source', () => { + it('reports lexicons in source that the authority does not publish', async () => { + const report = await verify({ sourceDir: sourceExtra }) + assert.equal(report.ok, false) + assert.equal(check(report, 'every source lexicon is published')?.ok, false) + assert.equal(detail(report, 'every source lexicon is published'), 'com.example.unpublished') + assert.equal(check(report, 'nothing is published that is not in source')?.ok, true) + assert.equal(detail(report, 'nothing is published that is not in source'), 'no strays') + }) + + it('reports published lexicons that are not in source', async () => { + const report = await verify({ sourceDir: sourceSubset }) + assert.equal(report.ok, false) + assert.equal(detail(report, 'every source lexicon is published'), '2 files') + assert.equal(check(report, 'nothing is published that is not in source')?.ok, false) + assert.equal( + detail(report, 'nothing is published that is not in source'), + 'com.example.nested.widget, com.example.thing', + ) + assert.equal(report.checks.some((c) => c.group === 'source' && c.name === 'com.example.thing'), false) + }) + + it('catches a served record that drifted from its source document', async () => { + const drifted = docs.map((doc) => structuredClone(doc)) + drifted[3].defs.main.description = 'edited after the build' + const world = await makeWorld({ keypair: base.keypair, writes: drifted.map((d) => lexiconWrite(d)) }) + + const report = await verify({ world, sourceDir: LEXICON_DIR }) + assert.equal(report.ok, false) + assert.equal(detail(report, 'com.example.thing', 'source'), 'served record differs from source') + assert.equal(detail(report, 'com.example.getThings', 'source'), 'matches source') + }) +}) + +describe('verifyAuthority — dns', () => { + /** @param {Partial & { world?: World, intercept?: Intercept }} [opts] */ + const verifyDns = (opts = {}) => verify({ skipDns: false, dnsServers: [`127.0.0.1:${dns.port}`], ...opts }) + + it('checks a TXT record for every authority domain the namespace uses', async () => { + const report = await verifyDns() + assert.equal(report.ok, true, JSON.stringify(failures(report))) + assert.equal(detail(report, '_lexicon.example.com'), `did=${DID} (3)`) + assert.equal(detail(report, '_lexicon.nested.example.com'), `did=${DID} (1)`) + }) + + it('reads the resolvers from this process when none are given', async () => { + const restore = useLocalDns(dns.port) + try { + const report = await verify({ skipDns: false }) + assert.equal(check(report, '_lexicon.example.com')?.ok, true) + } finally { + restore() + } + }) + + it('fails a domain whose TXT records carry no did=', async () => { + dns.setTxt('_lexicon.example.com', 'v=spf1 -all') + try { + const report = await verifyDns() + assert.equal(report.ok, false) + assert.equal(detail(report, '_lexicon.example.com'), 'no did= TXT record') + } finally { + dns.setTxt('_lexicon.example.com', `did=${DID}`) + } + }) + + it('fails a domain with more than one did= record', async () => { + dns.setTxt('_lexicon.example.com', [`did=${DID}`, 'did=did:web:someone-else.example']) + try { + const report = await verifyDns() + assert.equal(report.ok, false) + assert.equal(detail(report, '_lexicon.example.com'), '2 did= records; a resolver requires exactly one') + } finally { + dns.setTxt('_lexicon.example.com', `did=${DID}`) + } + }) + + it('fails a domain that points at another DID', async () => { + dns.setTxt('_lexicon.nested.example.com', 'did=did:web:someone-else.example') + try { + const report = await verifyDns() + assert.equal(detail(report, '_lexicon.nested.example.com'), `points at did:web:someone-else.example, not ${DID}`) + } finally { + dns.setTxt('_lexicon.nested.example.com', `did=${DID}`) + } + }) + + it('names the lexicons a missing record strands', async () => { + dns.remove('_lexicon.example.com') + dns.remove('_lexicon.nested.example.com') + try { + const report = await verifyDns() + assert.equal( + detail(report, '_lexicon.example.com'), + 'no TXT record — 3 lexicons unresolvable: com.example.authFull, com.example.getThings, com.example.thing', + ) + assert.equal( + detail(report, '_lexicon.nested.example.com'), + 'no TXT record — 1 lexicon unresolvable: com.example.nested.widget', + ) + } finally { + for (const d of ['_lexicon.example.com', '_lexicon.nested.example.com']) dns.setTxt(d, `did=${DID}`) + } + }) + + it('reports the resolver error code when the lookup fails outright', async () => { + dns.setFailure('_lexicon.nested.example.com', 'servfail') + try { + const report = await verifyDns() + assert.equal(check(report, '_lexicon.nested.example.com')?.ok, false) + assert.match(detail(report, '_lexicon.nested.example.com'), /^(ESERVFAIL|EREFUSED)/) + } finally { + dns.setTxt('_lexicon.nested.example.com', `did=${DID}`) + } + }) + + it('skips the whole group on request', async () => { + const report = await verify({ skipDns: true }) + assert.equal(report.checks.some((c) => c.group === 'dns'), false) + }) +}) + +describe('verifyAuthority — cache keys', () => { + it('has nothing to compare when only one record is published', async () => { + const report = await verify({ world: single }) + assert.equal(check(report, 'sync.getRecord varies by rkey')?.ok, true) + assert.equal(detail(report, 'sync.getRecord varies by rkey'), 'only one record published; nothing to compare') + assert.equal(report.ok, true, JSON.stringify(failures(report))) + }) + + it('catches a cache keyed on path alone, which is the failure this command exists for', async () => { + const frozen = base.proofs[base.nsids[0]] + const report = await verify({ + intercept: (url) => + url.pathname.endsWith('sync.getRecord') + ? new Response(/** @type {BodyInit} */ (frozen), { headers: { 'content-type': CAR_CONTENT_TYPE } }) + : undefined, + }) + assert.equal(report.ok, false) + + const varies = check(report, 'sync.getRecord varies by rkey') + assert.equal(varies?.ok, false) + assert.match(varies?.detail ?? '', /returned identical bytes/) + assert.match(varies?.detail ?? '', /caching on path alone and dropping the query string/) + + const unknown = check(report, 'unknown rkey is refused') + assert.equal(unknown?.ok, false) + assert.match(unknown?.detail ?? '', /HTTP 200 with [\d,]+ bytes of body for a record that does not exist/) + }) +}) + +describe('verifyProof', () => { + /** @type {World} */ + let world + /** @type {World} */ + let malformed + /** @type {import('@atproto/crypto').Keypair} */ + let stranger + + before(async () => { + world = base + stranger = await Secp256k1Keypair.create() + malformed = await makeWorld({ + keypair: base.keypair, + writes: [ + { rkey: 'com.example.untyped', record: { lexicon: 1, id: 'com.example.untyped', defs: {} } }, + { rkey: 'com.example.renamed', record: { $type: COLLECTION, lexicon: 1, id: 'com.example.other', defs: {} } }, + ], + }) + }) + + /** @param {string} nsid */ + const expectFor = (nsid) => ({ did: DID, rkey: nsid, signingKey: base.signingKey }) + + it('returns the record, its CID, the commit root and the rev', async () => { + const out = await verifyProof(world.proofs['com.example.thing'], expectFor('com.example.thing')) + assert.equal(out.cid, world.values['com.example.thing'].cid) + assert.equal(out.root, world.root) + assert.equal(out.rev, REV) + assert.equal(out.record.$type, COLLECTION) + assert.equal(out.record.id, 'com.example.thing') + }) + + it('rejects bytes that are not a CAR', async () => { + await assert.rejects( + () => verifyProof(new Uint8Array([1, 2, 3, 4]), expectFor('com.example.thing')), + /^Error: proof is not a readable CAR: /, + ) + }) + + it('rejects an empty body', async () => { + await assert.rejects(() => verifyProof(new Uint8Array(0), expectFor('com.example.thing')), /not a readable CAR/) + }) + + it('rejects a CAR whose root is not a commit', async () => { + const { blocks } = await readCar(world.proofs['com.example.thing']) + const car = await blocksToCarFile(world.cids['com.example.thing'], blocks) + await assert.rejects(() => verifyProof(car, expectFor('com.example.thing')), /^Error: proof root is not a repo commit: /) + }) + + it('rejects a proof for another repo', async () => { + await assert.rejects( + () => verifyProof(world.proofs['com.example.thing'], { ...expectFor('com.example.thing'), did: 'did:web:elsewhere.example' }), + new Error(`proof commit claims repo ${DID}, not did:web:elsewhere.example`), + ) + }) + + it('rejects a commit signed by a key the document does not publish', async () => { + await assert.rejects( + () => verifyProof(world.proofs['com.example.thing'], { ...expectFor('com.example.thing'), signingKey: stranger.did() }), + new Error(`commit signature does not verify against the published key ${stranger.did()}`), + ) + }) + + it('rejects a proof that does not carry the requested record', async () => { + await assert.rejects( + () => verifyProof(world.proofs['com.example.thing'], expectFor('com.example.missing')), + new Error(`no ${COLLECTION}/com.example.missing under the signed MST root`), + ) + }) + + it('rejects a record with the wrong $type', async () => { + await assert.rejects( + () => verifyProof(malformed.proofs['com.example.untyped'], { ...expectFor('com.example.untyped') }), + new Error(`record $type is undefined, expected ${COLLECTION}`), + ) + }) + + it('rejects a record whose id is not the rkey it was published under', async () => { + await assert.rejects( + () => verifyProof(malformed.proofs['com.example.renamed'], { ...expectFor('com.example.renamed') }), + new Error('record id is com.example.other, expected com.example.renamed'), + ) + }) +}) + +describe('authorityDomain', () => { + it('reverses everything but the final segment', () => { + assert.equal(authorityDomain('com.example.thing'), '_lexicon.example.com') + assert.equal(authorityDomain('com.example.nested.widget'), '_lexicon.nested.example.com') + }) + + it('de-duplicates and sorts the domains a namespace needs', () => { + assert.deepEqual(authorityDomains(['com.example.thing', 'com.example.other', 'com.example.nested.widget']), [ + '_lexicon.example.com', + '_lexicon.nested.example.com', + ]) + assert.deepEqual(authorityDomains([]), []) + }) +}) diff --git a/test/worker/bundle-size.test.js b/test/worker/bundle-size.test.js index 0044468..790327f 100644 --- a/test/worker/bundle-size.test.js +++ b/test/worker/bundle-size.test.js @@ -66,6 +66,33 @@ test('a realistic namespace is a rounding error against the limit', (t) => { assert.ok(size.gzip < FREE_GZIP_LIMIT * 0.05, `gzipped bundle ${size.gzip} is above 5% of the free limit`) }) +// `data.js` must stay close to the content it encodes: the records as JSON, plus the byte +// payloads as base64. Carrying a second copy of every record — which the bundle did until +// `list` was replaced by `order` — pushes this to about 1.25. Measured at 1.02–1.05 today, +// so the gap between "encodes its content once" and "encodes it twice" is unambiguous. +const DUPLICATION_RATIO = 1.15 + +/** @param {ReturnType} data the base64 and JSON floor `data.js` cannot go below */ +function contentFloor(data) { + const payloads = Object.values(data.proofs).reduce((sum, b) => sum + b.byteLength, 0) + data.repoCar.byteLength + return JSON.stringify(data.records).length + Math.ceil(payloads / 3) * 4 +} + +test('the bundle encodes each record once, not once per view of it', (t) => { + for (const [label, opts] of /** @type {[string, Parameters[0]][]} */ ([ + ['atcr.io-scale', { nsids: Array.from({ length: 28 }, (_, i) => `io.example.real${i}`), bodyBytes: 1100, proofBytes: 2100, carBytes: 36_835 }], + ['worst case', { nsids: Array.from({ length: 28 }, (_, i) => `com.example.worstcase${i}`), bodyBytes: 3500, proofBytes: 5600, carBytes: 28 * 3800 }], + ])) { + const data = makeData(opts) + const ratio = renderDataModule(data).length / contentFloor(data) + t.diagnostic(`${label}: data.js is ${ratio.toFixed(3)}x the content it encodes`) + assert.ok( + ratio < DUPLICATION_RATIO, + `${label}: data.js is ${ratio.toFixed(3)}x its content, at or above the ${DUPLICATION_RATIO}x that means a record is embedded twice`, + ) + } +}) + test('a locally built dist/data.js fits', (t) => { const built = findBuiltDataModule() if (!built) { diff --git a/test/worker/helpers.js b/test/worker/helpers.js index 6763b3f..08e527c 100644 --- a/test/worker/helpers.js +++ b/test/worker/helpers.js @@ -84,7 +84,7 @@ export function makeData(opts = {}) { }, commit: { cid: commitCid, rev }, describeRepo: { did: DID, collections: [COLLECTION], handleIsCorrect: false }, - list: [{ records: nsids.map((n) => records[n]) }], + order: [...nsids], records, proofs, repoCar: pseudoBytes(carBytes, 99),