From 1debe42e532e08bafc16908b09854671ac67cdc8 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Thu, 17 Sep 2026 10:55:16 -0500 Subject: [PATCH 1/5] First pass at just 302 AND 304 --- __tests__/core_provider_contract.test.js | 3 +- __tests__/utils.test.js | 41 +++++ controllers/crud.js | 11 +- controllers/history.js | 34 +--- controllers/utils.js | 29 +++ db-controller.js | 3 +- openapi/contracts/core-provider.openapi.yaml | 23 ++- public/API.html | 59 +++--- routes/__tests__/query.test.js | 180 ++++++++++++++----- routes/__tests__/route_wrappers.test.js | 9 + routes/query.js | 1 - 11 files changed, 277 insertions(+), 116 deletions(-) diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js index 7d983a60..d03bb31c 100644 --- a/__tests__/core_provider_contract.test.js +++ b/__tests__/core_provider_contract.test.js @@ -204,13 +204,12 @@ const requiredResponseCodes = { 'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'], 'GET /since/{id}': ['200', '404'], 'GET /history/{id}': ['200', '404'], - // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss; + // HEAD parity tests in routes/__tests__/{id,since,history}.test.js assert 404 on miss; // enforce that the contract declares the same so drift on either side is caught. 'HEAD /id/{id}': ['200', '404'], 'HEAD /id/{id}/expanded': ['200', '404'], 'HEAD /since/{id}': ['200', '404'], 'HEAD /history/{id}': ['200', '404'], - 'HEAD /api/query': ['200', '404'], 'POST /api/query': ['200', '400', '413', '415'], 'POST /api/search': ['200', '400', '413', '415'], 'POST /api/search/phrase': ['200', '400', '413', '415'] diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js index 5a8421d6..e9098b81 100644 --- a/__tests__/utils.test.js +++ b/__tests__/utils.test.js @@ -10,6 +10,7 @@ import { _contextid, idNegotiation, getPagination, + setNextPageLink, findLeafAnnotationsFor } from '../controllers/utils.js' import { db, resetMocks, createCursor } from '../database/index.js' @@ -514,6 +515,46 @@ describe('controllers/utils.js getPagination', () => { } }) +describe('controllers/utils.js setNextPageLink', () => { + /** Run setNextPageLink against a response double and hand back what it appended to 'Link'. */ + const appendedFor = (page, req = { baseUrl: '/v1/api/query', path: '/' }) => { + const appended = [] + setNextPageLink(req, { append: (name, value) => appended.push([name, value]) }, page) + assert.ok(appended.every(([name]) => name === 'Link'), 'only Link is appended to, never set over') + return appended.map(([, value]) => value) + } + + /** Run a test body with RERUM_PREFIX set to a value, or unset for undefined, then restore it. */ + const withPrefix = (value, body) => { + const original = process.env.RERUM_PREFIX + try { + if (value === undefined) delete process.env.RERUM_PREFIX + else process.env.RERUM_PREFIX = value + body() + } finally { + if (original === undefined) delete process.env.RERUM_PREFIX + else process.env.RERUM_PREFIX = original + } + } + + it('links the next page on the deployment origin, for the path that was paged, advancing skip by the limit', () => { + withPrefix('https://store.example.org/v1/', () => { + assert.deepStrictEqual(appendedFor({ limit: 100, skip: 0, hasNext: true }), ['; rel="next"']) + assert.deepStrictEqual(appendedFor({ limit: 100, skip: 300, hasNext: true }), ['; rel="next"']) + }) + }) + + it('builds a path-only link when the deployment origin is not configured', () => { + withPrefix(undefined, () => { + assert.deepStrictEqual(appendedFor({ limit: 10, skip: 0, hasNext: true }, { baseUrl: '/v1/api', path: '/query' }), ['; rel="next"']) + }) + }) + + it('adds nothing on the final page', () => { + assert.deepStrictEqual(appendedFor({ limit: 100, skip: 200, hasNext: false }), []) + }) +}) + describe('controllers/utils.js findLeafAnnotationsFor', () => { const ENTITY_URI = 'https://store.rerum.io/v1/id/entity-id' const SLUG_URI = 'https://store.rerum.io/v1/id/entity-slug' diff --git a/controllers/crud.js b/controllers/crud.js index 88cd4c6d..93941a5c 100644 --- a/controllers/crud.js +++ b/controllers/crud.js @@ -6,7 +6,7 @@ */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' +import { _contextid, idNegotiation, getPagination, setNextPageLink, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js' /** * Create a new Linked Open Data object in RERUM v1. @@ -70,6 +70,8 @@ const create = async function (req, res, next) { * Query the MongoDB for objects containing the key:value pairs provided in the JSON Object in the request body. * This will support wildcards and mongo params like {"key":{$exists:true}} * The return is always an array, even if 0 or 1 objects in the return. + * + * Results page in ascending '_id' order. The Link header carries rel="next" while another page exists. * */ const query = async function (req, res, next) { res.set("Content-Type", "application/json; charset=utf-8") @@ -84,9 +86,12 @@ const query = async function (req, res, next) { } const { limit, skip } = getPagination(req.query, { res }) try { - let matches = await db.find(props).sort({ _id: 1 }).limit(limit).skip(skip).toArray() - matches = matches.map(o => idNegotiation(o)) + // One record past the page is read only to learn whether another page exists. It is never served. + let matches = await db.find(props).sort({ _id: 1 }).limit(limit + 1).skip(skip).toArray() + const hasNext = matches.length > limit + matches = matches.slice(0, limit).map(o => idNegotiation(o)) res.set(utils.configureLDHeadersFor(matches)) + setNextPageLink(req, res, { limit, skip, hasNext }) res.json(matches) } catch (error) { return next(utils.createExpressError(error)) diff --git a/controllers/history.js b/controllers/history.js index 2bfeed0d..c0de06e9 100644 --- a/controllers/history.js +++ b/controllers/history.js @@ -2,13 +2,13 @@ /** * History controller for RERUM operations - * Handles history, since, and HEAD request operations + * Handles history and since operations * @author cubap, thehabes */ import { newID, isValidID, db } from '../database/index.js' import utils from '../utils.js' -import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, getAllVersions, getAllAncestors, getAllDescendants } from './utils.js' +import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, getAllVersions, getAllAncestors, getAllDescendants } from './utils.js' /** * Public facing servlet to gather for all versions downstream from a provided `key object`. @@ -79,32 +79,4 @@ const history = async function (req, res, next) { res.json(ancestors) } -/** - * Allow for HEAD requests via the RERUM getByProperties pattern /v1/api/query - * No objects are returned, but the Content-Length header is set. - */ -const queryHeadRequest = async function (req, res, next) { - res.set("Content-Type", "application/json; charset=utf-8") - let props = req.body - const { limit, skip } = getPagination(req.query, { res }) - try { - // Sorted the same way POST /query is, so the two verbs page over one order. - const matches = await db.find(props).sort({ _id: 1 }).limit(limit).skip(skip).toArray() - if (matches.length) { - const negotiated = matches.map(o => idNegotiation(o)) - const size = Buffer.byteLength(JSON.stringify(negotiated)) - res.set("Content-Length", size) - res.status(200).end() - return - } - let err = { - "message": `There are no objects in the database matching the query. Check the request body.`, - "status": 404 - } - return next(utils.createExpressError(err)) - } catch (error) { - return next(utils.createExpressError(error)) - } -} - -export { since, history, queryHeadRequest } +export { since, history } diff --git a/controllers/utils.js b/controllers/utils.js index e0fc4f26..7be3e4fd 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -134,6 +134,34 @@ function getPagination(query = {}, options = {}) { return { limit, skip } } +/** + * Report an RFC 8288 rel="next" link while another page exists, alongside any Link the response + * already carries. Its absence is what ends a walk, so nothing is added on the final page. + * + * 'next' advances 'skip' even when that passes the configured maximum. Following it is then answered + * with the 400 that names the maximum, so a walk too deep to finish ends loudly instead of looking + * complete. + * + * The link carries only paging parameters. The request body is not part of it, so a client sends + * the same body to it. The origin comes from the deployment's RERUM_PREFIX, never from the request. + * Call this after any res.set() of 'Link', which would replace it. + * + * @param req The Express request, for the path that was paged. + * @param res The Express response. + * @param page What was served. + * @param page.limit The applied limit. + * @param page.skip The applied skip. + * @param page.hasNext Whether any record lies past this page. + */ +function setNextPageLink(req, res, { limit, skip = 0, hasNext = false } = {}) { + if (!hasNext) return + // A route mounted at '/' leaves a trailing slash on the joined path. + const path = `${req.baseUrl ?? ""}${req.path ?? ""}`.replace(/(.)\/+$/, "$1") + const target = `${path}?${new URLSearchParams({ limit, skip: skip + limit })}` + const base = process.env.RERUM_PREFIX + res.append("Link", `<${URL.canParse(target, base) ? new URL(target, base).href : target}>; rel="next"`) +} + /** * Check if a @context value contains a known @id-id mapping context * @@ -694,6 +722,7 @@ export { findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS, getPagination, + setNextPageLink, generateSlugId, index, ObjectID, diff --git a/db-controller.js b/db-controller.js index 99f5c163..81d86092 100644 --- a/db-controller.js +++ b/db-controller.js @@ -13,7 +13,7 @@ import { searchAsWords, searchAsPhrase } from './controllers/search.js' import { deleteObj } from './controllers/delete.js' import { putUpdate, patchUpdate, patchSet, patchUnset, overwrite } from './controllers/update.js' import { bulkCreate, bulkUpdate } from './controllers/bulk.js' -import { since, history, queryHeadRequest } from './controllers/history.js' +import { since, history } from './controllers/history.js' import { release } from './controllers/release.js' import { _gog_fragments_from_manuscript, _gog_glosses_from_manuscript, expand, expandedId } from './controllers/gog.js' @@ -35,7 +35,6 @@ export default { idExpanded, bulkCreate, bulkUpdate, - queryHeadRequest, since, history, remove, diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index b5464689..4c6c216d 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -220,6 +220,9 @@ paths: responses: '200': description: Query results + headers: + Link: + $ref: '#/components/headers/NextPageLink' content: application/json: schema: @@ -230,14 +233,6 @@ paths: $ref: '#/components/responses/PayloadTooLarge' '415': $ref: '#/components/responses/UnsupportedMediaType' - head: - summary: Query object headers - operationId: headQueryObjects - responses: - '200': - description: Query result headers - '404': - $ref: '#/components/responses/NotFound' /api/search: post: summary: Search objects by keywords @@ -689,6 +684,18 @@ components: never been overwritten. schema: type: string + NextPageLink: + description: >- + RFC 8288 links. The JSON-LD context link, plus rel="next" only while another page exists, so + its absence ends a walk. rel="next" advances 'skip', and is still present when the next page + would start past the 'skip' maximum, where following it is answered with the 400 that names + the maximum. The link carries only paging URL parameters, so send the same request body to + it. + schema: + type: string + example: >- + ; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json", + ; rel="next" parameters: ObjectId: in: path diff --git a/public/API.html b/public/API.html index a598ebdc..4ff89c25 100644 --- a/public/API.html +++ b/public/API.html @@ -504,7 +504,7 @@

Custom Query

This simple format will be made more complex in the future, but should serve the basic needs as it is. - RERUM will test for property matches. By default, this is limited to 100 records in the response to guard against unreasonable queries. To allow for more records in the response one can add the URL parameter limit to the query requests. If you expect the query request will have a very large response with many objects, your application should use a paged query by also using the skip URL parameter. You will see an example of this below. + RERUM will test for property matches. By default, this is limited to 100 records in the response to guard against unreasonable queries. To allow for more records in the response one can add the URL parameter limit to the query requests. If you expect the query request will have a very large response with many objects, your application should use a paged query that follows the rel="next" link in each response's Link header until there is none. You will see an example of this below.

A limit above the maximum of 500 is not an error. It is reduced to 500, and the response says so — see Pagination parameters. Read the applied value from the Pagination-Limit response header rather than assuming you received everything you asked for.

@@ -551,27 +551,30 @@

Custom Query

Paged Query Javascript Example
 
-                const many_results = await pagedQuery(100, 0, {"type": "Thing"})
+                const many_results = await pagedQuery({"type": "Thing"})
                 

-                async function pagedQuery(lim, it = 0, queryObj, allResults = []) {
-                    const response = await fetch(`https://devstore.rerum.io/v1/api/query?limit=${lim}&skip=${it}`, {
-                         method: "POST",
-                        headers: {
-                            "Content-Type": "application/json; charset=utf-8"
-                        },
+                async function pagedQuery(queryObj, lim = 100) {
+                    // Every page is requested with the same body.
+                    const request = {
+                        method: "POST",
+                        headers: { "Content-Type": "application/json; charset=utf-8" },
                         body: JSON.stringify(queryObj)
-                    })
-                    const skipMax = Number(response.headers.get("Pagination-Skip-Max"))
-                    if (response.status === 400 && skipMax && it > skipMax) {
-                        console.warn(`Stopped at the skip ceiling of ${skipMax}. These results are incomplete. Narrow the query.`)
-                        return allResults
                     }
-                    if (!response.ok) throw new Error(await response.text())
-                    const results = await response.json()
-                    allResults.push(...results)
-                    const appliedLimit = Number(response.headers.get("Pagination-Limit")) || lim
-                    if (results.length < appliedLimit) return allResults
-                    return pagedQuery(lim, it + appliedLimit, queryObj, allResults)
+                    const allResults = []
+                    let url = `https://devstore.rerum.io/v1/api/query?limit=${lim}`
+                    while (url) {
+                        const response = await fetch(url, request)
+                        // A next link past the skip maximum is answered with a 400 that names the maximum.
+                        if (response.status === 400 && allResults.length) {
+                            console.warn(`Stopped at the skip maximum of ${response.headers.get("Pagination-Skip-Max")}. These results are incomplete. Narrow the query.`)
+                            return allResults
+                        }
+                        if (!response.ok) throw new Error(await response.text())
+                        allResults.push(...await response.json())
+                        // No next link means there are no more records.
+                        url = response.headers.get("Link")?.match(/<([^>]+)>;\s*rel="next"/)?.[1]
+                    }
+                    return allResults
                 }

@@ -604,7 +607,7 @@

Pagination parameters

- Every paged response reports what was applied and what the maximums are, so a client can configure itself from any single response and tell a truncated page from a genuine final one. These headers are readable cross-origin. + Every paged response reports what was applied and what the maximums are, so a client can configure itself from any single response. These headers are readable cross-origin.

Pagination-Limit-Max and Pagination-Skip-Max are reported on rejections too, including the 400 for a skip past the maximum. That is what lets a paged walk recognise the depth boundary and stop cleanly. @@ -617,9 +620,23 @@

Pagination parameters

Pagination-Skip-Max: 100000

+

Following rel="next"

+

+ While another page exists, Custom Query responses carry a rel="next" link in the Link header (RFC 8288), next to the JSON-LD context link. The final page has no rel="next". Follow it until it is gone. Do not decide you are done by counting records, because a short page is not always the last one. +

+

+ rel="next" advances skip. It is still present when the next page would start past the skip maximum, and following it is answered with the 400 that names the maximum. A walk longer than the maximum ends in that 400 rather than looking complete. +

+

Custom Query takes a POST. The link only carries the paging URL parameters, so send the same request body to it that you sent to the first page.

+

+ 

 
+                Link: <http://store.rerum.io/v1/context.json>; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json"
+                Link: <https://store.rerum.io/v1/api/query?limit=100&skip=200>; rel="next"
+
+

Result ordering

- Custom Query pages in ascending _id order, on both POST and HEAD. A skip offset only means something over a fixed order, so consecutive pages tile that order without repeating or missing a record. + Custom Query pages in ascending _id order. A skip offset only means something over a fixed order, so consecutive pages tile that order without repeating or missing a record.

diff --git a/routes/__tests__/query.test.js b/routes/__tests__/query.test.js index f4bad932..a7970cfa 100644 --- a/routes/__tests__/query.test.js +++ b/routes/__tests__/query.test.js @@ -11,9 +11,6 @@ const routeTester = new express() routeTester.use(express.json({ type: ["application/json", "application/ld+json"] })) // Mount our own /query route without auth that will use controller.query (POST) -// and controller.queryHeadRequest (HEAD). Order matters: `.head()` must precede -// `.use()` so the method-agnostic catch-all does not steal HEAD requests. -routeTester.head("/query", controller.queryHeadRequest) routeTester.use("/query", controller.query) const MOCK_AGENT = "https://store.rerum.io/v1/id/agent007" @@ -68,47 +65,9 @@ it("'/query' route functions", async () => { assert.strictEqual(response.body[0]._id, undefined) }) -describe('HEAD /query', () => { - const buildCursor = (docs) => ({ - sort() { return this }, - limit() { return this }, - skip() { return this }, - async toArray() { return docs } - }) - - // The Content-Length parity check only depends on what db.find returns — - // the body the controller would have produced is the same whether the query - // filter has 0 or N keys. supertest cannot send a JSON body on HEAD cleanly - // (superagent rejects the object), so the request body is omitted here. - - it("returns 200 with Content-Length matching the POST body length", async () => { - db.find.mockReturnValueOnce(buildCursor([mockDoc])) - const postResp = await request(routeTester) - .post("/query") - .set("Content-Type", "application/json") - .send({ test: "item" }) - const postLen = Number(postResp.headers['content-length']) - - db.find.mockReturnValueOnce(buildCursor([mockDoc])) - const headResp = await request(routeTester).head("/query") - - assert.strictEqual(headResp.statusCode, 200) - assert.ok(postLen > 0, 'POST must report a Content-Length') - assert.strictEqual(Number(headResp.headers['content-length']), postLen) - assert.ok(!headResp.body || Object.keys(headResp.body).length === 0) - }) - - it("returns 404 when no matches are found", async () => { - db.find.mockReturnValueOnce(buildCursor([])) - const response = await request(routeTester).head("/query") - assert.strictEqual(response.statusCode, 404) - }) -}) - describe('pagination parameters on /query', () => { const pagedTester = express() pagedTester.use(express.json({ type: ["application/json", "application/ld+json"] })) - pagedTester.head("/query", controller.queryHeadRequest) pagedTester.post("/query", rest.verifyJsonContentType, controller.query) pagedTester.use(rest.messenger) @@ -165,7 +124,8 @@ describe('pagination parameters on /query', () => { .set("Content-Type", "application/json") .send({ test: "item" }) - assert.strictEqual(applied.limit, 7) + // One record past the page is read to learn whether another page exists. + assert.strictEqual(applied.limit, 8) assert.strictEqual(applied.skip, 3) // A skip offset means nothing without a stable order to count into. assert.deepStrictEqual(applied.sort, { _id: 1 }) @@ -200,12 +160,6 @@ describe('pagination parameters on /query', () => { } }) - it("rejects the same values on HEAD /query", async () => { - db.find.mockReturnValueOnce(recordingCursor([mockDoc], {})) - const response = await request(pagedTester).head("/query?limit=abc") - assert.strictEqual(response.statusCode, 400) - }) - it("answers an unacceptable Content-Type before it reads a pagination parameter", async () => { // Both faults are present. The Content-Type is the one the endpoint can answer without // looking at the query string, so it is the one that should decide the status. @@ -221,3 +175,133 @@ describe('pagination parameters on /query', () => { } }) }) + +describe('rel="next" links on /query', () => { + const pagedTester = express() + pagedTester.use(express.json({ type: ["application/json", "application/ld+json"] })) + pagedTester.post("/query", rest.verifyJsonContentType, controller.query) + pagedTester.use(rest.messenger) + + /** A document in the shape the controller reads, whose '@id' ends in its '_id'. */ + const docWithId = (_id) => ({ ...mockDoc, _id, "@id": `${MOCK_PREFIX}${_id}` }) + + /** The document ids of a /query response, in the order the endpoint returned them. */ + const idsOf = (response) => response.body.map(o => o["@id"].split("/").pop()) + + /** The Link header of a response as { rel: url }. */ + const linksOf = (response) => Object.fromEntries( + [...(response.headers.link ?? "").matchAll(/<([^>]+)>;\s*rel="([^"]+)"/g)].map(([, url, rel]) => [rel, url]) + ) + + /** The query string of a link target, which may be absolute or path-only. */ + const paramsOf = (url) => new URL(url, "http://localhost").searchParams + + /** + * A find() double over a fixed, '_id' ordered set of documents. It honours the applied skip and + * limit, so a walk over it pages the way the collection would. + */ + const collectionOf = (docs) => () => { + const window = { skip: 0, limit: Infinity } + return { + sort() { return this }, + limit(n) { + window.limit = n + return this + }, + skip(n) { + window.skip = n + return this + }, + async toArray() { + return docs + .slice(window.skip, window.skip + window.limit) + .map(doc => structuredClone(doc)) + } + } + } + + const post = (path, body = { test: "item" }) => request(pagedTester) + .post(path) + .set("Content-Type", "application/json") + .send(body) + + it("links the next page while more records exist, and serves only the page", async () => { + db.find.mockImplementation(collectionOf(["a", "b", "c"].map(docWithId))) + const response = await post("/query?limit=2") + const links = linksOf(response) + + assert.strictEqual(response.statusCode, 200) + assert.deepStrictEqual(idsOf(response), ["a", "b"]) + assert.ok(links.next, "a next link is present while more records exist") + assert.strictEqual(paramsOf(links.next).get("limit"), "2") + assert.strictEqual(paramsOf(links.next).get("skip"), "2") + }) + + it("keeps the JSON-LD context link alongside the next link", async () => { + db.find.mockImplementation(collectionOf(["a", "b", "c"].map(docWithId))) + const links = linksOf(await post("/query?limit=2")) + assert.ok(links["http://www.w3.org/ns/json-ld#context"]) + assert.ok(links.next) + }) + + it("omits next on the final page", async () => { + db.find.mockImplementation(collectionOf(["a", "b"].map(docWithId))) + const response = await post("/query?limit=2") + const links = linksOf(response) + + assert.deepStrictEqual(idsOf(response), ["a", "b"]) + assert.deepStrictEqual(Object.keys(links), ["http://www.w3.org/ns/json-ld#context"], "only the JSON-LD context link remains") + }) + + it("never serves the record read past the page, even at the maximum limit", async () => { + db.find.mockImplementation(collectionOf([docWithId("a")])) + const limitMax = Number((await post("/query?limit=1")).headers['pagination-limit-max']) + const ids = Array.from({ length: limitMax + 1 }, (_, i) => `id${String(i).padStart(6, "0")}`) + db.find.mockImplementation(collectionOf(ids.map(docWithId))) + + const response = await post(`/query?limit=${limitMax + 100}`) + + assert.strictEqual(response.body.length, limitMax) + assert.ok(!idsOf(response).includes(ids.at(-1)), "the extra record must not be served") + assert.ok(linksOf(response).next) + }) + + it("still links next past the skip maximum, so a walk too deep to finish ends in the 400 that names it", async () => { + db.find.mockImplementation(collectionOf([docWithId("a")])) + const skipMax = Number((await post("/query?limit=1")).headers['pagination-skip-max']) + // Every page this double serves has a record past it, however deep the skip. + db.find.mockImplementation(() => ({ + sort() { return this }, + limit() { return this }, + skip() { return this }, + async toArray() { return ["a", "b", "c"].map(docWithId) } + })) + + const deepest = await post(`/query?limit=2&skip=${skipMax}`) + const next = linksOf(deepest).next + assert.strictEqual(deepest.statusCode, 200) + assert.strictEqual(paramsOf(next).get("skip"), String(skipMax + 2), "absence would tell the client the walk was complete") + + const beyond = await post(`/query?${paramsOf(next)}`) + assert.strictEqual(beyond.statusCode, 400) + assert.strictEqual(beyond.headers['pagination-skip-max'], String(skipMax)) + }) + + it("walks every record exactly once by following only rel=\"next\"", async () => { + const ids = ["d01", "d02", "d03", "d04", "d05", "d06", "d07"] + db.find.mockImplementation(collectionOf(ids.map(docWithId))) + + const walked = [] + let path = "/query?limit=3" + for (let requests = 0; path; requests++) { + assert.ok(requests < ids.length, "the walk must terminate") + const response = await post(path) + assert.strictEqual(response.statusCode, 200) + walked.push(...idsOf(response)) + const next = linksOf(response).next + path = next && `/query?${paramsOf(next)}` + } + + assert.deepStrictEqual(walked, ids) + }) +}) diff --git a/routes/__tests__/route_wrappers.test.js b/routes/__tests__/route_wrappers.test.js index c0bb1300..e6e92346 100644 --- a/routes/__tests__/route_wrappers.test.js +++ b/routes/__tests__/route_wrappers.test.js @@ -219,6 +219,15 @@ describe('unsupported-method 405 fallbacks', () => { assertUnsupportedMethodOnPath(router, path) }) } + + // HEAD is GET without content. A route with no GET has nothing for HEAD to mirror, so HEAD must + // fall through to the 405 fallback rather than reach a handler of its own. + for (const { label, router, path } of cases) { + if (getMethodLayers(router, path, 'get').length) continue + it(`has no HEAD handler of its own for ${label}`, () => { + assert.deepStrictEqual(getMethodLayers(router, path, 'head'), []) + }) + } }) describe('api routes discovery', () => { diff --git a/routes/query.js b/routes/query.js index b9882b82..15f0ddc5 100644 --- a/routes/query.js +++ b/routes/query.js @@ -6,7 +6,6 @@ import rest from '../rest.js' router.route('/') .post(rest.verifyJsonContentType, controller.query) - .head(controller.queryHeadRequest) .all((req, res, next) => { res.statusMessage = 'Improper request method for requesting objects with matching properties. Please use POST.' res.status(405).end() From 6f976425d7d3bf013c5421b7babc8d1159cafcb4 Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Thu, 17 Sep 2026 11:06:11 -0500 Subject: [PATCH 2/5] touch up from review --- controllers/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/controllers/utils.js b/controllers/utils.js index 7be3e4fd..43f76747 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -94,8 +94,8 @@ function readWholeNumberParam(raw, name, fallback, min) { * response readable. A 'skip' above its maximum is rejected, because clamping it would serve the * page at the maximum over and over. * - * The applied values and both maximums are reported in the response headers, so a client can tell - * a truncated page from a genuine final one and can configure itself from any single response. + * The applied values and both maximums are reported in the response headers, so a client can + * configure itself from any single response. * * @param query The Express 'req.query' object. * @param options Optional settings for this endpoint. From 98b4c361ada00d4d5d412cffa5e72fa0e09addea Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Thu, 17 Sep 2026 11:19:19 -0500 Subject: [PATCH 3/5] comment cleanup, deploy to dev for testing --- controllers/utils.js | 4 ---- openapi/contracts/core-provider.openapi.yaml | 5 +---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/controllers/utils.js b/controllers/utils.js index 43f76747..219d9ed2 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -142,10 +142,6 @@ function getPagination(query = {}, options = {}) { * with the 400 that names the maximum, so a walk too deep to finish ends loudly instead of looking * complete. * - * The link carries only paging parameters. The request body is not part of it, so a client sends - * the same body to it. The origin comes from the deployment's RERUM_PREFIX, never from the request. - * Call this after any res.set() of 'Link', which would replace it. - * * @param req The Express request, for the path that was paged. * @param res The Express response. * @param page What was served. diff --git a/openapi/contracts/core-provider.openapi.yaml b/openapi/contracts/core-provider.openapi.yaml index 4c6c216d..fb37d317 100644 --- a/openapi/contracts/core-provider.openapi.yaml +++ b/openapi/contracts/core-provider.openapi.yaml @@ -687,10 +687,7 @@ components: NextPageLink: description: >- RFC 8288 links. The JSON-LD context link, plus rel="next" only while another page exists, so - its absence ends a walk. rel="next" advances 'skip', and is still present when the next page - would start past the 'skip' maximum, where following it is answered with the 400 that names - the maximum. The link carries only paging URL parameters, so send the same request body to - it. + its absence ends a walk. schema: type: string example: >- From 13da5c6d6f5e5b34b12ae82c54325bdc55515bfc Mon Sep 17 00:00:00 2001 From: Bryan Haberberger Date: Thu, 17 Sep 2026 11:30:49 -0500 Subject: [PATCH 4/5] touch ups --- public/API.html | 7 ++----- routes/__tests__/route_wrappers.test.js | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/public/API.html b/public/API.html index 4ff89c25..5ff50dc6 100644 --- a/public/API.html +++ b/public/API.html @@ -554,7 +554,6 @@

Custom Query

const many_results = await pagedQuery({"type": "Thing"})
async function pagedQuery(queryObj, lim = 100) { - // Every page is requested with the same body. const request = { method: "POST", headers: { "Content-Type": "application/json; charset=utf-8" }, @@ -564,14 +563,12 @@

Custom Query

let url = `https://devstore.rerum.io/v1/api/query?limit=${lim}` while (url) { const response = await fetch(url, request) - // A next link past the skip maximum is answered with a 400 that names the maximum. if (response.status === 400 && allResults.length) { console.warn(`Stopped at the skip maximum of ${response.headers.get("Pagination-Skip-Max")}. These results are incomplete. Narrow the query.`) return allResults } if (!response.ok) throw new Error(await response.text()) allResults.push(...await response.json()) - // No next link means there are no more records. url = response.headers.get("Link")?.match(/<([^>]+)>;\s*rel="next"/)?.[1] } return allResults @@ -622,10 +619,10 @@

Pagination parameters

Following rel="next"

- While another page exists, Custom Query responses carry a rel="next" link in the Link header (RFC 8288), next to the JSON-LD context link. The final page has no rel="next". Follow it until it is gone. Do not decide you are done by counting records, because a short page is not always the last one. + While another page exists, Custom Query responses carry a rel="next" link in the Link header (RFC 8288), next to the JSON-LD context link. The final page has no rel="next". Follow it until it is gone.

- rel="next" advances skip. It is still present when the next page would start past the skip maximum, and following it is answered with the 400 that names the maximum. A walk longer than the maximum ends in that 400 rather than looking complete. + rel="next" advances skip. It is still present when the next page would start past the skip maximum, and following it is answered with the 400 that names the maximum.

Custom Query takes a POST. The link only carries the paging URL parameters, so send the same request body to it that you sent to the first page.

diff --git a/routes/__tests__/route_wrappers.test.js b/routes/__tests__/route_wrappers.test.js index e6e92346..492ed898 100644 --- a/routes/__tests__/route_wrappers.test.js +++ b/routes/__tests__/route_wrappers.test.js @@ -220,8 +220,7 @@ describe('unsupported-method 405 fallbacks', () => { }) } - // HEAD is GET without content. A route with no GET has nothing for HEAD to mirror, so HEAD must - // fall through to the 405 fallback rather than reach a handler of its own. + // HEAD will fall through to the 405 fallback rather than reach a handler of its own. for (const { label, router, path } of cases) { if (getMethodLayers(router, path, 'get').length) continue it(`has no HEAD handler of its own for ${label}`, () => { From 0d055df4850be46fc3caf89c37c2521aa673de31 Mon Sep 17 00:00:00 2001 From: cubap Date: Thu, 24 Sep 2026 12:13:29 -0500 Subject: [PATCH 5/5] fix: drop HEAD from Allow header on /query 405 PR #295 added the sendMethodNotAllowed helper with 'POST,HEAD' on main. This PR removes the HEAD handler, so the Allow header must advertise only POST per RFC 9110 section 15.5.6. Merge main in to keep the branch conflict-free. --- routes/query.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/query.js b/routes/query.js index d305f73c..e2af9691 100644 --- a/routes/query.js +++ b/routes/query.js @@ -7,7 +7,7 @@ import rest from '../rest.js' router.route('/') .post(rest.verifyJsonContentType, controller.query) .all((req, res, next) => { - rest.sendMethodNotAllowed(res, 'Improper request method for requesting objects with matching properties. Please use POST.', 'POST,HEAD') + rest.sendMethodNotAllowed(res, 'Improper request method for requesting objects with matching properties. Please use POST.', 'POST') }) export default router