diff --git a/__tests__/core_provider_contract.test.js b/__tests__/core_provider_contract.test.js
index 7d983a60..d03bb31c 100644
--- a/__tests__/core_provider_contract.test.js
+++ b/__tests__/core_provider_contract.test.js
@@ -204,13 +204,12 @@ const requiredResponseCodes = {
'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
'GET /since/{id}': ['200', '404'],
'GET /history/{id}': ['200', '404'],
- // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss;
+ // HEAD parity tests in routes/__tests__/{id,since,history}.test.js assert 404 on miss;
// enforce that the contract declares the same so drift on either side is caught.
'HEAD /id/{id}': ['200', '404'],
'HEAD /id/{id}/expanded': ['200', '404'],
'HEAD /since/{id}': ['200', '404'],
'HEAD /history/{id}': ['200', '404'],
- 'HEAD /api/query': ['200', '404'],
'POST /api/query': ['200', '400', '413', '415'],
'POST /api/search': ['200', '400', '413', '415'],
'POST /api/search/phrase': ['200', '400', '413', '415']
diff --git a/__tests__/utils.test.js b/__tests__/utils.test.js
index 5a8421d6..e9098b81 100644
--- a/__tests__/utils.test.js
+++ b/__tests__/utils.test.js
@@ -10,6 +10,7 @@ import {
_contextid,
idNegotiation,
getPagination,
+ setNextPageLink,
findLeafAnnotationsFor
} from '../controllers/utils.js'
import { db, resetMocks, createCursor } from '../database/index.js'
@@ -514,6 +515,46 @@ describe('controllers/utils.js getPagination', () => {
}
})
+describe('controllers/utils.js setNextPageLink', () => {
+ /** Run setNextPageLink against a response double and hand back what it appended to 'Link'. */
+ const appendedFor = (page, req = { baseUrl: '/v1/api/query', path: '/' }) => {
+ const appended = []
+ setNextPageLink(req, { append: (name, value) => appended.push([name, value]) }, page)
+ assert.ok(appended.every(([name]) => name === 'Link'), 'only Link is appended to, never set over')
+ return appended.map(([, value]) => value)
+ }
+
+ /** Run a test body with RERUM_PREFIX set to a value, or unset for undefined, then restore it. */
+ const withPrefix = (value, body) => {
+ const original = process.env.RERUM_PREFIX
+ try {
+ if (value === undefined) delete process.env.RERUM_PREFIX
+ else process.env.RERUM_PREFIX = value
+ body()
+ } finally {
+ if (original === undefined) delete process.env.RERUM_PREFIX
+ else process.env.RERUM_PREFIX = original
+ }
+ }
+
+ it('links the next page on the deployment origin, for the path that was paged, advancing skip by the limit', () => {
+ withPrefix('https://store.example.org/v1/', () => {
+ assert.deepStrictEqual(appendedFor({ limit: 100, skip: 0, hasNext: true }), ['
This simple format will be made more complex in the future, but should serve the basic needs as it is.
- RERUM will test for property matches. By default, this is limited to 100 records in the response to guard against unreasonable queries. To allow for more records in the response one can add the URL parameter A Custom Query
limit to the query requests. If you expect the query request will have a very large response with many objects, your application should use a paged query by also using the skip URL parameter. You will see an example of this below.
+ RERUM will test for property matches. By default, this is limited to 100 records in the response to guard against unreasonable queries. To allow for more records in the response one can add the URL parameter limit to the query requests. If you expect the query request will have a very large response with many objects, your application should use a paged query that follows the rel="next" link in each response's Link header until there is none. You will see an example of this below.
limit above the maximum of 500 is not an error. It is reduced to 500, and the response says so — see Pagination parameters. Read the applied value from the Pagination-Limit response header rather than assuming you received everything you asked for.
@@ -551,27 +551,27 @@
- const many_results = await pagedQuery(100, 0, {"type": "Thing"})
+ const many_results = await pagedQuery({"type": "Thing"})
- async function pagedQuery(lim, it = 0, queryObj, allResults = []) {
- const response = await fetch(`https://devstore.rerum.io/v1/api/query?limit=${lim}&skip=${it}`, {
- method: "POST",
- headers: {
- "Content-Type": "application/json; charset=utf-8"
- },
+ async function pagedQuery(queryObj, lim = 100) {
+ const request = {
+ method: "POST",
+ headers: { "Content-Type": "application/json; charset=utf-8" },
body: JSON.stringify(queryObj)
- })
- const skipMax = Number(response.headers.get("Pagination-Skip-Max"))
- if (response.status === 400 && skipMax && it > skipMax) {
- console.warn(`Stopped at the skip ceiling of ${skipMax}. These results are incomplete. Narrow the query.`)
- return allResults
}
- if (!response.ok) throw new Error(await response.text())
- const results = await response.json()
- allResults.push(...results)
- const appliedLimit = Number(response.headers.get("Pagination-Limit")) || lim
- if (results.length < appliedLimit) return allResults
- return pagedQuery(lim, it + appliedLimit, queryObj, allResults)
+ const allResults = []
+ let url = `https://devstore.rerum.io/v1/api/query?limit=${lim}`
+ while (url) {
+ const response = await fetch(url, request)
+ if (response.status === 400 && allResults.length) {
+ console.warn(`Stopped at the skip maximum of ${response.headers.get("Pagination-Skip-Max")}. These results are incomplete. Narrow the query.`)
+ return allResults
+ }
+ if (!response.ok) throw new Error(await response.text())
+ allResults.push(...await response.json())
+ url = response.headers.get("Link")?.match(/<([^>]+)>;\s*rel="next"/)?.[1]
+ }
+ return allResults
}
@@ -604,7 +604,7 @@ - Every paged response reports what was applied and what the maximums are, so a client can configure itself from any single response and tell a truncated page from a genuine final one. These headers are readable cross-origin. + Every paged response reports what was applied and what the maximums are, so a client can configure itself from any single response. These headers are readable cross-origin.
Pagination-Limit-Max and Pagination-Skip-Max are reported on rejections too, including the 400 for a skip past the maximum. That is what lets a paged walk recognise the depth boundary and stop cleanly.
@@ -617,9 +617,23 @@
rel="next"
+ While another page exists, Custom Query responses carry a rel="next" link in the Link header (RFC 8288), next to the JSON-LD context link. The final page has no rel="next". Follow it until it is gone.
+
+ rel="next" advances skip. It is still present when the next page would start past the skip maximum, and following it is answered with the 400 that names the maximum.
+
Custom Query takes a POST. The link only carries the paging URL parameters, so send the same request body to it that you sent to the first page.
+
+ Link: <http://store.rerum.io/v1/context.json>; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json"
+ Link: <https://store.rerum.io/v1/api/query?limit=100&skip=200>; rel="next"
+
+
- Custom Query pages in ascending _id order, on both POST and HEAD. A skip offset only means something over a fixed order, so consecutive pages tile that order without repeating or missing a record.
+ Custom Query pages in ascending _id order. A skip offset only means something over a fixed order, so consecutive pages tile that order without repeating or missing a record.
diff --git a/routes/__tests__/query.test.js b/routes/__tests__/query.test.js index f4bad932..a7970cfa 100644 --- a/routes/__tests__/query.test.js +++ b/routes/__tests__/query.test.js @@ -11,9 +11,6 @@ const routeTester = new express() routeTester.use(express.json({ type: ["application/json", "application/ld+json"] })) // Mount our own /query route without auth that will use controller.query (POST) -// and controller.queryHeadRequest (HEAD). Order matters: `.head()` must precede -// `.use()` so the method-agnostic catch-all does not steal HEAD requests. -routeTester.head("/query", controller.queryHeadRequest) routeTester.use("/query", controller.query) const MOCK_AGENT = "https://store.rerum.io/v1/id/agent007" @@ -68,47 +65,9 @@ it("'/query' route functions", async () => { assert.strictEqual(response.body[0]._id, undefined) }) -describe('HEAD /query', () => { - const buildCursor = (docs) => ({ - sort() { return this }, - limit() { return this }, - skip() { return this }, - async toArray() { return docs } - }) - - // The Content-Length parity check only depends on what db.find returns — - // the body the controller would have produced is the same whether the query - // filter has 0 or N keys. supertest cannot send a JSON body on HEAD cleanly - // (superagent rejects the object), so the request body is omitted here. - - it("returns 200 with Content-Length matching the POST body length", async () => { - db.find.mockReturnValueOnce(buildCursor([mockDoc])) - const postResp = await request(routeTester) - .post("/query") - .set("Content-Type", "application/json") - .send({ test: "item" }) - const postLen = Number(postResp.headers['content-length']) - - db.find.mockReturnValueOnce(buildCursor([mockDoc])) - const headResp = await request(routeTester).head("/query") - - assert.strictEqual(headResp.statusCode, 200) - assert.ok(postLen > 0, 'POST must report a Content-Length') - assert.strictEqual(Number(headResp.headers['content-length']), postLen) - assert.ok(!headResp.body || Object.keys(headResp.body).length === 0) - }) - - it("returns 404 when no matches are found", async () => { - db.find.mockReturnValueOnce(buildCursor([])) - const response = await request(routeTester).head("/query") - assert.strictEqual(response.statusCode, 404) - }) -}) - describe('pagination parameters on /query', () => { const pagedTester = express() pagedTester.use(express.json({ type: ["application/json", "application/ld+json"] })) - pagedTester.head("/query", controller.queryHeadRequest) pagedTester.post("/query", rest.verifyJsonContentType, controller.query) pagedTester.use(rest.messenger) @@ -165,7 +124,8 @@ describe('pagination parameters on /query', () => { .set("Content-Type", "application/json") .send({ test: "item" }) - assert.strictEqual(applied.limit, 7) + // One record past the page is read to learn whether another page exists. + assert.strictEqual(applied.limit, 8) assert.strictEqual(applied.skip, 3) // A skip offset means nothing without a stable order to count into. assert.deepStrictEqual(applied.sort, { _id: 1 }) @@ -200,12 +160,6 @@ describe('pagination parameters on /query', () => { } }) - it("rejects the same values on HEAD /query", async () => { - db.find.mockReturnValueOnce(recordingCursor([mockDoc], {})) - const response = await request(pagedTester).head("/query?limit=abc") - assert.strictEqual(response.statusCode, 400) - }) - it("answers an unacceptable Content-Type before it reads a pagination parameter", async () => { // Both faults are present. The Content-Type is the one the endpoint can answer without // looking at the query string, so it is the one that should decide the status. @@ -221,3 +175,133 @@ describe('pagination parameters on /query', () => { } }) }) + +describe('rel="next" links on /query', () => { + const pagedTester = express() + pagedTester.use(express.json({ type: ["application/json", "application/ld+json"] })) + pagedTester.post("/query", rest.verifyJsonContentType, controller.query) + pagedTester.use(rest.messenger) + + /** A document in the shape the controller reads, whose '@id' ends in its '_id'. */ + const docWithId = (_id) => ({ ...mockDoc, _id, "@id": `${MOCK_PREFIX}${_id}` }) + + /** The document ids of a /query response, in the order the endpoint returned them. */ + const idsOf = (response) => response.body.map(o => o["@id"].split("/").pop()) + + /** The Link header of a response as { rel: url }. */ + const linksOf = (response) => Object.fromEntries( + [...(response.headers.link ?? "").matchAll(/<([^>]+)>;\s*rel="([^"]+)"/g)].map(([, url, rel]) => [rel, url]) + ) + + /** The query string of a link target, which may be absolute or path-only. */ + const paramsOf = (url) => new URL(url, "http://localhost").searchParams + + /** + * A find() double over a fixed, '_id' ordered set of documents. It honours the applied skip and + * limit, so a walk over it pages the way the collection would. + */ + const collectionOf = (docs) => () => { + const window = { skip: 0, limit: Infinity } + return { + sort() { return this }, + limit(n) { + window.limit = n + return this + }, + skip(n) { + window.skip = n + return this + }, + async toArray() { + return docs + .slice(window.skip, window.skip + window.limit) + .map(doc => structuredClone(doc)) + } + } + } + + const post = (path, body = { test: "item" }) => request(pagedTester) + .post(path) + .set("Content-Type", "application/json") + .send(body) + + it("links the next page while more records exist, and serves only the page", async () => { + db.find.mockImplementation(collectionOf(["a", "b", "c"].map(docWithId))) + const response = await post("/query?limit=2") + const links = linksOf(response) + + assert.strictEqual(response.statusCode, 200) + assert.deepStrictEqual(idsOf(response), ["a", "b"]) + assert.ok(links.next, "a next link is present while more records exist") + assert.strictEqual(paramsOf(links.next).get("limit"), "2") + assert.strictEqual(paramsOf(links.next).get("skip"), "2") + }) + + it("keeps the JSON-LD context link alongside the next link", async () => { + db.find.mockImplementation(collectionOf(["a", "b", "c"].map(docWithId))) + const links = linksOf(await post("/query?limit=2")) + assert.ok(links["http://www.w3.org/ns/json-ld#context"]) + assert.ok(links.next) + }) + + it("omits next on the final page", async () => { + db.find.mockImplementation(collectionOf(["a", "b"].map(docWithId))) + const response = await post("/query?limit=2") + const links = linksOf(response) + + assert.deepStrictEqual(idsOf(response), ["a", "b"]) + assert.deepStrictEqual(Object.keys(links), ["http://www.w3.org/ns/json-ld#context"], "only the JSON-LD context link remains") + }) + + it("never serves the record read past the page, even at the maximum limit", async () => { + db.find.mockImplementation(collectionOf([docWithId("a")])) + const limitMax = Number((await post("/query?limit=1")).headers['pagination-limit-max']) + const ids = Array.from({ length: limitMax + 1 }, (_, i) => `id${String(i).padStart(6, "0")}`) + db.find.mockImplementation(collectionOf(ids.map(docWithId))) + + const response = await post(`/query?limit=${limitMax + 100}`) + + assert.strictEqual(response.body.length, limitMax) + assert.ok(!idsOf(response).includes(ids.at(-1)), "the extra record must not be served") + assert.ok(linksOf(response).next) + }) + + it("still links next past the skip maximum, so a walk too deep to finish ends in the 400 that names it", async () => { + db.find.mockImplementation(collectionOf([docWithId("a")])) + const skipMax = Number((await post("/query?limit=1")).headers['pagination-skip-max']) + // Every page this double serves has a record past it, however deep the skip. + db.find.mockImplementation(() => ({ + sort() { return this }, + limit() { return this }, + skip() { return this }, + async toArray() { return ["a", "b", "c"].map(docWithId) } + })) + + const deepest = await post(`/query?limit=2&skip=${skipMax}`) + const next = linksOf(deepest).next + assert.strictEqual(deepest.statusCode, 200) + assert.strictEqual(paramsOf(next).get("skip"), String(skipMax + 2), "absence would tell the client the walk was complete") + + const beyond = await post(`/query?${paramsOf(next)}`) + assert.strictEqual(beyond.statusCode, 400) + assert.strictEqual(beyond.headers['pagination-skip-max'], String(skipMax)) + }) + + it("walks every record exactly once by following only rel=\"next\"", async () => { + const ids = ["d01", "d02", "d03", "d04", "d05", "d06", "d07"] + db.find.mockImplementation(collectionOf(ids.map(docWithId))) + + const walked = [] + let path = "/query?limit=3" + for (let requests = 0; path; requests++) { + assert.ok(requests < ids.length, "the walk must terminate") + const response = await post(path) + assert.strictEqual(response.statusCode, 200) + walked.push(...idsOf(response)) + const next = linksOf(response).next + path = next && `/query?${paramsOf(next)}` + } + + assert.deepStrictEqual(walked, ids) + }) +}) diff --git a/routes/__tests__/route_wrappers.test.js b/routes/__tests__/route_wrappers.test.js index c2b471d8..41314d1b 100644 --- a/routes/__tests__/route_wrappers.test.js +++ b/routes/__tests__/route_wrappers.test.js @@ -232,6 +232,14 @@ describe('unsupported-method 405 fallbacks', () => { assertUnsupportedMethodOnPath(router, path) }) } + + // HEAD will fall through to the 405 fallback rather than reach a handler of its own. + for (const { label, router, path } of cases) { + if (getMethodLayers(router, path, 'get').length) continue + it(`has no HEAD handler of its own for ${label}`, () => { + assert.deepStrictEqual(getMethodLayers(router, path, 'head'), []) + }) + } }) describe('api routes discovery', () => { diff --git a/routes/query.js b/routes/query.js index 281770c8..e2af9691 100644 --- a/routes/query.js +++ b/routes/query.js @@ -6,9 +6,8 @@ import rest from '../rest.js' router.route('/') .post(rest.verifyJsonContentType, controller.query) - .head(controller.queryHeadRequest) .all((req, res, next) => { - rest.sendMethodNotAllowed(res, 'Improper request method for requesting objects with matching properties. Please use POST.', 'POST,HEAD') + rest.sendMethodNotAllowed(res, 'Improper request method for requesting objects with matching properties. Please use POST.', 'POST') }) export default router