/query optimization

__tests__/core_provider_contract.test.js

@@ -204,13 +204,12 @@ const requiredResponseCodes = {
   'POST /id/{id}/expanded': ['200', '400', '404', '413', '415'],
   'GET /since/{id}': ['200', '404'],
   'GET /history/{id}': ['200', '404'],
-  // HEAD parity tests in routes/__tests__/{id,since,history,query}.test.js assert 404 on miss;
+  // HEAD parity tests in routes/__tests__/{id,since,history}.test.js assert 404 on miss;
   // enforce that the contract declares the same so drift on either side is caught.
   'HEAD /id/{id}': ['200', '404'],
   'HEAD /id/{id}/expanded': ['200', '404'],
   'HEAD /since/{id}': ['200', '404'],
   'HEAD /history/{id}': ['200', '404'],
-  'HEAD /api/query': ['200', '404'],
   'POST /api/query': ['200', '400', '413', '415'],
   'POST /api/search': ['200', '400', '413', '415'],
   'POST /api/search/phrase': ['200', '400', '413', '415']

__tests__/utils.test.js

@@ -10,6 +10,7 @@ import {
   _contextid,
   idNegotiation,
   getPagination,
+  setNextPageLink,
   findLeafAnnotationsFor
 } from '../controllers/utils.js'
 import { db, resetMocks, createCursor } from '../database/index.js'
@@ -514,6 +515,46 @@ describe('controllers/utils.js getPagination', () => {
   }
 })
 
+describe('controllers/utils.js setNextPageLink', () => {
+  /** Run setNextPageLink against a response double and hand back what it appended to 'Link'. */
+  const appendedFor = (page, req = { baseUrl: '/v1/api/query', path: '/' }) => {
+    const appended = []
+    setNextPageLink(req, { append: (name, value) => appended.push([name, value]) }, page)
+    assert.ok(appended.every(([name]) => name === 'Link'), 'only Link is appended to, never set over')
+    return appended.map(([, value]) => value)
+  }
+
+  /** Run a test body with RERUM_PREFIX set to a value, or unset for undefined, then restore it. */
+  const withPrefix = (value, body) => {
+    const original = process.env.RERUM_PREFIX
+    try {
+      if (value === undefined) delete process.env.RERUM_PREFIX
+      else process.env.RERUM_PREFIX = value
+      body()
+    } finally {
+      if (original === undefined) delete process.env.RERUM_PREFIX
+      else process.env.RERUM_PREFIX = original
+    }
+  }
+
+  it('links the next page on the deployment origin, for the path that was paged, advancing skip by the limit', () => {
+    withPrefix('https://store.example.org/v1/', () => {
+      assert.deepStrictEqual(appendedFor({ limit: 100, skip: 0, hasNext: true }), ['<https://store.example.org/v1/api/query?limit=100&skip=100>; rel="next"'])
+      assert.deepStrictEqual(appendedFor({ limit: 100, skip: 300, hasNext: true }), ['<https://store.example.org/v1/api/query?limit=100&skip=400>; rel="next"'])
+    })
+  })
+
+  it('builds a path-only link when the deployment origin is not configured', () => {
+    withPrefix(undefined, () => {
+      assert.deepStrictEqual(appendedFor({ limit: 10, skip: 0, hasNext: true }, { baseUrl: '/v1/api', path: '/query' }), ['</v1/api/query?limit=10&skip=10>; rel="next"'])
+    })
+  })
+
+  it('adds nothing on the final page', () => {
+    assert.deepStrictEqual(appendedFor({ limit: 100, skip: 200, hasNext: false }), [])
+  })
+})
+
 describe('controllers/utils.js findLeafAnnotationsFor', () => {
   const ENTITY_URI = 'https://store.rerum.io/v1/id/entity-id'
   const SLUG_URI = 'https://store.rerum.io/v1/id/entity-slug'

controllers/crud.js

@@ -6,7 +6,7 @@
  */
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, idNegotiation, getPagination, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js'
+import { _contextid, idNegotiation, getPagination, setNextPageLink, generateSlugId, ObjectID, getAgentClaim, findLeafAnnotationsFor, PROTECTED_EXPANSION_KEYS } from './utils.js'
 
 /**
  * Create a new Linked Open Data object in RERUM v1.
@@ -70,6 +70,8 @@ const create = async function (req, res, next) {
  * Query the MongoDB for objects containing the key:value pairs provided in the JSON Object in the request body.
  * This will support wildcards and mongo params like {"key":{$exists:true}}
  * The return is always an array, even if 0 or 1 objects in the return.
+ *
+ * Results page in ascending '_id' order.  The Link header carries rel="next" while another page exists.
  * */
 const query = async function (req, res, next) {
     res.set("Content-Type", "application/json; charset=utf-8")
@@ -84,9 +86,12 @@ const query = async function (req, res, next) {
     }
     const { limit, skip } = getPagination(req.query, { res })
     try {
-        let matches = await db.find(props).sort({ _id: 1 }).limit(limit).skip(skip).toArray()
-        matches = matches.map(o => idNegotiation(o))
+        // One record past the page is read only to learn whether another page exists.  It is never served.
+        let matches = await db.find(props).sort({ _id: 1 }).limit(limit + 1).skip(skip).toArray()
+        const hasNext = matches.length > limit
+        matches = matches.slice(0, limit).map(o => idNegotiation(o))
         res.set(utils.configureLDHeadersFor(matches))
+        setNextPageLink(req, res, { limit, skip, hasNext })
         res.json(matches)
     } catch (error) {
         return next(utils.createExpressError(error))

controllers/history.js

@@ -2,13 +2,13 @@
 
 /**
  * History controller for RERUM operations
- * Handles history, since, and HEAD request operations
+ * Handles history and since operations
  * @author cubap, thehabes
  */
 
 import { newID, isValidID, db } from '../database/index.js'
 import utils from '../utils.js'
-import { _contextid, ObjectID, getAgentClaim, getPagination, parseDocumentID, idNegotiation, getAllVersions, getAllAncestors, getAllDescendants } from './utils.js'
+import { _contextid, ObjectID, getAgentClaim, parseDocumentID, idNegotiation, getAllVersions, getAllAncestors, getAllDescendants } from './utils.js'
 
 /**
  * Public facing servlet to gather for all versions downstream from a provided `key object`.
@@ -79,32 +79,4 @@ const history = async function (req, res, next) {
     res.json(ancestors)
 }
 
-/**
- * Allow for HEAD requests via the RERUM getByProperties pattern /v1/api/query
- * No objects are returned, but the Content-Length header is set. 
- */
-const queryHeadRequest = async function (req, res, next) {
-    res.set("Content-Type", "application/json; charset=utf-8")
-    let props = req.body
-    const { limit, skip } = getPagination(req.query, { res })
-    try {
-        // Sorted the same way POST /query is, so the two verbs page over one order.
-        const matches = await db.find(props).sort({ _id: 1 }).limit(limit).skip(skip).toArray()
-        if (matches.length) {
-            const negotiated = matches.map(o => idNegotiation(o))
-            const size = Buffer.byteLength(JSON.stringify(negotiated))
-            res.set("Content-Length", size)
-            res.status(200).end()
-            return
-        }
-        let err = {
-            "message": `There are no objects in the database matching the query. Check the request body.`,
-            "status": 404
-        } 
-        return next(utils.createExpressError(err))
-    } catch (error) {
-        return next(utils.createExpressError(error))
-    }
-}
-
-export { since, history, queryHeadRequest }
+export { since, history }

controllers/utils.js

@@ -94,8 +94,8 @@ function readWholeNumberParam(raw, name, fallback, min) {
  * response readable.  A 'skip' above its maximum is rejected, because clamping it would serve the
  * page at the maximum over and over.
  *
- * The applied values and both maximums are reported in the response headers, so a client can tell
- * a truncated page from a genuine final one and can configure itself from any single response.
+ * The applied values and both maximums are reported in the response headers, so a client can
+ * configure itself from any single response.
  *
  * @param query The Express 'req.query' object.
  * @param options Optional settings for this endpoint.
@@ -134,6 +134,30 @@ function getPagination(query = {}, options = {}) {
     return { limit, skip }
 }
 
+/**
+ * Report an RFC 8288 rel="next" link while another page exists, alongside any Link the response
+ * already carries.  Its absence is what ends a walk, so nothing is added on the final page.
+ *
+ * 'next' advances 'skip' even when that passes the configured maximum.  Following it is then answered
+ * with the 400 that names the maximum, so a walk too deep to finish ends loudly instead of looking
+ * complete.
+ *
+ * @param req The Express request, for the path that was paged.
+ * @param res The Express response.
+ * @param page What was served.
+ * @param page.limit The applied limit.
+ * @param page.skip The applied skip.
+ * @param page.hasNext Whether any record lies past this page.
+ */
+function setNextPageLink(req, res, { limit, skip = 0, hasNext = false } = {}) {
+    if (!hasNext) return
+    // A route mounted at '/' leaves a trailing slash on the joined path.
+    const path = `${req.baseUrl ?? ""}${req.path ?? ""}`.replace(/(.)\/+$/, "$1")
+    const target = `${path}?${new URLSearchParams({ limit, skip: skip + limit })}`
+    const base = process.env.RERUM_PREFIX
+    res.append("Link", `<${URL.canParse(target, base) ? new URL(target, base).href : target}>; rel="next"`)
+}
+
 /**
  * Check if a @context value contains a known @id-id mapping context
  *
@@ -694,6 +718,7 @@ export {
     findLeafAnnotationsFor,
     PROTECTED_EXPANSION_KEYS,
     getPagination,
+    setNextPageLink,
     generateSlugId,
     index,
     ObjectID,

db-controller.js

@@ -13,7 +13,7 @@ import { searchAsWords, searchAsPhrase } from './controllers/search.js'
 import { deleteObj } from './controllers/delete.js'
 import { putUpdate, patchUpdate, patchSet, patchUnset, overwrite } from './controllers/update.js'
 import { bulkCreate, bulkUpdate } from './controllers/bulk.js'
-import { since, history, queryHeadRequest } from './controllers/history.js'
+import { since, history } from './controllers/history.js'
 import { release } from './controllers/release.js'
 import { _gog_fragments_from_manuscript, _gog_glosses_from_manuscript, expand, expandedId } from './controllers/gog.js'
 
@@ -35,7 +35,6 @@ export default {
     idExpanded,
     bulkCreate,
     bulkUpdate,
-    queryHeadRequest,
     since,
     history,
     remove,

openapi/contracts/core-provider.openapi.yaml

@@ -220,6 +220,9 @@ paths:
       responses:
         '200':
           description: Query results
+          headers:
+            Link:
+              $ref: '#/components/headers/NextPageLink'
           content:
             application/json:
               schema:
@@ -230,14 +233,6 @@ paths:
           $ref: '#/components/responses/PayloadTooLarge'
         '415':
           $ref: '#/components/responses/UnsupportedMediaType'
-    head:
-      summary: Query object headers
-      operationId: headQueryObjects
-      responses:
-        '200':
-          description: Query result headers
-        '404':
-          $ref: '#/components/responses/NotFound'
   /api/search:
     post:
       summary: Search objects by keywords
@@ -689,6 +684,15 @@ components:
         never been overwritten.
       schema:
         type: string
+    NextPageLink:
+      description: >-
+        RFC 8288 links. The JSON-LD context link, plus rel="next" only while another page exists, so
+        its absence ends a walk.
+      schema:
+        type: string
+        example: >-
+          <http://store.rerum.io/v1/context.json>; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json",
+          <https://store.rerum.io/v1/api/query?limit=100&skip=200>; rel="next"
   parameters:
     ObjectId:
       in: path

public/API.html

@@ -504,7 +504,7 @@ <h3 id="custom-query">Custom Query</h3>
         </ul>
         <p>
             This simple format will be made more complex in the future, but should serve the basic needs as it is.
-            RERUM will test for property matches. By default, this is limited to 100 records in the response to guard against unreasonable queries. To allow for more records in the response one can add the URL parameter <code>limit</code> to the query requests. If you expect the query request will have a very large response with many objects, your application should use a paged query by also using the <code>skip</code> URL parameter. You will see an example of this below.
+            RERUM will test for property matches. By default, this is limited to 100 records in the response to guard against unreasonable queries. To allow for more records in the response one can add the URL parameter <code>limit</code> to the query requests. If you expect the query request will have a very large response with many objects, your application should use a paged query that follows the <code>rel="next"</code> link in each response's <code>Link</code> header until there is none. You will see an example of this below.
             <p class="alert">A <code>limit</code> above the maximum of 500 is not an error. It is reduced to 500, and the response says so &mdash; see <a href="#pagination-parameters">Pagination parameters</a>. Read the applied value from the <code>Pagination-Limit</code> response header rather than assuming you received everything you asked for.</p>
         </p>
         <p>
@@ -551,27 +551,27 @@ <h3 id="custom-query">Custom Query</h3>
         <p>
             <div class="exHeading">Paged Query Javascript Example</div>
             <pre><code class="jsExample"> 
-                <span>const many_results = await pagedQuery(100, 0, {"type": "Thing"})</span>
+                <span>const many_results = await pagedQuery({"type": "Thing"})</span>
                 <br>
-                <span>async function pagedQuery(lim, it = 0, queryObj, allResults = []) {</span>
-                    <span class="ind1">const response = await fetch(`https://devstore.rerum.io/v1/api/query?limit=${lim}&skip=${it}`, {</span>
-                        <span class="ind2"> method: "POST",</span>
-                        <span class="ind2">headers: {</span>
-                            <span class="ind3">"Content-Type": "application/json; charset=utf-8"</span>
-                        <span class="ind2">},</span>
+                <span>async function pagedQuery(queryObj, lim = 100) {</span>
+                    <span class="ind1">const request = {</span>
+                        <span class="ind2">method: "POST",</span>
+                        <span class="ind2">headers: { "Content-Type": "application/json; charset=utf-8" },</span>
                         <span class="ind2">body: JSON.stringify(queryObj)</span>
-                    <span class="ind1">})</span>
-                    <span class="ind1">const skipMax = Number(response.headers.get("Pagination-Skip-Max"))</span>
-                    <span class="ind1">if (response.status === 400 &amp;&amp; skipMax &amp;&amp; it &gt; skipMax) {</span>
-                        <span class="ind2">console.warn(`Stopped at the skip ceiling of ${skipMax}. These results are incomplete. Narrow the query.`)</span>
-                        <span class="ind2">return allResults</span>
                     <span class="ind1">}</span>
-                    <span class="ind1">if (!response.ok) throw new Error(await response.text())</span>
-                    <span class="ind1">const results = await response.json()</span>
-                    <span class="ind1">allResults.push(...results)</span>
-                    <span class="ind1">const appliedLimit = Number(response.headers.get("Pagination-Limit")) || lim</span>
-                    <span class="ind1">if (results.length &lt; appliedLimit) return allResults</span>
-                    <span class="ind1">return pagedQuery(lim, it + appliedLimit, queryObj, allResults)</span>
+                    <span class="ind1">const allResults = []</span>
+                    <span class="ind1">let url = `https://devstore.rerum.io/v1/api/query?limit=${lim}`</span>
+                    <span class="ind1">while (url) {</span>
+                        <span class="ind2">const response = await fetch(url, request)</span>
+                        <span class="ind2">if (response.status === 400 &amp;&amp; allResults.length) {</span>
+                            <span class="ind3">console.warn(`Stopped at the skip maximum of ${response.headers.get("Pagination-Skip-Max")}. These results are incomplete. Narrow the query.`)</span>
+                            <span class="ind3">return allResults</span>
+                        <span class="ind2">}</span>
+                        <span class="ind2">if (!response.ok) throw new Error(await response.text())</span>
+                        <span class="ind2">allResults.push(...await response.json())</span>
+                        <span class="ind2">url = response.headers.get("Link")?.match(/&lt;([^&gt;]+)&gt;;\s*rel="next"/)?.[1]</span>
+                    <span class="ind1">}</span>
+                    <span class="ind1">return allResults</span>
                 <span>}</span>
             </code></pre>
         </p>
@@ -604,7 +604,7 @@ <h3 id="pagination-parameters">Pagination parameters</h3>
             </tbody>
         </table>
         <p>
-            Every paged response reports what was applied and what the maximums are, so a client can configure itself from any single response and tell a truncated page from a genuine final one. These headers are readable cross-origin.
+            Every paged response reports what was applied and what the maximums are, so a client can configure itself from any single response. These headers are readable cross-origin.
         </p>
         <p>
             <code>Pagination-Limit-Max</code> and <code>Pagination-Skip-Max</code> are reported on rejections too, including the <code>400</code> for a <code>skip</code> past the maximum. That is what lets a paged walk recognise the depth boundary and stop cleanly.
@@ -617,9 +617,23 @@ <h3 id="pagination-parameters">Pagination parameters</h3>
                 <span>Pagination-Skip-Max: 100000</span>
             </code></pre>
         </p>
+        <h4 id="following-next">Following <code>rel="next"</code></h4>
+        <p>
+            While another page exists, <a href="#custom-query">Custom Query</a> responses carry a <code>rel="next"</code> link in the <code>Link</code> header (<a target="_blank" href="https://www.rfc-editor.org/rfc/rfc8288">RFC 8288</a>), next to the JSON-LD context link. The final page has no <code>rel="next"</code>. Follow it until it is gone.
+        </p>
+        <p>
+            <code>rel="next"</code> advances <code>skip</code>. It is still present when the next page would start past the <code>skip</code> maximum, and following it is answered with the <code>400</code> that names the maximum.
+        </p>
+        <p class="alert">Custom Query takes a <code>POST</code>. The link only carries the paging URL parameters, so send the same request body to it that you sent to the first page.</p>
+        <p>
+            <pre><code class="jsExample"> 
+                <span>Link: &lt;http://store.rerum.io/v1/context.json&gt;; rel="http://www.w3.org/ns/json-ld#context"; type="application/ld+json"</span>
+                <span>Link: &lt;https://store.rerum.io/v1/api/query?limit=100&amp;skip=200&gt;; rel="next"</span>
+            </code></pre>
+        </p>
         <h4>Result ordering</h4>
         <p>
-            <a href="#custom-query">Custom Query</a> pages in ascending <code>_id</code> order, on both <code>POST</code> and <code>HEAD</code>. A <code>skip</code> offset only means something over a fixed order, so consecutive pages tile that order without repeating or missing a record.
+            <a href="#custom-query">Custom Query</a> pages in ascending <code>_id</code> order. A <code>skip</code> offset only means something over a fixed order, so consecutive pages tile that order without repeating or missing a record.
         </p>
 
         <p>