v0.3.2: safer connection check and first() error handling

DEVELOPMENT.md

@@ -32,7 +32,8 @@ Verified here (offline): JSON validity, manifest links, request bodies and outpu
 Not verified, needs a Make account and a BatchData test token (a sandbox token returns mock data at no cost):
 
 - Import with the VS Code extension (the file layout follows public examples, not a Make validator).
-- Connection check. It POSTs `{"requests":[]}` to `/v1/address/verify` and marks the connection valid on HTTP 400 or 2xx through `response.valid`. If Make marks the 400 as an error before it reads `valid`, switch the check to `GET https://api.batchdata.com/api/v2/property/search-sessions?take=1` (valid on 2xx). Also confirm with BatchData that the rejected empty request is not billed.
+- Connection check. It POSTs a fixed autocomplete query (`searchCriteria.query` = `1600 Penn`, `take` 1) to `/v1/address/autocomplete` and is valid on HTTP 2xx only when `body.result` is present, so a 200 HTML error page is rejected as `InvalidAccessTokenError`. An earlier empty `/v1/address/verify` check (valid on 400 or 2xx) failed in Make because HTTP 400-599 are errors and `response.valid` cannot override that. Autocomplete was chosen over `GET /v2/property/search-sessions?take=1` because search-sessions can return 403 for tokens without that permission.
+- `first()` mappings (Get Property Details, Verify Address, and the billed instant trigger's property lookup) use `first(ifempty(path, emptyarray))` and `response.valid` so a 401 JSON body or a 200 HTML page surfaces as `InvalidAccessTokenError` instead of Make's `IMLError: Function 'first' finished with error! Invalid array.`
 - Optional fields. The templates use `ifempty(x, undefined)` and `omit()` so empty fields are not sent. Nested optional filters (for example Permit Status) are collection parameters that are dropped when empty. Check that an unfilled nested collection (for example Building) is not sent as `{}`.
 - Instant triggers. Check that turning a scenario on creates the property subscription (attach), that a test event arrives with `eventId` set, and that turning it off deletes the subscription (detach, `externalHookId`). Create hooks only with a sandbox token: a subscription counts towards the team's limit of 5 and cannot be edited.
 - Polling triggers (`Watch New Search Sessions`, `Watch New Properties in a Search Session`): confirm the date ordering and that a first run does not replay old items as expected.

connections/batchdata/batchdata.communication.iml.json

@@ -1,15 +1,24 @@
 {
-  "url": "https://api.batchdata.com/api/v1/address/verify",
+  "url": "https://api.batchdata.com/api/v1/address/autocomplete",
   "method": "POST",
   "headers": {
     "Authorization": "Bearer {{parameters.apiKey}}",
     "Content-Type": "application/json"
   },
   "body": {
-    "requests": []
+    "searchCriteria": {
+      "query": "1600 Penn"
+    },
+    "options": {
+      "take": 1
+    }
   },
   "response": {
-    "valid": "{{statusCode == 400 || (statusCode >= 200 && statusCode < 300)}}",
+    "valid": {
+      "condition": "{{statusCode >= 200 && statusCode < 300 && if(body.result, true, false)}}",
+      "type": "InvalidAccessTokenError",
+      "message": "The BatchData API did not return a usable response. Check that the API token is valid. Contact support@batchservice.com if it still fails."
+    },
     "data": {
       "apiKey": "{{parameters.apiKey}}"
     },

modules/lookupProperty/lookupProperty.communication.iml.json

@@ -18,6 +18,11 @@
     }
   },
   "response": {
-    "output": "{{first(body.results.properties)}}"
+    "valid": {
+      "condition": "{{if(body.results, true, false)}}",
+      "type": "InvalidAccessTokenError",
+      "message": "The BatchData API did not return a usable response. Check that the API token is valid. Contact support@batchservice.com if it still fails."
+    },
+    "output": "{{first(ifempty(body.results.properties, emptyarray))}}"
   }
 }

modules/verifyAddress/verifyAddress.communication.iml.json

@@ -12,6 +12,11 @@
     ]
   },
   "response": {
-    "output": "{{first(body.results.addresses)}}"
+    "valid": {
+      "condition": "{{if(body.results, true, false)}}",
+      "type": "InvalidAccessTokenError",
+      "message": "The BatchData API did not return a usable response. Check that the API token is valid. Contact support@batchservice.com if it still fails."
+    },
+    "output": "{{first(ifempty(body.results.addresses, emptyarray))}}"
   }
 }

modules/watchMonitorEventsWithProperty/watchMonitorEventsWithProperty.communication.iml.json

@@ -9,9 +9,14 @@
     ]
   },
   "response": {
+    "valid": {
+      "condition": "{{if(body.results, true, false)}}",
+      "type": "InvalidAccessTokenError",
+      "message": "The BatchData API did not return a usable response. Check that the API token is valid. Contact support@batchservice.com if it still fails."
+    },
     "output": {
       "{{...}}": "{{payload}}",
-      "property": "{{first(body.results.properties)}}"
+      "property": "{{first(ifempty(body.results.properties, emptyarray))}}"
     }
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "batchdata-make",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "private": true,
   "description": "BatchData custom app for Make",
   "type": "module",

scripts/catalog-modules.mjs

@@ -1,7 +1,7 @@
 // Turns the shared catalog (scripts/catalog.json) into Make modules, webhooks and RPCs.
 // The catalog is the single source of truth shared with the Zapier and n8n integrations.
 import fs from 'node:fs';
-import { root } from './lib.mjs';
+import { root, firstItem, requireApiObject } from './lib.mjs';
 import { limitParam, offsetPaging, cursorPaging } from './paging.mjs';
 
 export const catalog = JSON.parse(fs.readFileSync(`${root}/scripts/catalog.json`, 'utf8'));
@@ -390,7 +390,8 @@ catalogModules.watchMonitorEventsWithProperty = {
     method: 'POST',
     body: { requests: [{ propertyId: '{{payload.propertyId}}' }] },
     response: {
-      output: { '{{...}}': '{{payload}}', [ev.hydrate.into]: `{{first(body.${ev.hydrate.listPath})}}` },
+      valid: requireApiObject('body.results'),
+      output: { '{{...}}': '{{payload}}', [ev.hydrate.into]: firstItem(`body.${ev.hydrate.listPath}`) },
     },
   },
   staticParams: [],

scripts/generate.mjs

@@ -1,5 +1,5 @@
 import fs from 'node:fs';
-import { root, write, toInterface } from './lib.mjs';
+import { root, write, toInterface, firstItem, requireApiObject } from './lib.mjs';
 import * as P from './params.mjs';
 import { offsetPaging } from './paging.mjs';
 import { catalogModules, catalogGroups, webhooks, rpcs } from './catalog-modules.mjs';
@@ -104,7 +104,7 @@ const modules = {
           datasets: '{{if(length(parameters.datasets) > 0, parameters.datasets, undefined)}}',
         },
       },
-      response: { output: '{{first(body.results.properties)}}' },
+      response: { valid: requireApiObject('body.results'), output: firstItem('body.results.properties') },
     },
     params: P.lookupParams,
     sampleFrom: () => fx('property-lookup-all-attributes').results.properties[0],
@@ -133,7 +133,7 @@ const modules = {
       url: '/v1/address/verify',
       method: 'POST',
       body: { requests: [{ street: '{{parameters.street}}', city: u('parameters.city'), state: u('parameters.state'), zip: u('parameters.zip') }] },
-      response: { output: '{{first(body.results.addresses)}}' },
+      response: { valid: requireApiObject('body.results'), output: firstItem('body.results.addresses') },
     },
     params: P.verifyAddressParams,
     sampleFrom: () => fx('address-verify').results.addresses[0],
@@ -312,12 +312,16 @@ write('connections/batchdata/batchdata.params.iml.json', [
   },
 ]);
 write('connections/batchdata/batchdata.communication.iml.json', {
-  url: 'https://api.batchdata.com/api/v1/address/verify',
+  // Make treats HTTP 400-599 as errors and ignores response.valid for those statuses,
+  // so an empty /v1/address/verify (400) cannot mark a connection as valid.
+  // Autocomplete returns 200 with a valid token and 401 with an invalid one, and is
+  // available to typical API tokens (search-sessions can 403 without that permission).
+  url: 'https://api.batchdata.com/api/v1/address/autocomplete',
   method: 'POST',
   headers: { Authorization: 'Bearer {{parameters.apiKey}}', 'Content-Type': 'application/json' },
-  body: { requests: [] },
+  body: { searchCriteria: { query: '1600 Penn' }, options: { take: 1 } },
   response: {
-    valid: '{{statusCode == 400 || (statusCode >= 200 && statusCode < 300)}}',
+    valid: requireApiObject('body.result', 'statusCode >= 200 && statusCode < 300'),
     data: { apiKey: '{{parameters.apiKey}}' },
     // Make shows the app name beside every connection, so the label never repeats "BatchData".
     metadata: {

scripts/lib.mjs

@@ -41,3 +41,19 @@ export const toInterface = (value) => {
   }
   return out;
 };
+
+// Make's first() throws IMLError ("Invalid array") when the value is not an array,
+// which happens on a 401 JSON body or a 200 HTML error page. Guard with ifempty so
+// first() always receives an array; mark the response invalid when the parent object
+// is missing so the user sees InvalidAccessTokenError instead of IMLError.
+export const firstItem = (listPath) => `{{first(ifempty(${listPath}, emptyarray))}}`;
+
+export const UNUSABLE_RESPONSE = 'The BatchData API did not return a usable response. Check that the API token is valid. Contact support@batchservice.com if it still fails.';
+
+export const requireApiObject = (path, extraCondition = '') => ({
+  condition: extraCondition
+    ? `{{${extraCondition} && if(${path}, true, false)}}`
+    : `{{if(${path}, true, false)}}`,
+  type: 'InvalidAccessTokenError',
+  message: UNUSABLE_RESPONSE,
+});

test/iml.mjs

@@ -15,6 +15,8 @@ const fns = {
   formatDate: (d) => (d ? new Date(d).toISOString().slice(0, 10) : undefined),
   parseJSON: (s) => (s === undefined || s === null || s === '' ? undefined : JSON.parse(s)),
   toCollection: (arr, k, v) => Object.fromEntries((arr || []).map((e) => [e[k], e[v]])),
+  emptyarray: [],
+  emptystring: '',
 };
 
 const evalExpr = (expr, scope) => {

test/replay.test.mjs

@@ -158,15 +158,21 @@ test('api call: builds url, method, headers and query; wallet paths are redirect
   }
 });
 
-test('connection check treats 400 and 2xx as valid and 401/403 as invalid', () => {
+test('connection check uses address autocomplete and treats only 2xx JSON with a result as valid', () => {
   const c = read('connections/batchdata/batchdata.communication.iml.json');
-  assert.deepEqual(c.body, { requests: [] });
-  const valid = (statusCode) => render(c.response.valid, { statusCode });
-  assert.equal(valid(200), true);
-  assert.equal(valid(400), true);
-  assert.equal(valid(401), false);
-  assert.equal(valid(403), false);
-  assert.equal(valid(500), false);
+  assert.match(c.url, /address\/autocomplete$/);
+  assert.deepEqual(c.body, { searchCriteria: { query: '1600 Penn' }, options: { take: 1 } });
+  const valid = (scope) => render(c.response.valid.condition, scope);
+  const ok = { result: { suggestions: [] } };
+  assert.equal(valid({ statusCode: 200, body: ok }), true);
+  assert.equal(valid({ statusCode: 204, body: ok }), true);
+  assert.equal(valid({ statusCode: 200, body: {} }), false);
+  assert.equal(valid({ statusCode: 200, body: '<!DOCTYPE html><html><title>BatchData</title></html>' }), false);
+  assert.equal(valid({ statusCode: 400, body: ok }), false);
+  assert.equal(valid({ statusCode: 401, body: ok }), false);
+  assert.equal(valid({ statusCode: 403, body: ok }), false);
+  assert.equal(valid({ statusCode: 500, body: ok }), false);
+  assert.equal(c.response.valid.type, 'InvalidAccessTokenError');
   assert.equal(render(c.headers.Authorization, { parameters: { apiKey: 't' } }), 'Bearer t');
   assert.equal(render(c.response.data, { parameters: { apiKey: 't' } }).apiKey, 't');
 });
@@ -180,3 +186,41 @@ test('base: bearer header comes from the connection and errors read the BatchDat
   assert.match(fallback, /^\[500\]/);
   assert.equal(b.response.error['401'].type, 'InvalidAccessTokenError');
 });
+
+test('lookup, verify address and the billed instant trigger do not throw on a 401 or HTML body', () => {
+  const html = '<!DOCTYPE html><html><title>BatchData</title></html>';
+  const unauthorized = { status: { code: 401, text: 'Unauthorized', message: 'Invalid token.', data: [] } };
+  const lookupFx = fixture('property-lookup-all-attributes');
+  const verifyFx = fixture('address-verify');
+
+  const lookup = comm('lookupProperty');
+  assert.equal(render(lookup.response.valid.condition, { body: lookupFx }), true);
+  assert.equal(render(lookup.response.valid.condition, { body: unauthorized }), false);
+  assert.equal(render(lookup.response.valid.condition, { body: html }), false);
+  assert.equal(lookup.response.valid.type, 'InvalidAccessTokenError');
+  assert.deepEqual(respond('lookupProperty', lookupFx), lookupFx.results.properties[0]);
+  assert.equal(respond('lookupProperty', unauthorized), undefined);
+  assert.equal(respond('lookupProperty', html), undefined);
+
+  const verify = comm('verifyAddress');
+  assert.equal(render(verify.response.valid.condition, { body: verifyFx }), true);
+  assert.equal(render(verify.response.valid.condition, { body: unauthorized }), false);
+  assert.equal(render(verify.response.valid.condition, { body: html }), false);
+  assert.equal(verify.response.valid.type, 'InvalidAccessTokenError');
+  assert.equal(respond('verifyAddress', verifyFx).hash, verifyFx.results.addresses[0].hash);
+  assert.equal(respond('verifyAddress', unauthorized), undefined);
+  assert.equal(respond('verifyAddress', html), undefined);
+
+  const billed = comm('watchMonitorEventsWithProperty');
+  const payload = { eventId: 'e1', propertyId: 'p1' };
+  assert.equal(render(billed.response.valid.condition, { body: lookupFx, payload }), true);
+  assert.equal(render(billed.response.valid.condition, { body: unauthorized, payload }), false);
+  assert.equal(render(billed.response.valid.condition, { body: html, payload }), false);
+  assert.equal(billed.response.valid.type, 'InvalidAccessTokenError');
+  const hydrated = respond('watchMonitorEventsWithProperty', lookupFx, { payload });
+  assert.equal(hydrated.eventId, 'e1');
+  assert.equal(hydrated.property._id, lookupFx.results.properties[0]._id);
+  const failed = respond('watchMonitorEventsWithProperty', html, { payload });
+  assert.equal(failed.eventId, 'e1');
+  assert.equal(failed.property, undefined);
+});

test/structure.test.mjs

@@ -199,3 +199,29 @@ test('billed search modules default to Limit 1, and the billed session trigger d
   assert.match(take.help, /Default 5/);
 });
 
+
+test('every first() mapping is guarded and marks a missing API object as InvalidAccessTokenError', () => {
+  const files = jsonFiles.filter((f) => f.endsWith('.communication.iml.json') || f.endsWith('batchdata.communication.iml.json'));
+  let found = 0;
+  for (const f of files) {
+    const text = fs.readFileSync(f, 'utf8');
+    const rel = path.relative(root, f);
+    const matches = [...text.matchAll(/\{\{first\(([^}]+)\)\}\}/g)];
+    for (const m of matches) {
+      found += 1;
+      assert.match(m[1], /ifempty\(.+, emptyarray\)/, `${rel} unguarded first(${m[1]})`);
+    }
+  }
+  assert.ok(found >= 3, `expected at least 3 first() mappings, found ${found}`);
+
+  for (const [rel, parent] of [
+    ['modules/lookupProperty/lookupProperty.communication.iml.json', 'body.results'],
+    ['modules/verifyAddress/verifyAddress.communication.iml.json', 'body.results'],
+    ['modules/watchMonitorEventsWithProperty/watchMonitorEventsWithProperty.communication.iml.json', 'body.results'],
+  ]) {
+    const comm = read(rel);
+    assert.equal(comm.response.valid.type, 'InvalidAccessTokenError', rel);
+    assert.match(comm.response.valid.condition, new RegExp(parent.replace('.', '\\.')), rel);
+    assert.match(comm.response.valid.message, /API token is valid/);
+  }
+});