From 83b156314a52d2c4ef260c419583c8ae783fe184 Mon Sep 17 00:00:00 2001 From: Joe Elstner Date: Wed, 23 Sep 2026 22:34:48 -0500 Subject: [PATCH] feat(bot-middleware): label every bot hit with who the bot is; declared agents via Web Bot Auth (1.6.0) Step 1 (Measure) of the bot-swarm lane, web channel, in the shared middleware every fleet site imports. Lockstep with isimplifyme-site's inlined copy (#216) and stored by apex-portal (#498). - detectDeclaredAgent reads Signature-Agent (+ Signature-Input/Signature => signed); botKind = crawler | agent_fetch | declared_agent | unknown_bot; AGENT_FETCHERS = ChatGPT-User, meta-externalfetcher, Claude-User, Perplexity-User, MistralAI-User, DuckAssistBot. - 5 new AI fetcher patterns (48 -> 53). Every bot-hit payload adds kind, signed, agentDirectory; a signing agent with a browser UA is tracked as agent:; the AI-referrer path skips signing agents. - version 1.5.0 -> 1.6.0 (additive fields; no API change). vitest 77 pass (+14). tsc clean. Co-Authored-By: Claude Fable 5.1 --- package.json | 2 +- src/bot-middleware.test.ts | 107 +++++++++++++++++++++++++++++++++++++ src/bot-middleware.ts | 86 +++++++++++++++++++++++++++-- 3 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 src/bot-middleware.test.ts diff --git a/package.json b/package.json index 4bcbbe9..cef558c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@isimplifyme/ui", - "version": "1.5.0", + "version": "1.6.0", "description": "React/Next.js UI library for iSimplifyMe properties \u2014 design system, article layouts, SEO helpers, bot middleware, and the iSM Concierge widget.", "homepage": "https://isimplifyme.com", "type": "module", diff --git a/src/bot-middleware.test.ts b/src/bot-middleware.test.ts new file mode 100644 index 0000000..1e0cc57 --- /dev/null +++ b/src/bot-middleware.test.ts @@ -0,0 +1,107 @@ +import { describe, it, expect, vi } from "vitest"; +import { + AGENT_FETCHERS, + botKind, + declaredAgentBot, + detectBot, + detectDeclaredAgent, + trackBot, +} from "./bot-middleware"; + +const headers = (map: Record) => ({ + get: (name: string) => map[name.toLowerCase()] ?? null, +}); + +describe("detectDeclaredAgent (Web Bot Auth)", () => { + it("reads a quoted Signature-Agent URL and the signature headers", () => { + expect( + detectDeclaredAgent(headers({ + "signature-agent": '"https://chatgpt.com"', + "signature-input": 'sig1=("@authority" "signature-agent");created=1758600000;keyid="k1"', + signature: "sig1=:abc=:", + })), + ).toEqual({ directory: "chatgpt.com", signed: true }); + }); + it("accepts a bare host and reports unsigned when the signature is missing", () => { + expect(detectDeclaredAgent(headers({ "signature-agent": "Agent.Example.com" }))) + .toEqual({ directory: "agent.example.com", signed: false }); + }); + it("is null without the header", () => { + expect(detectDeclaredAgent(headers({ "user-agent": "Mozilla/5.0" }))).toBeNull(); + expect(detectDeclaredAgent(headers({ "signature-agent": '""' }))).toBeNull(); + }); +}); + +describe("botKind", () => { + it("declared beats every User-Agent class", () => { + const d = { directory: "chatgpt.com", signed: true }; + expect(botKind("GPTBot", d)).toBe("declared_agent"); + expect(botKind("Unknown", d)).toBe("declared_agent"); + }); + it("fetchers are agent_fetch; other bots are crawlers; Unknown is unknown_bot", () => { + for (const name of AGENT_FETCHERS) expect(botKind(name, null)).toBe("agent_fetch"); + expect(botKind("GPTBot", null)).toBe("crawler"); + expect(botKind("Meta-ExternalAgent", null)).toBe("crawler"); + expect(botKind("Googlebot", null)).toBe("crawler"); + expect(botKind("Unknown", null)).toBe("unknown_bot"); + }); +}); + +describe("new fetcher patterns", () => { + it("recognizes the user-triggered fetchers by name", () => { + expect(detectBot("Mozilla/5.0 (compatible; Claude-User/1.0; +claude.ai)")?.name).toBe("Claude-User"); + expect(detectBot("Perplexity-User/1.0")?.name).toBe("Perplexity-User"); + expect(detectBot("MistralAI-User/1.0")?.name).toBe("MistralAI-User"); + expect(detectBot("DuckAssistBot/1.0")?.name).toBe("DuckAssistBot"); + expect(detectBot("Mozilla/5.0 Claude-SearchBot")?.name).toBe("Claude-SearchBot"); + // ClaudeBot and PerplexityBot still win for their own UAs + expect(detectBot("ClaudeBot/1.0")?.name).toBe("ClaudeBot"); + expect(detectBot("PerplexityBot/1.0")?.name).toBe("PerplexityBot"); + }); +}); + +describe("trackBot payload", () => { + const fakeReq = (ua: string | null, extra: Record = {}) => ({ + url: "https://example.com/pricing", + headers: headers({ ...(ua ? { "user-agent": ua } : {}), ...extra }), + }); + const run = (req: unknown) => { + const fetchMock = vi.fn(() => Promise.resolve(new Response("{}"))); + vi.stubGlobal("fetch", fetchMock); + const waited: Promise[] = []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + trackBot(req as any, { waitUntil: (p: Promise) => waited.push(p) } as any, { + webhookUrl: "https://apex.example/api/webhooks/bot-hit", + webhookSecret: "s", + }); + vi.unstubAllGlobals(); + if (!fetchMock.mock.calls.length) return null; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return JSON.parse((fetchMock.mock.calls[0] as any)[1].body); + }; + + it("labels a fetcher as agent_fetch", () => { + const body = run(fakeReq("Mozilla/5.0 ChatGPT-User/1.0")); + expect(body).toMatchObject({ bot: "ChatGPT-User", kind: "agent_fetch", signed: false, agentDirectory: null }); + }); + it("labels a crawler as crawler", () => { + expect(run(fakeReq("GPTBot/1.0"))).toMatchObject({ bot: "GPTBot", kind: "crawler" }); + }); + it("counts a signing agent with a browser UA under its directory host", () => { + const body = run(fakeReq("Mozilla/5.0 (Macintosh) Chrome/130", { + "signature-agent": '"https://muse.meta.com"', + "signature-input": "sig1=()", + signature: "sig1=:x:", + })); + expect(body).toMatchObject({ + bot: "agent:muse.meta.com", type: "ai", kind: "declared_agent", signed: true, agentDirectory: "muse.meta.com", + }); + }); + it("still ignores a plain browser", () => { + expect(run(fakeReq("Mozilla/5.0 (Macintosh) Chrome/130"))).toBeNull(); + }); + it("declaredAgentBot shape", () => { + expect(declaredAgentBot({ directory: "chatgpt.com", signed: true })) + .toEqual({ name: "agent:chatgpt.com", company: "chatgpt.com", type: "ai" }); + }); +}); diff --git a/src/bot-middleware.ts b/src/bot-middleware.ts index e6b7ca3..033a5b1 100644 --- a/src/bot-middleware.ts +++ b/src/bot-middleware.ts @@ -31,7 +31,7 @@ interface BotPattern extends BotMatch { pattern: RegExp; } -/** 48 bots across 6 categories — kept in sync with apex-portal src/lib/bots/parser.ts */ +/** 53 bots across 6 categories — kept in sync with apex-portal src/lib/bots/parser.ts and each site's inlined middleware */ export const BOT_PATTERNS: readonly BotPattern[] = [ // AI crawlers { pattern: /GPTBot/i, name: "GPTBot", company: "OpenAI", type: "ai" }, @@ -39,7 +39,12 @@ export const BOT_PATTERNS: readonly BotPattern[] = [ { pattern: /OAI-SearchBot/i, name: "OAI-SearchBot", company: "OpenAI", type: "ai" }, { pattern: /ClaudeBot/i, name: "ClaudeBot", company: "Anthropic", type: "ai" }, { pattern: /anthropic-ai/i, name: "anthropic-ai", company: "Anthropic", type: "ai" }, + { pattern: /Claude-User/i, name: "Claude-User", company: "Anthropic", type: "ai" }, + { pattern: /Claude-SearchBot/i, name: "Claude-SearchBot", company: "Anthropic", type: "ai" }, { pattern: /PerplexityBot/i, name: "PerplexityBot", company: "Perplexity", type: "ai" }, + { pattern: /Perplexity-User/i, name: "Perplexity-User", company: "Perplexity", type: "ai" }, + { pattern: /MistralAI-User/i, name: "MistralAI-User", company: "Mistral", type: "ai" }, + { pattern: /DuckAssistBot/i, name: "DuckAssistBot", company: "DuckDuckGo", type: "ai" }, { pattern: /Google-Extended/i, name: "Google-Extended", company: "Google", type: "ai" }, { pattern: /Bytespider/i, name: "Bytespider", company: "ByteDance", type: "ai" }, { pattern: /Meta-ExternalAgent/i, name: "Meta-ExternalAgent", company: "Meta", type: "ai" }, @@ -109,6 +114,75 @@ export function detectUnknownBot(userAgent: string | null | undefined): BotMatch return null; } +// ─── Who is the bot? (2026-09) ───────────────────────────────────────────── +// +// Every bot hit carries a `kind` beside its name and category: +// crawler — training/index crawlers and every other classic bot +// agent_fetch — an AI assistant fetching because a person asked it +// something right now (ChatGPT-User, Claude-User, …) +// declared_agent — the request carries a Web Bot Auth signature (RFC 9421 +// HTTP Message Signatures): the agent named its key +// directory in `Signature-Agent`, whatever its User-Agent +// says. The only honest "an agent is here" signal on the +// web for zones without Cloudflare Bot Management. +// unknown_bot — bot-like User-Agent we have no pattern for + +export type BotKind = "crawler" | "agent_fetch" | "declared_agent" | "unknown_bot"; + +/** Bots that fetch because a person asked an AI assistant something right now. */ +export const AGENT_FETCHERS: ReadonlySet = new Set([ + "ChatGPT-User", + "meta-externalfetcher", + "Claude-User", + "Perplexity-User", + "MistralAI-User", + "DuckAssistBot", +]); + +export interface DeclaredAgent { + /** Host of the agent's key directory, from `Signature-Agent` (e.g. "chatgpt.com"). */ + directory: string; + /** Whether the request also carried `Signature-Input` and `Signature`. */ + signed: boolean; +} + +type HeaderReader = { get(name: string): string | null }; + +/** + * Web Bot Auth: a signing agent sends `Signature-Agent: "https://host"` (an + * sf-string, so quoted) plus `Signature-Input` and `Signature`. We record the + * directory host and whether the signature headers are present; verifying the + * signature against the directory's JWKS is a later step. + */ +export function detectDeclaredAgent(headers: HeaderReader): DeclaredAgent | null { + const raw = headers.get("signature-agent"); + if (!raw) return null; + const value = raw.trim().replace(/^"+|"+$/g, "").trim(); + if (!value) return null; + let host = value; + try { + host = new URL(value.includes("://") ? value : `https://${value}`).hostname; + } catch { + // keep the raw value; it is still evidence + } + host = host.toLowerCase().slice(0, 100); + if (!host) return null; + const signed = Boolean(headers.get("signature-input") && headers.get("signature")); + return { directory: host, signed }; +} + +export function botKind(botName: string, declared: DeclaredAgent | null): BotKind { + if (declared) return "declared_agent"; + if (botName === "Unknown") return "unknown_bot"; + if (AGENT_FETCHERS.has(botName)) return "agent_fetch"; + return "crawler"; +} + +/** A declared agent whose User-Agent matched no pattern still counts, under its directory host. */ +export function declaredAgentBot(declared: DeclaredAgent): BotMatch { + return { name: `agent:${declared.directory}`, company: declared.directory, type: "ai" }; +} + export interface TrackBotOptions { /** Full webhook URL, e.g. https://apex.isimplifyme.com/api/webhooks/bot-hit */ webhookUrl: string; @@ -138,7 +212,9 @@ export function trackBot( if (!options.webhookSecret) return; const ua = req.headers.get("user-agent"); - const bot = detectBot(ua) ?? detectUnknownBot(ua); + // A Web Bot Auth signature makes the request a bot hit whatever the UA says. + const declared = detectDeclaredAgent(req.headers); + const bot = detectBot(ua) ?? detectUnknownBot(ua) ?? (declared ? declaredAgentBot(declared) : null); if (!bot) return; const url = new URL(req.url); @@ -159,6 +235,10 @@ export function trackBot( referer: referer || null, region: options.region || "", unknown: bot.name === "Unknown", + // Who the bot is (2026-09): crawler | agent_fetch | declared_agent | unknown_bot + kind: botKind(bot.name, declared), + signed: declared?.signed ?? false, + agentDirectory: declared?.directory ?? null, }; event.waitUntil( @@ -261,7 +341,7 @@ export function trackAiReferrerVisit( const ua = req.headers.get("user-agent"); // Skip bots — those are tracked by trackBot(), not here. - if (detectBot(ua) || detectUnknownBot(ua)) return; + if (detectBot(ua) || detectUnknownBot(ua) || detectDeclaredAgent(req.headers)) return; // Require a real browser UA (Mozilla/5.0 prefix is ubiquitous for browsers) if (!ua || !ua.includes("Mozilla")) return;