feat(bot-middleware): label every bot hit with who the bot is; declared agents via Web Bot Auth (1.6.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@isimplifyme/ui",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "React/Next.js UI library for iSimplifyMe properties \u2014 design system, article layouts, SEO helpers, bot middleware, and the iSM Concierge widget.",
   "homepage": "https://isimplifyme.com",
   "type": "module",

src/bot-middleware.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect, vi } from "vitest";
+import {
+  AGENT_FETCHERS,
+  botKind,
+  declaredAgentBot,
+  detectBot,
+  detectDeclaredAgent,
+  trackBot,
+} from "./bot-middleware";
+
+const headers = (map: Record<string, string>) => ({
+  get: (name: string) => map[name.toLowerCase()] ?? null,
+});
+
+describe("detectDeclaredAgent (Web Bot Auth)", () => {
+  it("reads a quoted Signature-Agent URL and the signature headers", () => {
+    expect(
+      detectDeclaredAgent(headers({
+        "signature-agent": '"https://chatgpt.com"',
+        "signature-input": 'sig1=("@authority" "signature-agent");created=1758600000;keyid="k1"',
+        signature: "sig1=:abc=:",
+      })),
+    ).toEqual({ directory: "chatgpt.com", signed: true });
+  });
+  it("accepts a bare host and reports unsigned when the signature is missing", () => {
+    expect(detectDeclaredAgent(headers({ "signature-agent": "Agent.Example.com" })))
+      .toEqual({ directory: "agent.example.com", signed: false });
+  });
+  it("is null without the header", () => {
+    expect(detectDeclaredAgent(headers({ "user-agent": "Mozilla/5.0" }))).toBeNull();
+    expect(detectDeclaredAgent(headers({ "signature-agent": '""' }))).toBeNull();
+  });
+});
+
+describe("botKind", () => {
+  it("declared beats every User-Agent class", () => {
+    const d = { directory: "chatgpt.com", signed: true };
+    expect(botKind("GPTBot", d)).toBe("declared_agent");
+    expect(botKind("Unknown", d)).toBe("declared_agent");
+  });
+  it("fetchers are agent_fetch; other bots are crawlers; Unknown is unknown_bot", () => {
+    for (const name of AGENT_FETCHERS) expect(botKind(name, null)).toBe("agent_fetch");
+    expect(botKind("GPTBot", null)).toBe("crawler");
+    expect(botKind("Meta-ExternalAgent", null)).toBe("crawler");
+    expect(botKind("Googlebot", null)).toBe("crawler");
+    expect(botKind("Unknown", null)).toBe("unknown_bot");
+  });
+});
+
+describe("new fetcher patterns", () => {
+  it("recognizes the user-triggered fetchers by name", () => {
+    expect(detectBot("Mozilla/5.0 (compatible; Claude-User/1.0; +claude.ai)")?.name).toBe("Claude-User");
+    expect(detectBot("Perplexity-User/1.0")?.name).toBe("Perplexity-User");
+    expect(detectBot("MistralAI-User/1.0")?.name).toBe("MistralAI-User");
+    expect(detectBot("DuckAssistBot/1.0")?.name).toBe("DuckAssistBot");
+    expect(detectBot("Mozilla/5.0 Claude-SearchBot")?.name).toBe("Claude-SearchBot");
+    // ClaudeBot and PerplexityBot still win for their own UAs
+    expect(detectBot("ClaudeBot/1.0")?.name).toBe("ClaudeBot");
+    expect(detectBot("PerplexityBot/1.0")?.name).toBe("PerplexityBot");
+  });
+});
+
+describe("trackBot payload", () => {
+  const fakeReq = (ua: string | null, extra: Record<string, string> = {}) => ({
+    url: "https://example.com/pricing",
+    headers: headers({ ...(ua ? { "user-agent": ua } : {}), ...extra }),
+  });
+  const run = (req: unknown) => {
+    const fetchMock = vi.fn(() => Promise.resolve(new Response("{}")));
+    vi.stubGlobal("fetch", fetchMock);
+    const waited: Promise<unknown>[] = [];
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    trackBot(req as any, { waitUntil: (p: Promise<unknown>) => waited.push(p) } as any, {
+      webhookUrl: "https://apex.example/api/webhooks/bot-hit",
+      webhookSecret: "s",
+    });
+    vi.unstubAllGlobals();
+    if (!fetchMock.mock.calls.length) return null;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    return JSON.parse((fetchMock.mock.calls[0] as any)[1].body);
+  };
+
+  it("labels a fetcher as agent_fetch", () => {
+    const body = run(fakeReq("Mozilla/5.0 ChatGPT-User/1.0"));
+    expect(body).toMatchObject({ bot: "ChatGPT-User", kind: "agent_fetch", signed: false, agentDirectory: null });
+  });
+  it("labels a crawler as crawler", () => {
+    expect(run(fakeReq("GPTBot/1.0"))).toMatchObject({ bot: "GPTBot", kind: "crawler" });
+  });
+  it("counts a signing agent with a browser UA under its directory host", () => {
+    const body = run(fakeReq("Mozilla/5.0 (Macintosh) Chrome/130", {
+      "signature-agent": '"https://muse.meta.com"',
+      "signature-input": "sig1=()",
+      signature: "sig1=:x:",
+    }));
+    expect(body).toMatchObject({
+      bot: "agent:muse.meta.com", type: "ai", kind: "declared_agent", signed: true, agentDirectory: "muse.meta.com",
+    });
+  });
+  it("still ignores a plain browser", () => {
+    expect(run(fakeReq("Mozilla/5.0 (Macintosh) Chrome/130"))).toBeNull();
+  });
+  it("declaredAgentBot shape", () => {
+    expect(declaredAgentBot({ directory: "chatgpt.com", signed: true }))
+      .toEqual({ name: "agent:chatgpt.com", company: "chatgpt.com", type: "ai" });
+  });
+});

src/bot-middleware.ts

@@ -31,15 +31,20 @@ interface BotPattern extends BotMatch {
   pattern: RegExp;
 }
 
-/** 48 bots across 6 categories — kept in sync with apex-portal src/lib/bots/parser.ts */
+/** 53 bots across 6 categories — kept in sync with apex-portal src/lib/bots/parser.ts and each site's inlined middleware */
 export const BOT_PATTERNS: readonly BotPattern[] = [
   // AI crawlers
   { pattern: /GPTBot/i, name: "GPTBot", company: "OpenAI", type: "ai" },
   { pattern: /ChatGPT-User/i, name: "ChatGPT-User", company: "OpenAI", type: "ai" },
   { pattern: /OAI-SearchBot/i, name: "OAI-SearchBot", company: "OpenAI", type: "ai" },
   { pattern: /ClaudeBot/i, name: "ClaudeBot", company: "Anthropic", type: "ai" },
   { pattern: /anthropic-ai/i, name: "anthropic-ai", company: "Anthropic", type: "ai" },
+  { pattern: /Claude-User/i, name: "Claude-User", company: "Anthropic", type: "ai" },
+  { pattern: /Claude-SearchBot/i, name: "Claude-SearchBot", company: "Anthropic", type: "ai" },
   { pattern: /PerplexityBot/i, name: "PerplexityBot", company: "Perplexity", type: "ai" },
+  { pattern: /Perplexity-User/i, name: "Perplexity-User", company: "Perplexity", type: "ai" },
+  { pattern: /MistralAI-User/i, name: "MistralAI-User", company: "Mistral", type: "ai" },
+  { pattern: /DuckAssistBot/i, name: "DuckAssistBot", company: "DuckDuckGo", type: "ai" },
   { pattern: /Google-Extended/i, name: "Google-Extended", company: "Google", type: "ai" },
   { pattern: /Bytespider/i, name: "Bytespider", company: "ByteDance", type: "ai" },
   { pattern: /Meta-ExternalAgent/i, name: "Meta-ExternalAgent", company: "Meta", type: "ai" },
@@ -109,6 +114,75 @@ export function detectUnknownBot(userAgent: string | null | undefined): BotMatch
   return null;
 }
 
+// ─── Who is the bot? (2026-09) ─────────────────────────────────────────────
+//
+// Every bot hit carries a `kind` beside its name and category:
+//   crawler        — training/index crawlers and every other classic bot
+//   agent_fetch    — an AI assistant fetching because a person asked it
+//                    something right now (ChatGPT-User, Claude-User, …)
+//   declared_agent — the request carries a Web Bot Auth signature (RFC 9421
+//                    HTTP Message Signatures): the agent named its key
+//                    directory in `Signature-Agent`, whatever its User-Agent
+//                    says. The only honest "an agent is here" signal on the
+//                    web for zones without Cloudflare Bot Management.
+//   unknown_bot    — bot-like User-Agent we have no pattern for
+
+export type BotKind = "crawler" | "agent_fetch" | "declared_agent" | "unknown_bot";
+
+/** Bots that fetch because a person asked an AI assistant something right now. */
+export const AGENT_FETCHERS: ReadonlySet<string> = new Set([
+  "ChatGPT-User",
+  "meta-externalfetcher",
+  "Claude-User",
+  "Perplexity-User",
+  "MistralAI-User",
+  "DuckAssistBot",
+]);
+
+export interface DeclaredAgent {
+  /** Host of the agent's key directory, from `Signature-Agent` (e.g. "chatgpt.com"). */
+  directory: string;
+  /** Whether the request also carried `Signature-Input` and `Signature`. */
+  signed: boolean;
+}
+
+type HeaderReader = { get(name: string): string | null };
+
+/**
+ * Web Bot Auth: a signing agent sends `Signature-Agent: "https://host"` (an
+ * sf-string, so quoted) plus `Signature-Input` and `Signature`. We record the
+ * directory host and whether the signature headers are present; verifying the
+ * signature against the directory's JWKS is a later step.
+ */
+export function detectDeclaredAgent(headers: HeaderReader): DeclaredAgent | null {
+  const raw = headers.get("signature-agent");
+  if (!raw) return null;
+  const value = raw.trim().replace(/^"+|"+$/g, "").trim();
+  if (!value) return null;
+  let host = value;
+  try {
+    host = new URL(value.includes("://") ? value : `https://${value}`).hostname;
+  } catch {
+    // keep the raw value; it is still evidence
+  }
+  host = host.toLowerCase().slice(0, 100);
+  if (!host) return null;
+  const signed = Boolean(headers.get("signature-input") && headers.get("signature"));
+  return { directory: host, signed };
+}
+
+export function botKind(botName: string, declared: DeclaredAgent | null): BotKind {
+  if (declared) return "declared_agent";
+  if (botName === "Unknown") return "unknown_bot";
+  if (AGENT_FETCHERS.has(botName)) return "agent_fetch";
+  return "crawler";
+}
+
+/** A declared agent whose User-Agent matched no pattern still counts, under its directory host. */
+export function declaredAgentBot(declared: DeclaredAgent): BotMatch {
+  return { name: `agent:${declared.directory}`, company: declared.directory, type: "ai" };
+}
+
 export interface TrackBotOptions {
   /** Full webhook URL, e.g. https://apex.isimplifyme.com/api/webhooks/bot-hit */
   webhookUrl: string;
@@ -138,7 +212,9 @@ export function trackBot(
   if (!options.webhookSecret) return;
 
   const ua = req.headers.get("user-agent");
-  const bot = detectBot(ua) ?? detectUnknownBot(ua);
+  // A Web Bot Auth signature makes the request a bot hit whatever the UA says.
+  const declared = detectDeclaredAgent(req.headers);
+  const bot = detectBot(ua) ?? detectUnknownBot(ua) ?? (declared ? declaredAgentBot(declared) : null);
   if (!bot) return;
 
   const url = new URL(req.url);
@@ -159,6 +235,10 @@ export function trackBot(
     referer: referer || null,
     region: options.region || "",
     unknown: bot.name === "Unknown",
+    // Who the bot is (2026-09): crawler | agent_fetch | declared_agent | unknown_bot
+    kind: botKind(bot.name, declared),
+    signed: declared?.signed ?? false,
+    agentDirectory: declared?.directory ?? null,
   };
 
   event.waitUntil(
@@ -261,7 +341,7 @@ export function trackAiReferrerVisit(
 
   const ua = req.headers.get("user-agent");
   // Skip bots — those are tracked by trackBot(), not here.
-  if (detectBot(ua) || detectUnknownBot(ua)) return;
+  if (detectBot(ua) || detectUnknownBot(ua) || detectDeclaredAgent(req.headers)) return;
   // Require a real browser UA (Mozilla/5.0 prefix is ubiquitous for browsers)
   if (!ua || !ua.includes("Mozilla")) return;