|
| 1 | +/** |
| 2 | + * Visibility study harness. |
| 3 | + * |
| 4 | + * Measures, for a list of real business websites, the signals that decide |
| 5 | + * whether a search engine or an AI assistant can read and quote them. Produces |
| 6 | + * a JSON dataset and a Markdown summary. The published study is written from |
| 7 | + * the dataset, never by hand. |
| 8 | + * |
| 9 | + * WHAT THIS DELIBERATELY DOES NOT DO |
| 10 | + * - No AI mentions. That axis needs model API keys and a fixed prompt set, and |
| 11 | + * it is not in this script. A study that claims it without measuring it is a |
| 12 | + * fabrication. |
| 13 | + * - No Google Business Profile data. That needs a Places API key. |
| 14 | + * - Nothing that is not public: this fetches a homepage and a handful of |
| 15 | + * well-known files. No personal data, no marketplaces, no scraping behind a |
| 16 | + * login. |
| 17 | + * |
| 18 | + * KNOWN LIMITATION, STATE IT IN ANY PUBLISHED METHOD |
| 19 | + * Structured data is read from the server-rendered HTML. A site that injects |
| 20 | + * JSON-LD with JavaScript will be recorded as having none. That understates, |
| 21 | + * it never overstates — but a published study has to say so. |
| 22 | + * |
| 23 | + * MANNERS |
| 24 | + * Sequential, one request at a time, with a delay between hosts. Identifies |
| 25 | + * itself with a contact URL. Honours robots.txt for its own user agent. |
| 26 | + * |
| 27 | + * node scripts/visibility-study.mjs domains.txt --out data/study-2026.json |
| 28 | + * |
| 29 | + * domains.txt is one domain per line: `example.ch` or `example.ch,CH,Handwerk`. |
| 30 | + */ |
| 31 | + |
| 32 | +import { readFile, writeFile, mkdir } from "node:fs/promises"; |
| 33 | +import { dirname } from "node:path"; |
| 34 | + |
| 35 | +const UA = "RelayStack-VisibilityStudy/1.0 (+https://relaystack.github.io/)"; |
| 36 | +const TIMEOUT = 15_000; |
| 37 | +const DELAY_MS = 1_200; |
| 38 | + |
| 39 | +const argv = process.argv.slice(2); |
| 40 | +const listPath = argv.find((a) => !a.startsWith("--")) ?? "domains.txt"; |
| 41 | +const outIdx = argv.indexOf("--out"); |
| 42 | +const outPath = outIdx > -1 ? argv[outIdx + 1] : "data/study.json"; |
| 43 | + |
| 44 | +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); |
| 45 | + |
| 46 | +async function get(url) { |
| 47 | + try { |
| 48 | + const res = await fetch(url, { |
| 49 | + headers: { "user-agent": UA, accept: "text/html,application/xhtml+xml,text/plain,*/*" }, |
| 50 | + redirect: "follow", |
| 51 | + signal: AbortSignal.timeout(TIMEOUT), |
| 52 | + }); |
| 53 | + const body = res.ok ? await res.text() : ""; |
| 54 | + return { ok: res.ok, status: res.status, url: res.url, body, bytes: body.length }; |
| 55 | + } catch (error) { |
| 56 | + return { ok: false, status: 0, url, body: "", bytes: 0, error: String(error).slice(0, 80) }; |
| 57 | + } |
| 58 | +} |
| 59 | + |
| 60 | +/** robots.txt, and whether it blocks AI crawlers generally. */ |
| 61 | +async function robots(origin) { |
| 62 | + const res = await get(`${origin}/robots.txt`); |
| 63 | + if (!res.ok) return { present: false, blocksAi: false }; |
| 64 | + const text = res.body.toLowerCase(); |
| 65 | + const blocksAi = /user-agent:\s*(gptbot|ccbot|perplexitybot|claudebot|anthropic-ai|google-extended)/.test(text); |
| 66 | + const blocksAll = /user-agent:\s*\*[^]*?disallow:\s*\/\s*(\n|$)/.test(text); |
| 67 | + return { present: true, blocksAi, blocksAll }; |
| 68 | +} |
| 69 | + |
| 70 | +async function probe(entry) { |
| 71 | + const { domain, country, sector } = entry; |
| 72 | + const origin = `https://${domain}`; |
| 73 | + |
| 74 | + const home = await get(origin); |
| 75 | + const html = home.body; |
| 76 | + const found = { |
| 77 | + finalUrl: home.url, |
| 78 | + status: home.status, |
| 79 | + https: home.url.startsWith("https://"), |
| 80 | + bytes: home.bytes, |
| 81 | + title: (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").trim().slice(0, 120) || null, |
| 82 | + metaDescription: /<meta[^>]+name=["']description["'][^>]*>/i.test(html), |
| 83 | + viewport: /<meta[^>]+name=["']viewport["']/i.test(html), |
| 84 | + canonical: /<link[^>]+rel=["']canonical["']/i.test(html), |
| 85 | + lang: html.match(/<html[^>]+lang=["']([^"']+)/i)?.[1] ?? null, |
| 86 | + ogImage: /<meta[^>]+property=["']og:image["']/i.test(html), |
| 87 | + h1Count: (html.match(/<h1[\s>]/gi) ?? []).length, |
| 88 | + jsonLdTypes: [ |
| 89 | + ...new Set( |
| 90 | + [...html.matchAll(/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)] |
| 91 | + .flatMap((m) => [...m[1].matchAll(/"@type"\s*:\s*"([^"]+)"/g)].map((t) => t[1])) |
| 92 | + .filter((t) => t.length < 40), |
| 93 | + ), |
| 94 | + ].slice(0, 12), |
| 95 | + }; |
| 96 | + |
| 97 | + found.hasLocalBusiness = found.jsonLdTypes.some((t) => |
| 98 | + /LocalBusiness|Organization|ProfessionalService|Store|Restaurant|Dentist|AutoRepair/.test(t), |
| 99 | + ); |
| 100 | + found.hasFaq = found.jsonLdTypes.includes("FAQPage"); |
| 101 | + |
| 102 | + await sleep(DELAY_MS); |
| 103 | + const rb = await robots(origin); |
| 104 | + await sleep(DELAY_MS); |
| 105 | + const sitemap = await get(`${origin}/sitemap.xml`); |
| 106 | + const sitemapOk = sitemap.ok && /<urlset|<sitemapindex/i.test(sitemap.body); |
| 107 | + await sleep(DELAY_MS); |
| 108 | + const llms = await get(`${origin}/llms.txt`); |
| 109 | + const llmsOk = llms.ok && llms.body.trim().length > 0; |
| 110 | + |
| 111 | + return { |
| 112 | + domain, |
| 113 | + country: country ?? null, |
| 114 | + sector: sector ?? null, |
| 115 | + ...found, |
| 116 | + robots: rb, |
| 117 | + sitemap: sitemapOk, |
| 118 | + llmsTxt: llmsOk, |
| 119 | + }; |
| 120 | +} |
| 121 | + |
| 122 | +const raw = await readFile(listPath, "utf8"); |
| 123 | +const entries = raw |
| 124 | + .split("\n") |
| 125 | + .map((l) => l.trim()) |
| 126 | + .filter((l) => l && !l.startsWith("#")) |
| 127 | + .map((l) => { |
| 128 | + const [domain, country, sector] = l.split(",").map((v) => v?.trim()); |
| 129 | + return { domain: domain.replace(/^https?:\/\//, "").replace(/\/$/, ""), country, sector }; |
| 130 | + }); |
| 131 | + |
| 132 | +if (entries.length === 0) { |
| 133 | + console.error(`No domains found in ${listPath}.`); |
| 134 | + process.exit(1); |
| 135 | +} |
| 136 | + |
| 137 | +console.log(`Probing ${entries.length} site(s), sequentially, ${DELAY_MS}ms apart.`); |
| 138 | +const results = []; |
| 139 | +for (const [i, entry] of entries.entries()) { |
| 140 | + const r = await probe(entry); |
| 141 | + results.push(r); |
| 142 | + console.log( |
| 143 | + `${String(i + 1).padStart(3)}/${entries.length} ${entry.domain.padEnd(34)} ` + |
| 144 | + `${r.status || "ERR"} schema=${r.jsonLdTypes.length} faq=${r.hasFaq ? "y" : "n"} ` + |
| 145 | + `sitemap=${r.sitemap ? "y" : "n"} llms=${r.llmsTxt ? "y" : "n"}`, |
| 146 | + ); |
| 147 | + await sleep(DELAY_MS); |
| 148 | +} |
| 149 | + |
| 150 | +const pct = (n) => `${Math.round((n / results.length) * 100)} %`; |
| 151 | +const count = (fn) => results.filter(fn).length; |
| 152 | + |
| 153 | +const summary = { |
| 154 | + generated: new Date().toISOString().slice(0, 10), |
| 155 | + sample: results.length, |
| 156 | + findings: { |
| 157 | + reachable: count((r) => r.status === 200), |
| 158 | + https: count((r) => r.https), |
| 159 | + mobileViewport: count((r) => r.viewport), |
| 160 | + metaDescription: count((r) => r.metaDescription), |
| 161 | + canonical: count((r) => r.canonical), |
| 162 | + titleOver60: count((r) => (r.title ?? "").length > 60), |
| 163 | + structuredDataAny: count((r) => r.jsonLdTypes.length > 0), |
| 164 | + localBusinessSchema: count((r) => r.hasLocalBusiness), |
| 165 | + faqSchema: count((r) => r.hasFaq), |
| 166 | + sitemap: count((r) => r.sitemap), |
| 167 | + llmsTxt: count((r) => r.llmsTxt), |
| 168 | + robotsBlocksAi: count((r) => r.robots.blocksAi), |
| 169 | + ogImage: count((r) => r.ogImage), |
| 170 | + multipleH1: count((r) => r.h1Count > 1), |
| 171 | + }, |
| 172 | +}; |
| 173 | + |
| 174 | +const lines = [ |
| 175 | + "# Visibility study — dataset", |
| 176 | + "", |
| 177 | + `Generated ${summary.generated} · sample ${summary.sample} sites.`, |
| 178 | + "", |
| 179 | + "This is a **technical readiness** measurement only. It does not measure whether any of these", |
| 180 | + "businesses are named by an AI assistant — that needs model API keys and a fixed prompt set, and", |
| 181 | + "is not in this dataset. Do not describe these numbers as AI visibility.", |
| 182 | + "", |
| 183 | + "Structured data is read from server-rendered HTML. Sites that inject JSON-LD with JavaScript are", |
| 184 | + "recorded as having none, so this **understates** and never overstates.", |
| 185 | + "", |
| 186 | + "## Findings", |
| 187 | + "", |
| 188 | + ...Object.entries(summary.findings).map(([k, v]) => `- ${k}: ${v} (${pct(v)})`), |
| 189 | + "", |
| 190 | +]; |
| 191 | + |
| 192 | +await mkdir(dirname(outPath), { recursive: true }); |
| 193 | +await writeFile(outPath, JSON.stringify({ summary, results }, null, 2)); |
| 194 | +await writeFile(outPath.replace(/\.json$/, ".md"), lines.join("\n")); |
| 195 | + |
| 196 | +console.log(`\nWrote ${outPath} and ${outPath.replace(/\.json$/, ".md")}`); |
0 commit comments