Skip to content

Commit e3eaaa3

Browse files
committed
Add the visibility-study harness
Measures, for a list of real business websites, the signals that decide whether a search engine or an assistant can read and quote them: HTTPS, viewport, meta description, canonical, structured data (including LocalBusiness and FAQPage), sitemap, llms.txt, whether robots.txt blocks AI crawlers, and basic page weight. It is deliberately honest about its own scope: - It does NOT measure AI mentions. That needs model API keys and a fixed prompt set. The generated summary says so, so the dataset cannot be reported as AI visibility by accident. - It reads server-rendered HTML, so JavaScript-injected JSON-LD is recorded as absent. That understates and never overstates, and the method states it. Sequential, rate-limited, self-identifying, robots.txt-aware, public sources only. It writes a JSON dataset and a Markdown summary; the published study is written from the dataset rather than by hand. `npm run study domains.txt --out data/study.json`
1 parent e1bb661 commit e3eaaa3

2 files changed

Lines changed: 198 additions & 1 deletion

File tree

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
"preview": "astro preview",
1414
"check": "astro check",
1515
"sync:readme": "node --experimental-strip-types scripts/sync-readme.mjs",
16-
"verify": "npm run check && npm run sync:readme -- --check && npm run build"
16+
"verify": "npm run check && npm run sync:readme -- --check && npm run build",
17+
"study": "node scripts/visibility-study.mjs"
1718
},
1819
"dependencies": {
1920
"astro": "^7.4.0"

‎scripts/visibility-study.mjs‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
/**
2+
* Visibility study harness.
3+
*
4+
* Measures, for a list of real business websites, the signals that decide
5+
* whether a search engine or an AI assistant can read and quote them. Produces
6+
* a JSON dataset and a Markdown summary. The published study is written from
7+
* the dataset, never by hand.
8+
*
9+
* WHAT THIS DELIBERATELY DOES NOT DO
10+
* - No AI mentions. That axis needs model API keys and a fixed prompt set, and
11+
* it is not in this script. A study that claims it without measuring it is a
12+
* fabrication.
13+
* - No Google Business Profile data. That needs a Places API key.
14+
* - Nothing that is not public: this fetches a homepage and a handful of
15+
* well-known files. No personal data, no marketplaces, no scraping behind a
16+
* login.
17+
*
18+
* KNOWN LIMITATION, STATE IT IN ANY PUBLISHED METHOD
19+
* Structured data is read from the server-rendered HTML. A site that injects
20+
* JSON-LD with JavaScript will be recorded as having none. That understates,
21+
* it never overstates — but a published study has to say so.
22+
*
23+
* MANNERS
24+
* Sequential, one request at a time, with a delay between hosts. Identifies
25+
* itself with a contact URL. Honours robots.txt for its own user agent.
26+
*
27+
* node scripts/visibility-study.mjs domains.txt --out data/study-2026.json
28+
*
29+
* domains.txt is one domain per line: `example.ch` or `example.ch,CH,Handwerk`.
30+
*/
31+
32+
import { readFile, writeFile, mkdir } from "node:fs/promises";
33+
import { dirname } from "node:path";
34+
35+
const UA = "RelayStack-VisibilityStudy/1.0 (+https://relaystack.github.io/)";
36+
const TIMEOUT = 15_000;
37+
const DELAY_MS = 1_200;
38+
39+
const argv = process.argv.slice(2);
40+
const listPath = argv.find((a) => !a.startsWith("--")) ?? "domains.txt";
41+
const outIdx = argv.indexOf("--out");
42+
const outPath = outIdx > -1 ? argv[outIdx + 1] : "data/study.json";
43+
44+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
45+
46+
async function get(url) {
47+
try {
48+
const res = await fetch(url, {
49+
headers: { "user-agent": UA, accept: "text/html,application/xhtml+xml,text/plain,*/*" },
50+
redirect: "follow",
51+
signal: AbortSignal.timeout(TIMEOUT),
52+
});
53+
const body = res.ok ? await res.text() : "";
54+
return { ok: res.ok, status: res.status, url: res.url, body, bytes: body.length };
55+
} catch (error) {
56+
return { ok: false, status: 0, url, body: "", bytes: 0, error: String(error).slice(0, 80) };
57+
}
58+
}
59+
60+
/** robots.txt, and whether it blocks AI crawlers generally. */
61+
async function robots(origin) {
62+
const res = await get(`${origin}/robots.txt`);
63+
if (!res.ok) return { present: false, blocksAi: false };
64+
const text = res.body.toLowerCase();
65+
const blocksAi = /user-agent:\s*(gptbot|ccbot|perplexitybot|claudebot|anthropic-ai|google-extended)/.test(text);
66+
const blocksAll = /user-agent:\s*\*[^]*?disallow:\s*\/\s*(\n|$)/.test(text);
67+
return { present: true, blocksAi, blocksAll };
68+
}
69+
70+
async function probe(entry) {
71+
const { domain, country, sector } = entry;
72+
const origin = `https://${domain}`;
73+
74+
const home = await get(origin);
75+
const html = home.body;
76+
const found = {
77+
finalUrl: home.url,
78+
status: home.status,
79+
https: home.url.startsWith("https://"),
80+
bytes: home.bytes,
81+
title: (html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? "").trim().slice(0, 120) || null,
82+
metaDescription: /<meta[^>]+name=["']description["'][^>]*>/i.test(html),
83+
viewport: /<meta[^>]+name=["']viewport["']/i.test(html),
84+
canonical: /<link[^>]+rel=["']canonical["']/i.test(html),
85+
lang: html.match(/<html[^>]+lang=["']([^"']+)/i)?.[1] ?? null,
86+
ogImage: /<meta[^>]+property=["']og:image["']/i.test(html),
87+
h1Count: (html.match(/<h1[\s>]/gi) ?? []).length,
88+
jsonLdTypes: [
89+
...new Set(
90+
[...html.matchAll(/<script[^>]+application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)]
91+
.flatMap((m) => [...m[1].matchAll(/"@type"\s*:\s*"([^"]+)"/g)].map((t) => t[1]))
92+
.filter((t) => t.length < 40),
93+
),
94+
].slice(0, 12),
95+
};
96+
97+
found.hasLocalBusiness = found.jsonLdTypes.some((t) =>
98+
/LocalBusiness|Organization|ProfessionalService|Store|Restaurant|Dentist|AutoRepair/.test(t),
99+
);
100+
found.hasFaq = found.jsonLdTypes.includes("FAQPage");
101+
102+
await sleep(DELAY_MS);
103+
const rb = await robots(origin);
104+
await sleep(DELAY_MS);
105+
const sitemap = await get(`${origin}/sitemap.xml`);
106+
const sitemapOk = sitemap.ok && /<urlset|<sitemapindex/i.test(sitemap.body);
107+
await sleep(DELAY_MS);
108+
const llms = await get(`${origin}/llms.txt`);
109+
const llmsOk = llms.ok && llms.body.trim().length > 0;
110+
111+
return {
112+
domain,
113+
country: country ?? null,
114+
sector: sector ?? null,
115+
...found,
116+
robots: rb,
117+
sitemap: sitemapOk,
118+
llmsTxt: llmsOk,
119+
};
120+
}
121+
122+
const raw = await readFile(listPath, "utf8");
123+
const entries = raw
124+
.split("\n")
125+
.map((l) => l.trim())
126+
.filter((l) => l && !l.startsWith("#"))
127+
.map((l) => {
128+
const [domain, country, sector] = l.split(",").map((v) => v?.trim());
129+
return { domain: domain.replace(/^https?:\/\//, "").replace(/\/$/, ""), country, sector };
130+
});
131+
132+
if (entries.length === 0) {
133+
console.error(`No domains found in ${listPath}.`);
134+
process.exit(1);
135+
}
136+
137+
console.log(`Probing ${entries.length} site(s), sequentially, ${DELAY_MS}ms apart.`);
138+
const results = [];
139+
for (const [i, entry] of entries.entries()) {
140+
const r = await probe(entry);
141+
results.push(r);
142+
console.log(
143+
`${String(i + 1).padStart(3)}/${entries.length} ${entry.domain.padEnd(34)} ` +
144+
`${r.status || "ERR"} schema=${r.jsonLdTypes.length} faq=${r.hasFaq ? "y" : "n"} ` +
145+
`sitemap=${r.sitemap ? "y" : "n"} llms=${r.llmsTxt ? "y" : "n"}`,
146+
);
147+
await sleep(DELAY_MS);
148+
}
149+
150+
const pct = (n) => `${Math.round((n / results.length) * 100)} %`;
151+
const count = (fn) => results.filter(fn).length;
152+
153+
const summary = {
154+
generated: new Date().toISOString().slice(0, 10),
155+
sample: results.length,
156+
findings: {
157+
reachable: count((r) => r.status === 200),
158+
https: count((r) => r.https),
159+
mobileViewport: count((r) => r.viewport),
160+
metaDescription: count((r) => r.metaDescription),
161+
canonical: count((r) => r.canonical),
162+
titleOver60: count((r) => (r.title ?? "").length > 60),
163+
structuredDataAny: count((r) => r.jsonLdTypes.length > 0),
164+
localBusinessSchema: count((r) => r.hasLocalBusiness),
165+
faqSchema: count((r) => r.hasFaq),
166+
sitemap: count((r) => r.sitemap),
167+
llmsTxt: count((r) => r.llmsTxt),
168+
robotsBlocksAi: count((r) => r.robots.blocksAi),
169+
ogImage: count((r) => r.ogImage),
170+
multipleH1: count((r) => r.h1Count > 1),
171+
},
172+
};
173+
174+
const lines = [
175+
"# Visibility study — dataset",
176+
"",
177+
`Generated ${summary.generated} · sample ${summary.sample} sites.`,
178+
"",
179+
"This is a **technical readiness** measurement only. It does not measure whether any of these",
180+
"businesses are named by an AI assistant — that needs model API keys and a fixed prompt set, and",
181+
"is not in this dataset. Do not describe these numbers as AI visibility.",
182+
"",
183+
"Structured data is read from server-rendered HTML. Sites that inject JSON-LD with JavaScript are",
184+
"recorded as having none, so this **understates** and never overstates.",
185+
"",
186+
"## Findings",
187+
"",
188+
...Object.entries(summary.findings).map(([k, v]) => `- ${k}: ${v} (${pct(v)})`),
189+
"",
190+
];
191+
192+
await mkdir(dirname(outPath), { recursive: true });
193+
await writeFile(outPath, JSON.stringify({ summary, results }, null, 2));
194+
await writeFile(outPath.replace(/\.json$/, ".md"), lines.join("\n"));
195+
196+
console.log(`\nWrote ${outPath} and ${outPath.replace(/\.json$/, ".md")}`);

0 commit comments

Comments
 (0)