Search engine sweet

.github/workflows/deploy.yml

@@ -132,6 +132,14 @@ jobs:
           fi
           bun run discovery:key
 
+      - name: Validate the discovery artifacts, including the key file
+        # The build pipeline cannot require the key verification file (the build
+        # never sees the key); this is the one place it is guaranteed to exist.
+        # When INDEXNOW_KEY is unset the requirement is a no-op.
+        env:
+          INDEXNOW_KEY: ${{ secrets.INDEXNOW_KEY }}
+        run: bun run discovery:validate --require-indexnow-key
+
       - name: Publish the previous discovery manifest for change detection
         run: |
           mkdir -p src/dist/_discovery

README.md

@@ -388,18 +388,39 @@ site URL automatically:
   `https://<owner>.github.io`.
 - **Project Pages** (`<owner>/<repo>`): base `/<repo>`, site
   `https://<owner>.github.io`.
-- **Custom domain** (recommended for `purview.dev`): once DNS is configured at
-  your registrar (CNAME/ALIAS to `purview-dev.github.io`, or A/AAAA per GitHub's
-  guidance), set repository **variables** `SITE_URL=https://purview.dev` and
-  `PAGES_BASE=/`, and restore a `public/CNAME` file containing `purview.dev`.
+- **Custom domain** (`purview.dev`): once DNS is configured at your registrar
+  (CNAME/ALIAS to `purview-dev.github.io`, or A/AAAA per GitHub's guidance), set
+  the repository **variables** `SITE_URL=https://purview.dev` and `PAGES_BASE=/`,
+  and configure the custom domain in **Settings → Pages → Custom domain**. The
+  domain is managed through the Pages settings for the GitHub Actions
+  deployment; no `public/CNAME` file is required.
 
 Configuration notes:
 
 - Enable Pages with **Source: GitHub Actions** in the repository settings and
   create the `github-pages` environment.
-- The `public/CNAME` file is intentionally absent until the `purview.dev` DNS
-  records are configured, so the site stays reachable at the root of
-  `purview-dev.github.io`.
+- The custom domain is configured in the repository's Pages settings
+  (**Settings → Pages → Custom domain**), not by a committed `public/CNAME`
+  file, so no `CNAME` needs to be maintained in the source tree.
+
+### HTTPS for the custom domain
+
+HTTPS is enforced at the **Cloudflare** edge, not by GitHub Pages. Because
+`purview.dev` is proxied through Cloudflare (orange cloud), GitHub cannot
+complete the Let's Encrypt HTTP-01 challenge, so the repository's **Enforce
+HTTPS** toggle stays unavailable with the message *"your domain is not properly
+configured to support HTTPS"*. This is expected, not a misconfiguration:
+
+- Cloudflare's **Always Use HTTPS** setting redirects `http://purview.dev/…` to
+  `https://purview.dev/…`, so the canonical origin is always HTTPS.
+- The only side effect is that GitHub's default-domain redirect
+  (`purview-dev.github.io` → `purview.dev`) targets `http://`, adding one extra
+  hop before the HTTPS redirect. Search engines follow the chain to the
+  canonical URL and treat it as a redirect, not a duplicate.
+
+Do not try to force the GitHub toggle on: it would require temporarily setting
+the Cloudflare DNS record to **DNS only** (grey cloud) so GitHub can issue the
+certificate, and renewals can fail again once the record is proxied.
 
 The workflow validates (`just validate`), refreshes live release/documentation
 data, builds, and uploads `./src/dist`.

docs/discovery.md

@@ -62,6 +62,7 @@ accurate in CI.
 
 ```shell
 bun run discovery:validate                     # validate the built artifacts in dist/
+bun run discovery:validate --require-indexnow-key  # also require /<INDEXNOW_KEY>.txt
 bun run discovery:manifest                     # regenerate artifacts in dist/ from current caches
 bun run discovery:key                          # write /<INDEXNOW_KEY>.txt from the secret
 bun run discovery:indexnow --dry-run           # show what would be submitted (no network, no key)
@@ -75,8 +76,15 @@ bun run discovery:indexnow --dry-run --verbose # include the URL lists
 fails the build on: missing/malformed sitemaps, non-absolute or off-origin URLs,
 duplicate canonicals, a `robots.txt` that does not advertise the sitemap, a
 `discover.json` that references unknown projects, missing per-project LLM
-resources, development URLs in production output, and a missing IndexNow key file
-when the key is configured.
+resources, and development URLs in production output.
+
+The IndexNow key file is a **deploy-time** artifact: the build never sees the
+key, so the build pipeline does not require it. The deploy workflow runs
+`bun run discovery:key` and then
+`bun run discovery:validate --require-indexnow-key`, which additionally fails
+when `INDEXNOW_KEY` is configured but the key file is missing, or when the file
+does not contain the configured key. Run `bun run discovery:validate` locally
+without the flag and a configured key is simply ignored.
 
 ## IndexNow
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "purview-dev",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "private": true,
   "description": "Purview-Dev public website and unified documentation portal (workspace root).",
   "license": "MIT",

src/scripts/discovery/validate.ts

@@ -40,7 +40,53 @@ export interface DiscoveryValidationResult {
   errors: string[];
 }
 
-export function validateDiscovery(dist = resolve('dist')): DiscoveryValidationResult {
+export interface DiscoveryValidationOptions {
+  /**
+   * Require the IndexNow key verification file (`/<INDEXNOW_KEY>.txt`) when
+   * `INDEXNOW_KEY` is configured.
+   *
+   * The build never writes this file — it is written from the secret by the
+   * deploy workflow via `bun run discovery:key`, *after* the build — so the
+   * build pipeline (`just validate` / `ci:build`) cannot require it. Only the
+   * deploy pipeline, which has just written the file, opts in with
+   * `--require-indexnow-key`.
+   */
+  requireIndexNowKey?: boolean;
+}
+
+/**
+ * Validate the IndexNow key verification file (`/<key>.txt`).
+ *
+ * Returns the error messages to report: empty when the key is unset/invalid
+ * (nothing to check), when the file is present and correct, or when it is
+ * absent and not required. `require` is only set by the deploy pipeline, which
+ * writes the file after the build.
+ */
+export function validateIndexNowKeyFile(
+  readFile: (path: string) => string | null,
+  rawKey: string | undefined,
+  options: { require?: boolean } = {},
+): string[] {
+  const key = rawKey?.trim();
+  if (!key || !/^[A-Za-z0-9-]{8,128}$/.test(key)) {
+    return [];
+  }
+  const content = readFile(`/${key}.txt`);
+  if (content === null) {
+    return options.require
+      ? ['INDEXNOW_KEY is configured but the key verification file is missing from the build.']
+      : [];
+  }
+  if (content.trim() !== key) {
+    return ['The IndexNow key verification file does not contain the configured key.'];
+  }
+  return [];
+}
+
+export function validateDiscovery(
+  dist = resolve('dist'),
+  options: DiscoveryValidationOptions = {},
+): DiscoveryValidationResult {
   const errors: string[] = [];
   const fail = (message: string): void => {
     errors.push(message);
@@ -201,23 +247,24 @@ export function validateDiscovery(dist = resolve('dist')): DiscoveryValidationRe
     }
   }
 
-  // --- IndexNow key file (only when configured) ---------------------------
-  const indexNowKey = process.env.INDEXNOW_KEY?.trim();
-  if (indexNowKey && /^[A-Za-z0-9-]{8,128}$/.test(indexNowKey)) {
-    const content = readDist(`/${indexNowKey}.txt`);
-    if (content === null) {
-      fail('INDEXNOW_KEY is configured but the key verification file is missing from the build.');
-    } else if (content.trim() !== indexNowKey) {
-      fail('The IndexNow key verification file does not contain the configured key.');
-    }
+  // --- IndexNow key file --------------------------------------------------
+  // The build never writes this file (the key is a deploy-only secret), so its
+  // presence is only required when the caller opts in — the deploy workflow
+  // runs this after `discovery:key`. When the file is present it must always
+  // contain the configured key.
+  for (const error of validateIndexNowKeyFile(readDist, process.env.INDEXNOW_KEY, {
+    require: options.requireIndexNowKey,
+  })) {
+    fail(error);
   }
 
   return { errors };
 }
 
 if (import.meta.main) {
   console.log('Validating discovery artifacts...');
-  const { errors } = validateDiscovery();
+  const requireIndexNowKey = process.argv.includes('--require-indexnow-key');
+  const { errors } = validateDiscovery(resolve('dist'), { requireIndexNowKey });
   if (errors.length > 0) {
     console.error(`Discovery validation failed with ${errors.length} issue(s):`);
     for (const error of errors) {

src/src/layouts/SiteLayout.astro

@@ -19,6 +19,12 @@ interface Props {
    * hint for tooling; every page also exposes a normal crawlable `<a href>`.
    */
   alternateLinks?: { href: string; type: string; title?: string }[];
+  /**
+   * Keep this page out of search indexes: emits `<meta name="robots"
+   * content="noindex">` and omits the canonical link. Used by the 404 page,
+   * which has no canonical URL of its own (its route, `/404/`, does not exist).
+   */
+  noindex?: boolean;
 }
 
 const {
@@ -27,6 +33,7 @@ const {
   image,
   structuredData,
   alternateLinks = [],
+  noindex = false,
 } = Astro.props;
 const canonical = absoluteUrl(Astro.url.pathname === '' ? '/' : Astro.url.pathname);
 const ogImage = absoluteUrl(image ?? '/og/default.png');
@@ -39,7 +46,13 @@ const ogImage = absoluteUrl(image ?? '/og/default.png');
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>{title}</title>
     <meta name="description" content={description} />
-    <link rel="canonical" href={canonical} />
+    {
+      noindex ? (
+        <meta name="robots" content="noindex" />
+      ) : (
+        <link rel="canonical" href={canonical} />
+      )
+    }
     <meta property="og:type" content="website" />
     <meta property="og:site_name" content={SITE.fullName} />
     <meta property="og:title" content={title} />

src/src/pages/404.astro

@@ -7,6 +7,7 @@ import { withBase } from '~/lib/urls';
 <SiteLayout
   title={`Page not found — ${SITE.fullName}`}
   description="The page you were looking for could not be found."
+  noindex
 >
   <div class="pv-container flex flex-col items-center py-24 text-center md:py-32">
     <p class="pv-chip mb-6 border-brand/30 bg-brand/5 text-brand">404</p>

src/tests/unit/discovery/discovery-validate.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, test } from 'bun:test';
+
+import { validateIndexNowKeyFile } from '../../../scripts/discovery/validate';
+
+/** A reader over an in-memory file map, matching the `readDist` shape. */
+function fileAt(files: Record<string, string>): (path: string) => string | null {
+  return (path) => files[path] ?? null;
+}
+
+/**
+ * The IndexNow key verification file is written by the deploy workflow *after*
+ * the build (the build never sees the key), so the build pipeline must not
+ * require it. Only the deploy pipeline opts in with `require: true`.
+ */
+describe('IndexNow key file validation', () => {
+  const KEY = 'abc12345key';
+
+  test('does nothing when no key is configured', () => {
+    expect(validateIndexNowKeyFile(fileAt({}), undefined)).toEqual([]);
+    expect(validateIndexNowKeyFile(fileAt({}), '')).toEqual([]);
+    expect(validateIndexNowKeyFile(fileAt({}), '   ')).toEqual([]);
+  });
+
+  test('does nothing when the key is not a valid IndexNow key', () => {
+    expect(validateIndexNowKeyFile(fileAt({}), 'short')).toEqual([]);
+    expect(validateIndexNowKeyFile(fileAt({}), 'bad.key')).toEqual([]);
+  });
+
+  test('accepts a present file that contains the key', () => {
+    expect(validateIndexNowKeyFile(fileAt({ [`/${KEY}.txt`]: `${KEY}\n` }), KEY)).toEqual([]);
+  });
+
+  test('rejects a present file that does not contain the key', () => {
+    const errors = validateIndexNowKeyFile(fileAt({ [`/${KEY}.txt`]: 'something-else' }), KEY);
+    expect(errors).toHaveLength(1);
+    expect(errors[0]).toContain('does not contain the configured key');
+  });
+
+  test('ignores a missing file unless the key file is required', () => {
+    // The build pipeline: the build never writes the file, so a configured key
+    // must not fail validation.
+    expect(validateIndexNowKeyFile(fileAt({}), KEY)).toEqual([]);
+    expect(validateIndexNowKeyFile(fileAt({}), KEY, { require: false })).toEqual([]);
+  });
+
+  test('requires the file when the deploy pipeline opts in', () => {
+    const errors = validateIndexNowKeyFile(fileAt({}), KEY, { require: true });
+    expect(errors).toHaveLength(1);
+    expect(errors[0]).toContain('key verification file is missing');
+  });
+});