diff --git a/purview-build.json b/.config/purview-build.json similarity index 100% rename from purview-build.json rename to .config/purview-build.json diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 96e8ba0..5c7429e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -132,6 +132,14 @@ jobs: fi bun run discovery:key + - name: Validate the discovery artifacts, including the key file + # The build pipeline cannot require the key verification file (the build + # never sees the key); this is the one place it is guaranteed to exist. + # When INDEXNOW_KEY is unset the requirement is a no-op. + env: + INDEXNOW_KEY: ${{ secrets.INDEXNOW_KEY }} + run: bun run discovery:validate --require-indexnow-key + - name: Publish the previous discovery manifest for change detection run: | mkdir -p src/dist/_discovery diff --git a/README.md b/README.md index 539448a..171b637 100644 --- a/README.md +++ b/README.md @@ -388,18 +388,39 @@ site URL automatically: `https://.github.io`. - **Project Pages** (`/`): base `/`, site `https://.github.io`. -- **Custom domain** (recommended for `purview.dev`): once DNS is configured at - your registrar (CNAME/ALIAS to `purview-dev.github.io`, or A/AAAA per GitHub's - guidance), set repository **variables** `SITE_URL=https://purview.dev` and - `PAGES_BASE=/`, and restore a `public/CNAME` file containing `purview.dev`. +- **Custom domain** (`purview.dev`): once DNS is configured at your registrar + (CNAME/ALIAS to `purview-dev.github.io`, or A/AAAA per GitHub's guidance), set + the repository **variables** `SITE_URL=https://purview.dev` and `PAGES_BASE=/`, + and configure the custom domain in **Settings → Pages → Custom domain**. The + domain is managed through the Pages settings for the GitHub Actions + deployment; no `public/CNAME` file is required. Configuration notes: - Enable Pages with **Source: GitHub Actions** in the repository settings and create the `github-pages` environment. -- The `public/CNAME` file is intentionally absent until the `purview.dev` DNS - records are configured, so the site stays reachable at the root of - `purview-dev.github.io`. +- The custom domain is configured in the repository's Pages settings + (**Settings → Pages → Custom domain**), not by a committed `public/CNAME` + file, so no `CNAME` needs to be maintained in the source tree. + +### HTTPS for the custom domain + +HTTPS is enforced at the **Cloudflare** edge, not by GitHub Pages. Because +`purview.dev` is proxied through Cloudflare (orange cloud), GitHub cannot +complete the Let's Encrypt HTTP-01 challenge, so the repository's **Enforce +HTTPS** toggle stays unavailable with the message *"your domain is not properly +configured to support HTTPS"*. This is expected, not a misconfiguration: + +- Cloudflare's **Always Use HTTPS** setting redirects `http://purview.dev/…` to + `https://purview.dev/…`, so the canonical origin is always HTTPS. +- The only side effect is that GitHub's default-domain redirect + (`purview-dev.github.io` → `purview.dev`) targets `http://`, adding one extra + hop before the HTTPS redirect. Search engines follow the chain to the + canonical URL and treat it as a redirect, not a duplicate. + +Do not try to force the GitHub toggle on: it would require temporarily setting +the Cloudflare DNS record to **DNS only** (grey cloud) so GitHub can issue the +certificate, and renewals can fail again once the record is proxied. The workflow validates (`just validate`), refreshes live release/documentation data, builds, and uploads `./src/dist`. diff --git a/docs/discovery.md b/docs/discovery.md index bebe611..cdf3757 100644 --- a/docs/discovery.md +++ b/docs/discovery.md @@ -62,6 +62,7 @@ accurate in CI. ```shell bun run discovery:validate # validate the built artifacts in dist/ +bun run discovery:validate --require-indexnow-key # also require /.txt bun run discovery:manifest # regenerate artifacts in dist/ from current caches bun run discovery:key # write /.txt from the secret bun run discovery:indexnow --dry-run # show what would be submitted (no network, no key) @@ -75,8 +76,15 @@ bun run discovery:indexnow --dry-run --verbose # include the URL lists fails the build on: missing/malformed sitemaps, non-absolute or off-origin URLs, duplicate canonicals, a `robots.txt` that does not advertise the sitemap, a `discover.json` that references unknown projects, missing per-project LLM -resources, development URLs in production output, and a missing IndexNow key file -when the key is configured. +resources, and development URLs in production output. + +The IndexNow key file is a **deploy-time** artifact: the build never sees the +key, so the build pipeline does not require it. The deploy workflow runs +`bun run discovery:key` and then +`bun run discovery:validate --require-indexnow-key`, which additionally fails +when `INDEXNOW_KEY` is configured but the key file is missing, or when the file +does not contain the configured key. Run `bun run discovery:validate` locally +without the flag and a configured key is simply ignored. ## IndexNow diff --git a/package.json b/package.json index 145200f..f38595f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "purview-dev", - "version": "0.4.0", + "version": "0.5.0", "private": true, "description": "Purview-Dev public website and unified documentation portal (workspace root).", "license": "MIT", diff --git a/src/scripts/discovery/validate.ts b/src/scripts/discovery/validate.ts index bd5ff52..e8180c1 100644 --- a/src/scripts/discovery/validate.ts +++ b/src/scripts/discovery/validate.ts @@ -40,7 +40,53 @@ export interface DiscoveryValidationResult { errors: string[]; } -export function validateDiscovery(dist = resolve('dist')): DiscoveryValidationResult { +export interface DiscoveryValidationOptions { + /** + * Require the IndexNow key verification file (`/.txt`) when + * `INDEXNOW_KEY` is configured. + * + * The build never writes this file — it is written from the secret by the + * deploy workflow via `bun run discovery:key`, *after* the build — so the + * build pipeline (`just validate` / `ci:build`) cannot require it. Only the + * deploy pipeline, which has just written the file, opts in with + * `--require-indexnow-key`. + */ + requireIndexNowKey?: boolean; +} + +/** + * Validate the IndexNow key verification file (`/.txt`). + * + * Returns the error messages to report: empty when the key is unset/invalid + * (nothing to check), when the file is present and correct, or when it is + * absent and not required. `require` is only set by the deploy pipeline, which + * writes the file after the build. + */ +export function validateIndexNowKeyFile( + readFile: (path: string) => string | null, + rawKey: string | undefined, + options: { require?: boolean } = {}, +): string[] { + const key = rawKey?.trim(); + if (!key || !/^[A-Za-z0-9-]{8,128}$/.test(key)) { + return []; + } + const content = readFile(`/${key}.txt`); + if (content === null) { + return options.require + ? ['INDEXNOW_KEY is configured but the key verification file is missing from the build.'] + : []; + } + if (content.trim() !== key) { + return ['The IndexNow key verification file does not contain the configured key.']; + } + return []; +} + +export function validateDiscovery( + dist = resolve('dist'), + options: DiscoveryValidationOptions = {}, +): DiscoveryValidationResult { const errors: string[] = []; const fail = (message: string): void => { errors.push(message); @@ -201,15 +247,15 @@ export function validateDiscovery(dist = resolve('dist')): DiscoveryValidationRe } } - // --- IndexNow key file (only when configured) --------------------------- - const indexNowKey = process.env.INDEXNOW_KEY?.trim(); - if (indexNowKey && /^[A-Za-z0-9-]{8,128}$/.test(indexNowKey)) { - const content = readDist(`/${indexNowKey}.txt`); - if (content === null) { - fail('INDEXNOW_KEY is configured but the key verification file is missing from the build.'); - } else if (content.trim() !== indexNowKey) { - fail('The IndexNow key verification file does not contain the configured key.'); - } + // --- IndexNow key file -------------------------------------------------- + // The build never writes this file (the key is a deploy-only secret), so its + // presence is only required when the caller opts in — the deploy workflow + // runs this after `discovery:key`. When the file is present it must always + // contain the configured key. + for (const error of validateIndexNowKeyFile(readDist, process.env.INDEXNOW_KEY, { + require: options.requireIndexNowKey, + })) { + fail(error); } return { errors }; @@ -217,7 +263,8 @@ export function validateDiscovery(dist = resolve('dist')): DiscoveryValidationRe if (import.meta.main) { console.log('Validating discovery artifacts...'); - const { errors } = validateDiscovery(); + const requireIndexNowKey = process.argv.includes('--require-indexnow-key'); + const { errors } = validateDiscovery(resolve('dist'), { requireIndexNowKey }); if (errors.length > 0) { console.error(`Discovery validation failed with ${errors.length} issue(s):`); for (const error of errors) { diff --git a/src/src/layouts/SiteLayout.astro b/src/src/layouts/SiteLayout.astro index a23b40b..41bdf53 100644 --- a/src/src/layouts/SiteLayout.astro +++ b/src/src/layouts/SiteLayout.astro @@ -19,6 +19,12 @@ interface Props { * hint for tooling; every page also exposes a normal crawlable ``. */ alternateLinks?: { href: string; type: string; title?: string }[]; + /** + * Keep this page out of search indexes: emits `` and omits the canonical link. Used by the 404 page, + * which has no canonical URL of its own (its route, `/404/`, does not exist). + */ + noindex?: boolean; } const { @@ -27,6 +33,7 @@ const { image, structuredData, alternateLinks = [], + noindex = false, } = Astro.props; const canonical = absoluteUrl(Astro.url.pathname === '' ? '/' : Astro.url.pathname); const ogImage = absoluteUrl(image ?? '/og/default.png'); @@ -39,7 +46,13 @@ const ogImage = absoluteUrl(image ?? '/og/default.png'); {title} - + { + noindex ? ( + + ) : ( + + ) + } diff --git a/src/src/pages/404.astro b/src/src/pages/404.astro index 9a9fe29..185feab 100644 --- a/src/src/pages/404.astro +++ b/src/src/pages/404.astro @@ -7,6 +7,7 @@ import { withBase } from '~/lib/urls';

404

diff --git a/src/tests/unit/discovery/discovery-validate.test.ts b/src/tests/unit/discovery/discovery-validate.test.ts new file mode 100644 index 0000000..2e1d01e --- /dev/null +++ b/src/tests/unit/discovery/discovery-validate.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from 'bun:test'; + +import { validateIndexNowKeyFile } from '../../../scripts/discovery/validate'; + +/** A reader over an in-memory file map, matching the `readDist` shape. */ +function fileAt(files: Record): (path: string) => string | null { + return (path) => files[path] ?? null; +} + +/** + * The IndexNow key verification file is written by the deploy workflow *after* + * the build (the build never sees the key), so the build pipeline must not + * require it. Only the deploy pipeline opts in with `require: true`. + */ +describe('IndexNow key file validation', () => { + const KEY = 'abc12345key'; + + test('does nothing when no key is configured', () => { + expect(validateIndexNowKeyFile(fileAt({}), undefined)).toEqual([]); + expect(validateIndexNowKeyFile(fileAt({}), '')).toEqual([]); + expect(validateIndexNowKeyFile(fileAt({}), ' ')).toEqual([]); + }); + + test('does nothing when the key is not a valid IndexNow key', () => { + expect(validateIndexNowKeyFile(fileAt({}), 'short')).toEqual([]); + expect(validateIndexNowKeyFile(fileAt({}), 'bad.key')).toEqual([]); + }); + + test('accepts a present file that contains the key', () => { + expect(validateIndexNowKeyFile(fileAt({ [`/${KEY}.txt`]: `${KEY}\n` }), KEY)).toEqual([]); + }); + + test('rejects a present file that does not contain the key', () => { + const errors = validateIndexNowKeyFile(fileAt({ [`/${KEY}.txt`]: 'something-else' }), KEY); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('does not contain the configured key'); + }); + + test('ignores a missing file unless the key file is required', () => { + // The build pipeline: the build never writes the file, so a configured key + // must not fail validation. + expect(validateIndexNowKeyFile(fileAt({}), KEY)).toEqual([]); + expect(validateIndexNowKeyFile(fileAt({}), KEY, { require: false })).toEqual([]); + }); + + test('requires the file when the deploy pipeline opts in', () => { + const errors = validateIndexNowKeyFile(fileAt({}), KEY, { require: true }); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain('key verification file is missing'); + }); +});