Add Astro website with searchable catalog and compatible data exports

.github/workflows/gh-pages.yml

@@ -6,23 +6,42 @@ on:
     types: [completed]
     branches: [master]
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest
-    if: ${{ github.event.repository.fork == false && github.event.workflow_run.conclusion == 'success' }}
+    timeout-minutes: 10
+    if: >-
+      vars.ASTRO_PUBLISH_ENABLED != 'true' &&
+      github.repository == 'LOLBAS-Project/LOLBAS' &&
+      github.event.repository.fork == false &&
+      github.event.workflow_run.conclusion == 'success' &&
+      github.event.workflow_run.head_branch == 'master' &&
+      github.event.workflow_run.event == 'push'
+    concurrency:
+      group: lolbas-external-site-publish
+      cancel-in-progress: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.workflow_run.head_sha }}
+          persist-credentials: false
 
       - name: Change .yml to .md
         run: |
-          for x in $(find yml/ -name '*.yml'); do echo "---" >> "$x"; mv "$x" "${x/%\.yml/.md}"; done
+          while IFS= read -r -d '' x; do
+            echo "---" >> "$x"
+            mv "$x" "${x/%.yml/.md}"
+          done < <(find yml/ -name '*.yml' -print0)
           mv yml/OSBinaries yml/Binaries
           mv yml/OSLibraries yml/Libraries
           mv yml/OSScripts yml/Scripts
           rm -r yml/HonorableMentions
 
       - name: Deploy to LOLBAS-Project.github.io repo
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
         with:
           deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
           external_repository: LOLBAS-Project/LOLBAS-Project.github.io

.github/workflows/site.yml

@@ -0,0 +1,129 @@
+name: Validate and publish Astro site
+
+on:
+  push:
+    branches: ['**']
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+# PR validation cannot cancel a production build or access the deploy key.
+concurrency:
+  group: astro-site-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build:
+    # Fork branches can produce review artifacts; upstream validates master/PRs.
+    if: >-
+      github.event_name != 'push' ||
+      github.repository != 'LOLBAS-Project/LOLBAS' ||
+      github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    defaults:
+      run:
+        working-directory: website
+    env:
+      ASTRO_TELEMETRY_DISABLED: '1'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install canonical catalog validators
+        working-directory: .
+        run: python -m pip install yamllint==1.37.1 pydantic==2.11.9 PyYAML==6.0.3
+      - name: Lint catalog YAML
+        working-directory: .
+        run: yamllint -c .github/.yamllint yml/**/
+      - name: Validate catalog schemas
+        working-directory: .
+        run: python .github/workflows/validation.py
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22.22.0'
+          cache: npm
+          cache-dependency-path: website/package-lock.json
+      - run: npm ci --no-audit --no-fund
+      - name: Check source formatting
+        run: npm run format:check
+      - name: Check source and generated data
+        run: npm run check
+      - name: Test data compatibility
+        run: npm run test:data
+      - name: Build and verify static output
+        run: npm run build
+      - name: Record artifact source
+        run: printf '%s\n' "$GITHUB_SHA" > dist/source-revision.txt
+      - run: npx playwright install --with-deps chromium
+      - name: Test the production artifact
+        run: npm test
+      - name: Upload browser evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: browser-evidence-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            website/playwright-report/
+            website/test-results/
+          if-no-files-found: ignore
+          retention-days: 14
+      - name: Upload verified site
+        uses: actions/upload-artifact@v4
+        with:
+          name: verified-site-${{ github.sha }}-${{ github.run_attempt }}
+          path: website/dist/
+          if-no-files-found: error
+          retention-days: 14
+      - name: Summarize review artifacts
+        run: |
+          {
+            printf '### Verified LOLBAS site\n\n'
+            printf 'Source: **%s**\n\n' "$GITHUB_SHA"
+            printf 'Download **verified-site-%s-%s** from this run to preview the tested site locally.\n\n' "$GITHUB_SHA" "$GITHUB_RUN_ATTEMPT"
+            printf 'Inside the extracted artifact, run:\n\n    python3 -m http.server 8000\n\nThen open http://localhost:8000/.\n\n'
+            printf 'Browser screenshots and test results are in **browser-evidence-%s-%s**.\n\n' "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT"
+            printf 'Fork and pull-request runs build review artifacts only. Production publication requires a separate upstream cutover.\n'
+          } >> "$GITHUB_STEP_SUMMARY"
+
+  deploy:
+    needs: build
+    if: >-
+      vars.ASTRO_PUBLISH_ENABLED == 'true' &&
+      github.repository == 'LOLBAS-Project/LOLBAS' &&
+      github.ref == 'refs/heads/master' &&
+      (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    environment:
+      name: github-pages
+      url: https://lolbas-project.github.io
+    concurrency:
+      group: lolbas-external-site-publish
+      cancel-in-progress: false
+    steps:
+      # No checkout or rebuild: publish only this run's tested artifact.
+      - uses: actions/download-artifact@v4
+        with:
+          name: verified-site-${{ github.sha }}-${{ github.run_attempt }}
+          path: verified-site
+      - name: Verify artifact source
+        run: test "$(cat verified-site/source-revision.txt)" = "$GITHUB_SHA"
+      - name: Publish verified Astro site to external repository
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
+          external_repository: LOLBAS-Project/LOLBAS-Project.github.io
+          publish_branch: master
+          publish_dir: verified-site
+          enable_jekyll: false
+          keep_files: false
+          commit_message: 'Publish verified Astro site from ${{ github.sha }}'
+          user_name: 'github-actions[bot]'
+          user_email: 'github-actions[bot]@users.noreply.github.com'

.gitignore

@@ -0,0 +1,10 @@
+.context/
+website/node_modules/
+website/dist/
+website/.astro/
+website/src/data/
+website/public/api/
+website/public/assets/lolbas-count.svg
+website/public/mitre_attack_navigator_layer.json
+website/test-results/
+website/playwright-report/

README.md

@@ -55,6 +55,23 @@ If you have found a new LOLBin or LOLScript that you would like to contribute, p
 
 A template for the required format has been provided here: https://github.com/LOLBAS-Project/LOLBAS/blob/master/YML-Template.yml
 
+## Build the website locally
+
+The Astro website builds from this repository's YAML catalog. With Node.js
+22.22.0 installed, run:
+
+```sh
+cd website
+npm ci
+npm run check
+npm run test:data
+npm run build
+npm run preview -- --port 4387
+```
+
+See [the website guide](website/README.md) for browser tests, development commands,
+and the maintainer-controlled deployment cutover and rollback procedure.
+
 ## The History of the LOLBin
 
 The phrase "Living off the land" was coined by Christopher Campbell (@obscuresec) & Matt Graeber (@mattifestation) at [DerbyCon 3](https://www.youtube.com/watch?v=j-r6UonEkUw).

docs/ASTRO_CHANGE_CONTRACT.md

@@ -0,0 +1,27 @@
+# Astro site migration contract
+
+Implementation, fork push, fork CI build, isolated fork-hosted preview and upstream
+pull request are authorized. Fork preview deployment belongs to a separate
+fork-owned workflow outside the upstream PR.
+Production publication and merging remain separate maintainer actions.
+Canonical content remains `yml/`; source records must not change. Branding is
+LOLBAS only. No analytics, external runtime data dependencies, or backend required.
+
+| Behavior | Acceptance evidence |
+| --- | --- |
+| LOL-SITE-001 | Static catalog includes every legacy-published record; links retain `/lolbas/{type}/{filename}/` and category anchors. |
+| LOL-SITE-002 | Search matches name, descriptions, ATT&CK IDs, command metadata and legacy `/function` / `#type` syntax. Combined category, type, technique, detection filters, column sorting, clear, pagination and URL reload work. |
+| LOL-SITE-003 | Details expose all commands, paths, detections/IOCs, aliases, tags, resources, code samples, author and acknowledgements; copy controls provide honest feedback. |
+| LOL-SITE-004 | Timeline aggregates YAML `Created` dates by year, explains methodology and has accessible chart data and catalog filter links. |
+| LOL-API-001 | JSON retains legacy fields and URLs; CSV retains exact 17-column header and one row per command; Navigator and count badge remain available. Source-derived tests and escaping fixtures. |
+| LOL-CI-001 | Fork branch and PR builds are read-only, cached and tested against production output, producing review artifacts. External publication is opt-in, restricted to upstream master, uses the same tested artifact, and cannot be triggered by forks or PRs. Legacy publication remains until deliberate cutover. |
+
+Assurance: A2 site/API with A3 publication boundary review. Independent review
+required. No authentication boundary exists. Evidence: data tests, Astro check,
+production build verification, desktop/mobile/no-JS/URL/API/console browser checks,
+visual screenshots and independent review. Remote CI/publication not run locally;
+repository settings and external branch backup precede future cutover.
+
+Stop at a fork-hosted branch, verified fork CI build, clickable fork preview and
+upstream PR with screenshots and recorded evidence. Do not merge or publish the
+official LOLBAS site.

website/.prettierignore

@@ -0,0 +1,2 @@
+src/data/
+package-lock.json

website/.prettierrc.json

@@ -0,0 +1 @@
+{ "plugins": ["prettier-plugin-astro"], "singleQuote": true }

website/README.md

@@ -0,0 +1,121 @@
+# LOLBAS website
+
+The Astro site builds directly from the repository's `yml/` catalog. Edit those
+YAML records to update the catalog; generated data and static output are build
+products. The existing YAML lint and schema workflow remains a separate gate.
+
+## Local development and verification
+
+Use Node.js **22.22.0** and the committed npm lockfile. From the repository root:
+
+```sh
+cd website
+npm ci
+npm run check
+npm run test:data
+npm run build
+npx playwright install chromium
+npm test
+npm run preview -- --port 4387
+```
+
+`check` regenerates catalog data and runs Astro's checks. `test:data` runs the
+data and publication regression tests. `build` regenerates data, produces `dist/`, and verifies
+the generated output. Playwright tests the production build through the preview
+server on port 4388; run `build` before `test`. The live preview uses port 4387
+so other workspaces can keep using the default Astro port. On Linux CI, install browser system dependencies
+with `npx playwright install --with-deps chromium`.
+
+For interactive development, run `npm run dev`. Run `npm run prepare:data` after
+changing catalog YAML during an existing development session. Do not execute the
+catalog's example commands: they are documentation rendered by the site.
+
+## Charts and compatibility
+
+The growth chart groups YAML `Created` dates by year and shows cumulative entries;
+it is catalog growth, not observed threat activity. Function counts count entries
+with a matching command; an entry can have multiple functions. Detection counts
+include published links and exclude IOC-only records.
+
+Legacy JSON fields, CSV columns, detail URLs/category anchors, the Navigator layer
+and count badge remain available. Generated API files do not include UI metadata.
+Search also accepts the legacy `/function` and `#type` syntax. Recently added
+sorting uses the recorded `Created` date; it does not imply last-modified dates.
+
+Every catalog column header toggles ascending/descending order. Function lists
+and ATT&CK lists sort alphabetically and numerically respectively; detections
+sort by published resource count. The dropdown, header indicators and URL share
+one sort state. Older browsers retain native filter menus through progressive CSS.
+
+## CI and publication boundary
+
+`.github/workflows/site.yml` validates all fork branch pushes, upstream `master`
+pushes, pull requests targeting `master`, and manual runs. Fork builds require
+Actions to be enabled in the fork. They upload review artifacts and browser
+evidence without publishing a site or needing deployment secrets. Download the
+verified site from the run and serve the extracted directory with
+`python3 -m http.server 8000` to preview it. It caches npm downloads, installs with `npm ci`, runs
+checks and data tests, builds and verifies the site, and runs Chromium against
+the production output. Only a successful run uploads the `verified-site` artifact
+for publication. `source-revision.txt` records its source SHA. Browser evidence is
+uploaded separately, including on test failure when evidence exists.
+
+The UI also supports an optional Astro `SITE_BASE` (for example `/LOLBAS`) for
+project-site previews. A separate fork-owned workflow can build with that value,
+run the preview smoke test via `SITE_BASE=/LOLBAS npm test`, and publish to the
+fork's Pages site. That preview publisher is intentionally outside this upstream
+change. Canonical API export URLs remain unchanged in prefixed previews.
+
+Publication defaults to **disabled**. The repository Actions variable
+`ASTRO_PUBLISH_ENABLED` must equal the literal string `true`. The publisher also
+requires the canonical `LOLBAS-Project/LOLBAS` repository, the `master` branch,
+and either a push or manual dispatch. Pull requests and `workflow_run` events
+cannot publish Astro output. The `github-pages` environment can require human
+approval. Use a **repository variable**, not an environment variable, because
+both workflow job conditions must see the switch before entering an environment.
+
+The publisher downloads the tested artifact from the same workflow run and
+verifies its recorded SHA. It does not rebuild. Using the existing
+`ACTIONS_DEPLOY_KEY`, it replaces the root of the external repository
+`LOLBAS-Project/LOLBAS-Project.github.io` on branch `master`, with Jekyll disabled.
+This is an external-repository publication, not an `actions/deploy-pages` upload.
+
+While the switch is unset or differs from `true`, the legacy Jekyll catalog
+publisher remains enabled. It accepts only successful YAML-validation runs
+originating from a push, checks out that validated run's exact SHA, and updates
+the external repository's `_lolbas` directory. Both publishers share a deployment
+concurrency group.
+
+## Maintainer cutover and rollback
+
+The following is a human-operated release procedure; local implementation and
+review do not authorize a live publication.
+
+1. Review the migration diff and independent release review, including catalog
+   compatibility, URLs, production browser results, and the generated artifact.
+   Confirm the master revision has passed both site and YAML/schema checks.
+2. Back up the external repository's current `master` Jekyll source to a retained
+   branch or tag and record its commit SHA. Preserve any custom domain, redirects,
+   and other site assets needed after replacing the root. Confirm the external
+   repository's Pages settings serve `master` from the root.
+3. Configure the source repository's `github-pages` environment and desired
+   required reviewers. Confirm the existing deploy key can write only to the
+   intended external repository. Wait for pending legacy publications to finish.
+4. Set the **repository Actions variable** `ASTRO_PUBLISH_ENABLED=true`, then
+   manually run the site workflow on `master`. Review and approve the environment
+   deployment if protection requires it. This publishes that run's tested artifact.
+5. Check the live homepage, a legacy detail URL, search/filter behavior, API
+   exports, and `source-revision.txt`. Retain the tested artifact and deployment
+   evidence with the release record.
+
+To roll back, unset `ASTRO_PUBLISH_ENABLED` (or set it to `false`) and cancel any
+queued or running Astro deployment before restoring the external `master` branch
+to the backed-up Jekyll source. Restore Pages/custom-domain settings if changed.
+Verify the live legacy site, then allow future successful push-triggered YAML
+validation runs to update `_lolbas` again. Merely unsetting the variable does not
+restore the Jekyll files removed by the Astro root publication.
+
+Required release review can be performed locally against workflows and build
+evidence. Live deployment, repository-variable changes, environment changes, and
+rollback operations are skipped during local validation and require a maintainer
+to execute this release procedure.

website/astro.config.mjs

@@ -0,0 +1,8 @@
+import { defineConfig } from 'astro/config';
+export default defineConfig({
+  site: 'https://lolbas-project.github.io',
+  base: process.env.SITE_BASE || '/',
+  output: 'static',
+  trailingSlash: 'always',
+  devToolbar: { enabled: false },
+});