diff --git a/.github/workflows/kit-ci.yml b/.github/workflows/kit-ci.yml index f5c461a..1c77a13 100644 --- a/.github/workflows/kit-ci.yml +++ b/.github/workflows/kit-ci.yml @@ -56,6 +56,9 @@ jobs: for f in bin/workshop-kit.mjs src/index.mjs src/config/*.mjs src/commands/*.mjs tools/tenant-pulse-dashboard/server.mjs; do node --check "$f" done + # Shared Python that no build step exercises, so nothing else would + # catch a syntax error in it before an author ran it against a deck. + python3 -c "import ast,sys; ast.parse(open(sys.argv[1]).read(), sys.argv[1])" tools/extract-pptx-assets.py - name: CLI responds run: | @@ -124,3 +127,105 @@ jobs: grep -q 'keep={{WORKSHOP_UNDECLARED}}' "$work/got.txt" || { echo 'FAIL: undeclared token should stay literal'; exit 1; } grep -q '{{WORKSHOP_UIPATH_ORG_NAME}}' "$work/downloads/project/AGENTS.md" || { echo 'FAIL: source file was mutated'; exit 1; } echo 'zip pipeline OK' + + # This prune DELETES files from static/downloads/, so it is covered the same + # way. The regression it guards: static/downloads/ is gitignored, so a stale + # ZIP is invisible to CI and code review, yet `codedapp pack` packs the local + # build/ dir that Docusaurus fills from static/ -- so it still reaches the + # deployed site. + - name: ZIP prune removes orphans, spares current zips and non-zips + run: | + set -e + work="$(mktemp -d)" + mkdir -p "$work/config" "$work/downloads/keep-me" "$work/static/downloads" + cat > "$work/config/workshop-targets.json" <<'JSON' + { + "title": "Fixture", "tagline": "t", "repo": "org/repo", + "defaultTarget": "local", + "targets": {"local": { + "siteUrl": "http://localhost:3000", "baseUrl": "/", + "workshop": {"uipathOrgName": "org", "uipathTenantName": "tenant", + "uipathTenantUrl": "https://example.invalid/org/tenant", + "orchestratorParentFolder": "Shared"}, + "codedApp": {"name": "fixture"} + }} + } + JSON + printf 'content\n' > "$work/downloads/keep-me/file.txt" + : > "$work/static/downloads/orphan.zip" + : > "$work/static/downloads/notes.txt" + # Seed the CURRENT zip too. This models a rebuild, which is the only + # state in which a prune that ignores its keep-set is observable: on a + # fresh tree there is no current zip present for it to wrongly delete. + : > "$work/static/downloads/keep-me.zip" + + out="$( cd "$work" && node "$GITHUB_WORKSPACE/bin/workshop-kit.mjs" zip-downloads 2>&1 )" + echo "$out" + echo "$out" | grep -q 'Removed orphaned static/downloads/orphan.zip' || { echo 'FAIL: orphan not reported'; exit 1; } + [ -e "$work/static/downloads/orphan.zip" ] && { echo 'FAIL: orphan zip survived'; exit 1; } + [ -e "$work/static/downloads/keep-me.zip" ] || { echo 'FAIL: current zip missing'; exit 1; } + [ -e "$work/static/downloads/notes.txt" ] || { echo 'FAIL: non-zip file was deleted'; exit 1; } + # The prune runs BEFORE generation, so deleting a current zip is invisible + # by file existence alone -- it is immediately recreated. Assert on the log + # instead, or a prune that ignores its keep-set passes unnoticed. + echo "$out" | grep -q 'Removed orphaned static/downloads/keep-me.zip' && { echo 'FAIL: current zip reported as orphaned'; exit 1; } + + # Emptying downloads/ makes every remaining zip an orphan. This is the + # exact shape of the CA bug: the folder was deleted, the zip was not. + rm -rf "$work/downloads/keep-me" + out2="$( cd "$work" && node "$GITHUB_WORKSPACE/bin/workshop-kit.mjs" zip-downloads 2>&1 )" + echo "$out2" + [ -e "$work/static/downloads/keep-me.zip" ] && { echo 'FAIL: zip survived deletion of its source folder'; exit 1; } + [ -e "$work/static/downloads/notes.txt" ] || { echo 'FAIL: non-zip file was deleted'; exit 1; } + + # No downloads/ at all means the kit is not managing that directory, so + # it must not delete anything there. + rm -rf "$work/downloads" + : > "$work/static/downloads/untouched.zip" + ( cd "$work" && node "$GITHUB_WORKSPACE/bin/workshop-kit.mjs" zip-downloads >/dev/null 2>&1 ) + [ -e "$work/static/downloads/untouched.zip" ] || { echo 'FAIL: pruned with no downloads/ dir'; exit 1; } + echo 'zip prune OK' + + # The prune DELETES build artifacts, so it gets the same treatment as the + # ZIP pipeline. The case that matters is the second one: one repo packs a + # different app name per training target, and a global "keep the newest N" + # would destroy an older target's only package. + - name: Package prune keeps N per package name + run: | + set -e + work="$(mktemp -d)" + mkdir -p "$work/.uipath" + cd "$work/.uipath" + # older <- -> newer, within each name + for f in only-once.0.1.0-20260101000000 \ + many.0.1.0-20260101000000 many.0.1.0-20260102000000 \ + many.0.1.0-20260103000000 many.0.1.0-20260104000000; do + echo x > "$f.nupkg" + done + touch -t 202601010000 only-once.0.1.0-20260101000000.nupkg + touch -t 202601010000 many.0.1.0-20260101000000.nupkg + touch -t 202601020000 many.0.1.0-20260102000000.nupkg + touch -t 202601030000 many.0.1.0-20260103000000.nupkg + touch -t 202601040000 many.0.1.0-20260104000000.nupkg + echo '{}' > app.config.json + echo 'not-a-package' > loose.nupkg + + cd "$work" + node --input-type=module -e " + const {prunePackages} = await import('file://$GITHUB_WORKSPACE/src/commands/codedapp-deploy.mjs'); + prunePackages(); + " + + # The single old package of a name never packed again must survive, + # even though four newer files exist under a different name. + [ -f .uipath/only-once.0.1.0-20260101000000.nupkg ] \ + || { echo 'FAIL: pruned the only package of an untouched name'; exit 1; } + [ "$(ls .uipath/many.*.nupkg | wc -l)" -eq 3 ] \ + || { echo 'FAIL: expected 3 survivors for "many"'; exit 1; } + [ -f .uipath/many.0.1.0-20260101000000.nupkg ] \ + && { echo 'FAIL: oldest of the group survived'; exit 1; } + [ -f .uipath/many.0.1.0-20260104000000.nupkg ] \ + || { echo 'FAIL: newest of the group was deleted'; exit 1; } + [ -f .uipath/app.config.json ] || { echo 'FAIL: app.config.json removed'; exit 1; } + [ -f .uipath/loose.nupkg ] || { echo 'FAIL: unparseable filename removed'; exit 1; } + echo 'package prune OK' diff --git a/.gitignore b/.gitignore index d7daaee..f88f941 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ npm-debug.log* # Build output /dist + +# Byte cache from the Python extractor +__pycache__/ +*.pyc diff --git a/README.md b/README.md index 007f337..69ae014 100644 --- a/README.md +++ b/README.md @@ -22,8 +22,8 @@ src/commands/ one module per CLI subcommand src/components/ WorkshopEnv React components src/css/ shared stylesheet agents/base.md THE shared "General Workshop Instructions" -schemas/ JSON Schema for config/workshop-targets.json -tools/ tenant pulse dashboard +schemas/ JSON Schema for config/workshop-targets.json and the PPTX slide map +tools/ tenant pulse dashboard, PPTX asset extractor template/ scaffold for `workshop-kit init` (a complete, building site) ``` @@ -35,9 +35,10 @@ workshop-kit agents build regenerate AGENTS.md from base + product workshop-kit agents check fail if AGENTS.md is stale or hand-edited inside the fence workshop-kit check:tenant-access advisory tenant permission report (never fails a build) workshop-kit build --target -workshop-kit codedapp +workshop-kit codedapp pack prunes old nupkgs workshop-kit preview --target workshop-kit pulse +workshop-kit extract-pptx --pptx --output-root --manifest workshop-kit validate-config check every target in config/workshop-targets.json workshop-kit doctor [target] workshop-kit repin move a content repo onto a released kit tag @@ -119,6 +120,83 @@ Everything else is content. Replace the `REPLACE_ME` tenant values before delivering; `validate-config` deliberately passes with them in place, since it cannot know what is real. +## Packing and deploying the Coded App + +`workshop-kit codedapp ` builds the site for a +target and ships it as a UiPath Coded Web App. `check` prints the resolved +target, app name and whether the deploy env is present, and touches nothing. + +Every `pack` writes a fresh timestamped `.uipath/..nupkg`, and +`uip` never removes one. Left alone that grows without bound — one repo reached +147 MB across 21 files in two months. After a successful pack the kit keeps the +newest **3 packages per package name** and deletes the rest, reporting what it +removed. Override with `UIPATH_CODEDAPP_KEEP`; a value below 1 or non-numeric +falls back to 3 with a warning. + +Per *name*, not per directory, because a repo packs a different app name per +training target — `tc20260721-ca-workshop`, `ifca20260722-ca-workshop`, and so +on. A global "keep the newest 3" would delete an older target's only package the +moment you pack a different one, and `publish` is a separate command that runs +against the local file, so that would break pack-now-publish-later. Grouping by +name also guarantees the file the current run just produced survives. + +Local packages are build artifacts, not the deliverable: `publish` uploads to +Orchestrator. `.uipath/` and `uipath.json` are gitignored in every content repo +and must never be committed. Files in `.uipath/` that are not named +`..nupkg` — including `app.config.json` and `metadata.json` — are +left alone. + +## Extracting a workshop from a facilitator deck + +`workshop-kit extract-pptx` pulls screenshots, ordered text blocks and callout +geometry out of a `.pptx` and writes a manifest beside them. Two workshops were +built this way from a lab-guide deck. + +```bash +npx workshop-kit extract-pptx \ + --pptx ~/decks/5_\ CM\ Lab\ Guide.pptx \ + --output-root static/img \ + --manifest extraction/manifest.json \ + --slide-map extraction/slide-map.json +``` + +Run it once without `--slide-map` first. Everything lands in +`static/img/unassigned/`, which is what you want before you know where the +exercise boundaries are. Then write the map and run it again: + +```json +{ + "deck": "5_ CM Lab Guide.pptx", + "pages": [ + {"page": "pre-reqs", "slides": [2]}, + {"page": "exercise-1", "slides": ["3-18"]}, + {"page": "exercise-2", "slides": ["20-24"]} + ], + "roles": [ + {"role": "answer-reveal", "match": ["answers"]}, + {"role": "sign-in", "slides": [2]} + ] +} +``` + +`schemas/pptx-slide-map.schema.json` has the full shape. The map is the *only* +deck-specific input; it lives in the content repo, or nowhere at all once the +docs are the source of truth, and the deck itself is never committed. + +Two things this tool exists to get right, both learned the hard way: + +- **Placeholder pictures count.** An image dropped into a PowerPoint content + placeholder is a `PlaceholderPicture` with `shape_type == PLACEHOLDER`, not + `PICTURE`. An earlier per-repo copy filtered on `shape_type` alone and silently + dropped 18 of 33 screenshots from one deck, with no error and no non-zero exit. +- **Numbered callout ovals are separate shapes.** They vanish the moment a + screenshot is extracted on its own, so each is recorded as a percentage offset + inside the picture it overlaps, ready for `.workshop-click-marker` in MDX. + +`python-pptx` and `Pillow` are required, and are deliberately not project +dependencies — this is a one-off authoring step, not part of any build. The +command checks for them and prints the `pip install` line if they are absent. + ## Why the components are compiled `npm run build` emits `src/components/*.tsx` to `dist/` as plain JS plus @@ -134,9 +212,38 @@ against a `file:` link. ## Versioning -- `agents/base.md` wording, or a new optional config field → **minor** +- new CLI command, `agents/base.md` wording, or a new optional config field → **minor** - check or pipeline bug fix → **patch** - renamed component, removed config field, changed CLI flag → **major** -Content repos pin a caret range and rely on Dependabot to surface kit releases, -so a shared change is immediately CI-tested against every workshop. +**The version number is a signal, not a trigger. Nothing updates on its own.** + +A content repo pins one exact tag: + +```json +"@uipath-lab-tec/workshop-kit": "github:UiPath-LAB-TEC/workshop-kit#v1.3.0" +``` + +That is a tag, not a caret range, and it resolves through `package-lock.json` to +a specific commit. Three consequences follow, and all three are deliberate: + +1. **Dependabot cannot bump it.** There is no registry to poll and no semver + range to compare for a `github:` tag dependency. The `dependabot.yml` in each + content repo covers `@docusaurus/*` and GitHub Actions only — a kit group + used to sit there doing nothing, which read as if the bumps were automatic. +2. **`npm install` cannot bump it either.** The lockfile records the resolved + commit of the *previous* tag, so npm honours the lock, prints "changed 1 + package", and keeps the old kit even after you edit `package.json`. See + "Releasing, and repinning consumers" above. +3. **Only `workshop-kit repin ` moves a repo**, and someone has to run it, + per repo, on purpose. + +So a **major** does not break every workshop the day it is tagged. It breaks the +next repo that repins, whenever that is. A workshop already delivered can sit on +an old tag indefinitely and keep building — which is the point, because a repo +is often frozen mid-delivery while the kit moves on. + +The cost of that safety is that a shared fix is *not* live everywhere until you +repin each repo. Cross-repo CI proves a change is safe; it does not deliver it. +Treat "tag the kit" and "repin the consumers" as two separate jobs, and expect +repos to sit on different tags in between. diff --git a/agents/base.md b/agents/base.md index 1441ce1..7021ff1 100644 --- a/agents/base.md +++ b/agents/base.md @@ -14,6 +14,33 @@ This section is shared workshop guidance. Keep it exactly the same across worksh - Keep edits narrow: only add content that significantly improves the exercise or workshop flow. - When an exercise has multiple possible paths, make the target org, tenant, folder, bucket, app, or automation surface explicit. +### Shared Naming Conventions + +These names are identical in every workshop repo. A participant who has done one +workshop should recognise the structure of the next one. Do not diverge locally; +change the convention here and roll it out everywhere. + +- **The prerequisites page is titled `Prerequisites`.** Where a workshop splits its + prerequisites across several pages, title each one `Prerequisites - `, for + example `Prerequisites - Sign-in`. Never `Pre-req`, `Pre-reqs`, or a bare step + name. Body text that cross-references the page uses the same title verbatim. +- **Every exercise page closes with `## Success check`.** One sentence-case + heading holding one checklist of what must be true before moving on. Do not add + a second near-duplicate section such as `Verification Checks` — fold those items + into the same list. A `## Recap` after it is optional, for a short narrative + wrap-up where one genuinely adds something. An up-front `## Requirements` or + `## Objectives` section is a different thing and may stay. +- **Participant placeholders use angle brackets:** ``, and + `` where initials specifically are wanted. Never `[participant + name]`, `[your-name]`, or `{{your-name}}`. Angle brackets read as replace-me and + cannot be mistaken for a build-time token. +- **`{{ }}` is reserved for `{{WORKSHOP_*}}` substitution tokens.** Never use that + shape for anything a participant is supposed to replace by hand: it looks exactly + like a token that will be rendered at build time, and silently is not one. +- **`sidebar_position` runs contiguously from 1**, in the same order as + `sidebars.ts`. Gaps are drift, even though the explicit sidebar hides them. +- **Front matter `title` and the page `# H1` must match exactly.** + ### Screenshots And Visual Evidence - Blur, crop, or replace sensitive data in screenshots before adding them to workshop materials. Verify what is sensitive data. - Do not expose customer names, emails, tenant IDs, org IDs, folder IDs, tokens, URLs with secrets, queue item data, or internal-only environment details unless the user explicitly approves. @@ -70,11 +97,17 @@ an explicit CLI argument (`workshop-kit build `), then the Other rules for these values: - Use `WorkshopValue`, `WorkshopLink`, or `WorkshopCodeBlock` from `src/components/WorkshopEnv.tsx` when showing these values in MDX. - In copyable prompts, use `WorkshopCodeBlock` and tokens such as `{{WORKSHOP_UIPATH_ORG_NAME}}` so rendered prompts include the current session values. -- Keep participant-specific names such as participant folder names and exercise bucket names as explicit placeholders unless a dedicated workshop variable exists for them. +- Keep participant-specific names such as participant folder names and exercise bucket names as explicit `` placeholders unless a dedicated workshop variable exists for them. +- **No delivery-specific literal belongs in `docs/`.** A tenant URL, an org, a join + link, a Studio Web template name, a cohort, a city or a date that changes between + deliveries goes in `config/workshop-targets.json` and renders through + `WorkshopValue` / `WorkshopLink`. Declare anything beyond the four base fields + under `extraFields`. If you find yourself typing a city or a month into a docs + page, add the variable instead. - When adding, renaming, or removing workshop variables or `WorkshopEnv` imports, run `npm run check:workshop-vars`; stale tokens, unknown fields, unused imports, and defined-but-unused workshop variables should fail the check. ### Exercise Design -- Each exercise should have clear objectives, steps, success criteria, and verification checks. +- Each exercise should have clear objectives, steps, and one closing `## Success check` list. See Shared Naming Conventions above. - Follow the project-specific convention for objective sections and learning objectives. - Include realistic prompts or participant inputs when the exercise calls for them; do not turn exercise pages into theory-heavy lessons unless asked. - Prefer deterministic, lightweight assets that are easy to recreate and reset. @@ -84,11 +117,13 @@ Other rules for these values: ### Downloads And Assets - Participant source folders live under `downloads//`. - Each `downloads//` directory produces exactly one `static/downloads/.zip`; name the directory for the ZIP participants receive. +- Name a download for what is inside it, not just for the exercise number: `exercise-3-paystubs`, `exercise-2-taxonomy`, `workshop-project`. A participant with three ZIPs in their Downloads folder should be able to tell them apart. - Loose files placed directly at the `downloads/` root are not packaged into any ZIP and are skipped without failing the build. - Generated ZIP files are created under `static/downloads/` by `npm run zip:downloads`. - Downloaded project instructions such as `downloads/workshop-project/AGENTS.md` may use `{{WORKSHOP_*}}` tokens; the ZIP workflow must render them from the selected target before packaging. - Keep original filenames organized under `downloads/` and rely on the build-time ZIP workflow where available. - Exercise pages should link their own relevant ZIP downloads close to the step or section that uses them. +- Serve participant files from the site itself through `WorkshopDownloadLink`. Do not link an external file host such as Google Drive, Dropbox, or a SharePoint file: those links leak outside the repo, cannot be versioned with the content, and go stale between deliveries. - Track missing participant files in a project-specific checklist when downloads are still incomplete. ### Validation diff --git a/bin/workshop-kit.mjs b/bin/workshop-kit.mjs index 94a2238..cf4b6bd 100755 --- a/bin/workshop-kit.mjs +++ b/bin/workshop-kit.mjs @@ -24,6 +24,8 @@ const USAGE = `workshop-kit preview [] [] serve build/ at the target's baseUrl codedapp pulse tenant pulse dashboard + extract-pptx --pptx --output-root --manifest [--slide-map ] + pull screenshots, text and callout geometry from a deck validate-config check every target in config/workshop-targets.json doctor [target] report kit version, resolved target and drift repin [--no-verify] move this repo to a kit tag, regenerate AGENTS.md, verify @@ -162,6 +164,11 @@ async function main() { case 'pulse': return runScript('tools/tenant-pulse-dashboard/server.mjs', argv); + case 'extract-pptx': { + const {extractPptx} = await import(join(commandsDir, 'extract-pptx.mjs')); + return extractPptx({args: argv}); + } + case 'validate-config': { const {validateConfig} = await import(join(commandsDir, 'validate-config.mjs')); return validateConfig({}); diff --git a/package.json b/package.json index ca52f42..07be75d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@uipath-lab-tec/workshop-kit", - "version": "1.3.0", + "version": "1.4.0", "description": "Shared Docusaurus infrastructure for UiPath LAB TEC workshop repos.", "license": "UNLICENSED", "type": "module", @@ -23,6 +23,7 @@ }, "./css/workshop.css": "./src/css/workshop.css", "./schemas/workshop-targets.schema.json": "./schemas/workshop-targets.schema.json", + "./schemas/pptx-slide-map.schema.json": "./schemas/pptx-slide-map.schema.json", "./agents/base.md": "./agents/base.md", "./components/WorkshopHome": { "types": "./dist/components/WorkshopHome.d.ts", diff --git a/schemas/pptx-slide-map.schema.json b/schemas/pptx-slide-map.schema.json new file mode 100644 index 0000000..2b61ea5 --- /dev/null +++ b/schemas/pptx-slide-map.schema.json @@ -0,0 +1,94 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/UiPath-LAB-TEC/workshop-kit/schemas/pptx-slide-map.schema.json", + "title": "PPTX slide map", + "description": "Per-deck input to `workshop-kit extract-pptx`. Holds the only two things that were ever deck-specific in the extractor: which slides belong to which docs page, and how to name a slide's role. Optional -- without it every slide lands in one folder.", + "type": "object", + "additionalProperties": false, + "properties": { + "$schema": { + "type": "string" + }, + "deck": { + "type": "string", + "description": "Filename of the deck this map was written against, recorded in the manifest. Decks are not kept in the content repos, so this is the only trace of which revision the slide numbers came from." + }, + "pages": { + "type": "array", + "description": "Slide-to-page assignment. A slide no rule claims goes to --default-page.", + "items": { + "type": "object", + "required": [ + "page", + "slides" + ], + "additionalProperties": false, + "properties": { + "page": { + "type": "string", + "description": "Output folder name, normally the docs page slug (`exercise-1`, `pre-reqs`)." + }, + "slides": { + "$ref": "#/definitions/slideSelector" + } + } + } + }, + "roles": { + "type": "array", + "description": "Advisory `likely_exercise_role` per slide. First matching rule wins, so order is priority order. A slide matching nothing falls back to its page assignment.", + "items": { + "type": "object", + "required": [ + "role" + ], + "additionalProperties": false, + "properties": { + "role": { + "type": "string" + }, + "slides": { + "$ref": "#/definitions/slideSelector" + }, + "match": { + "type": "array", + "description": "Case-insensitive substrings tested against all text on the slide.", + "items": { + "type": "string" + } + } + }, + "anyOf": [ + { + "required": [ + "slides" + ] + }, + { + "required": [ + "match" + ] + } + ] + } + } + }, + "definitions": { + "slideSelector": { + "type": "array", + "description": "Slide numbers, 1-based. An integer is one slide; \"N-M\" is an inclusive range.", + "items": { + "oneOf": [ + { + "type": "integer", + "minimum": 1 + }, + { + "type": "string", + "pattern": "^\\s*\\d+\\s*(-\\s*\\d+\\s*)?$" + } + ] + } + } + } +} diff --git a/src/commands/codedapp-deploy.mjs b/src/commands/codedapp-deploy.mjs index 6d29937..4c12bf6 100644 --- a/src/commands/codedapp-deploy.mjs +++ b/src/commands/codedapp-deploy.mjs @@ -3,10 +3,14 @@ import fs from 'node:fs'; import path from 'node:path'; import process from 'node:process'; import {spawnSync} from 'node:child_process'; +import {pathToFileURL} from 'node:url'; import {getTarget} from '../config/workshop-target.mjs'; import {readTargetsConfig} from '../config/workshop-fields.mjs'; const root = process.cwd(); +// Enough to roll back to the previously deployed build without keeping +// every pack ever run. Override with UIPATH_CODEDAPP_KEEP. +const DEFAULT_KEEP = 3; function loadEnvFile(filePath) { if (!fs.existsSync(filePath)) { @@ -78,6 +82,83 @@ function generatedVersion() { return `0.1.0-${stamp}`; } +/** + * `..nupkg`. The name may contain dots, so the version is + * anchored as the trailing semver rather than assumed to start at the first + * dot. A file that does not match this shape is not ours to reason about, and + * is never deleted. + */ +function parsePackageFilename(filename) { + const match = filename.match(/^(.*)\.(\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)\.nupkg$/); + return match ? {name: match[1], version: match[2]} : undefined; +} + +/** + * `uip codedapp pack` writes a fresh timestamped nupkg on every run and never + * removes one. Two months of packing left 147 MB across 21 files in one repo. + * + * Pruning is per package NAME, not across the directory. One repo packs a + * different app name per training target, so a global "keep the newest 3" + * would delete an older target's only package the moment you pack a different + * one -- and `publish` is a separate command that runs against the local file, + * so that breaks pack-now-publish-later. Grouping by name also guarantees the + * file the current invocation just produced survives, since it is the newest + * in its own group. + */ +export function prunePackages() { + const dir = path.join(root, '.uipath'); + if (!fs.existsSync(dir)) return; + + const raw = process.env.UIPATH_CODEDAPP_KEEP?.trim(); + let keep = DEFAULT_KEEP; + if (raw) { + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < 1) { + console.warn(`UIPATH_CODEDAPP_KEEP="${raw}" is not a positive integer; keeping ${DEFAULT_KEEP}.`); + } else { + keep = parsed; + } + } + + const groups = new Map(); + for (const filename of fs.readdirSync(dir)) { + const parsed = parsePackageFilename(filename); + if (!parsed) continue; + const full = path.join(dir, filename); + let mtimeMs; + try { + mtimeMs = fs.statSync(full).mtimeMs; + } catch { + continue; + } + if (!groups.has(parsed.name)) groups.set(parsed.name, []); + groups.get(parsed.name).push({filename, full, mtimeMs}); + } + + let removed = 0; + let bytes = 0; + for (const [name, files] of groups) { + if (files.length <= keep) continue; + // Newest first. Versions are timestamps, so the filename is a sound + // tiebreak when two packs land in the same millisecond. + files.sort((a, b) => b.mtimeMs - a.mtimeMs || b.filename.localeCompare(a.filename)); + for (const file of files.slice(keep)) { + try { + bytes += fs.statSync(file.full).size; + fs.unlinkSync(file.full); + removed += 1; + } catch (error) { + console.warn(`Could not remove ${file.filename}: ${error.message}`); + } + } + console.log(`Pruned ${files.length - keep} old package(s) for "${name}", kept the newest ${keep}.`); + } + + if (removed > 0) { + console.log(`Freed ${(bytes / 1024 / 1024).toFixed(1)} MB from .uipath/.`); + } +} + function targetAppUrl(target) { return new URL(target.baseUrl.replace(/^\//, ''), `${target.siteUrl}/`).toString().replace(/\/$/, ''); } @@ -132,6 +213,7 @@ function main() { '--description', description, ]); + prunePackages(); return; } @@ -166,6 +248,7 @@ function main() { '--description', description, ]); + prunePackages(); run('uip', ['codedapp', 'publish', '-n', packageName, '--version', packageVersion]); const folderKey = requireEnv('UIPATH_FOLDER_KEY'); run('uip', ['codedapp', 'deploy', '-n', packageName, '--folder-key', folderKey]); @@ -180,4 +263,6 @@ function main() { throw new Error(`Unknown command "${command}". Use check, pack, publish, deploy, or all.`); } -main(); +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/src/commands/doctor.mjs b/src/commands/doctor.mjs index 24cdcc9..927a615 100644 --- a/src/commands/doctor.mjs +++ b/src/commands/doctor.mjs @@ -67,6 +67,7 @@ export function doctor({root = process.cwd(), target: explicitTarget} = {}) { 'scripts/docusaurus-target.mjs', 'scripts/uipath-codedapp-deploy.mjs', 'scripts/check-tenant-access.mjs', + 'scripts/extract-pptx-assets.py', 'src/css/custom.css', 'src/pages/index.module.css', 'AGENTS.product.md', diff --git a/src/commands/extract-pptx.mjs b/src/commands/extract-pptx.mjs new file mode 100644 index 0000000..1752821 --- /dev/null +++ b/src/commands/extract-pptx.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/** + * Wrapper around tools/extract-pptx-assets.py. + * + * The extractor is Python because python-pptx is the only sane way to read + * shape geometry out of a deck, and the kit is Node. This module exists to + * fail with a useful message instead of a Python traceback when the two + * dependencies are missing -- which they will be, on a machine that has only + * ever built the site. + */ +import {existsSync} from 'node:fs'; +import {join, dirname} from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {spawnSync} from 'node:child_process'; +import process from 'node:process'; + +const kitRoot = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const script = join(kitRoot, 'tools', 'extract-pptx-assets.py'); + +const REQUIRED = [ + ['pptx', 'python-pptx'], + ['PIL', 'Pillow'], +]; + +/** First interpreter that runs at all. Windows ships `python`, not `python3`. */ +function findPython() { + for (const candidate of [process.env.PYTHON, 'python3', 'python'].filter(Boolean)) { + const probe = spawnSync(candidate, ['-c', 'import sys; print(sys.version_info[0])'], { + encoding: 'utf8', + }); + if (probe.status === 0 && probe.stdout.trim() === '3') return candidate; + } + return undefined; +} + +export function extractPptx({args = []} = {}) { + if (!existsSync(script)) { + console.error(`Missing ${script}. This kit install is incomplete.`); + return 1; + } + + const python = findPython(); + if (!python) { + console.error( + 'No Python 3 interpreter found. Install one, or set PYTHON to its path.', + ); + return 1; + } + + const missing = REQUIRED.filter( + ([module]) => spawnSync(python, ['-c', `import ${module}`], {stdio: 'ignore'}).status !== 0, + ); + if (missing.length > 0) { + const packages = missing.map(([, pkg]) => pkg).join(' '); + console.error(`Python packages missing: ${packages}\n`); + console.error(` ${python} -m pip install ${packages}\n`); + console.error( + 'Extraction is a one-off authoring step, so these are deliberately not\n' + + 'project dependencies. A virtualenv is fine.', + ); + return 1; + } + + // Runs in the consumer repo, so relative --pptx / --output-root resolve + // against the workshop the operator is standing in. + const result = spawnSync(python, [script, ...args], { + stdio: 'inherit', + cwd: process.cwd(), + env: process.env, + }); + return result.status ?? 1; +} diff --git a/src/commands/zip-downloads.mjs b/src/commands/zip-downloads.mjs index b6be66b..098ffcf 100644 --- a/src/commands/zip-downloads.mjs +++ b/src/commands/zip-downloads.mjs @@ -50,6 +50,35 @@ function renderWorkshopTemplate(filePath) { writeFileSync(filePath, content); } +/** + * Deletes `static/downloads/*.zip` files that no longer have a matching + * `downloads//` folder. + * + * Without this, deleting a download folder left its ZIP behind forever. That is + * not merely untidy: `static/downloads/` is gitignored, so CI never sees the + * stale file and no review catches it, but `codedapp pack` packs the local + * `build/` directory and Docusaurus copies all of `static/` into it. A ZIP + * orphaned on one machine therefore kept shipping to the deployed site. CA + * served `exercise-1.zip` and `exercise-2.zip` for a month that way. + * + * Only `.zip` files are considered, so anything else a repo chooses to put in + * `static/downloads/` is left alone. + */ +function pruneOrphanedZips(folderNames) { + if (!existsSync(staticDownloadsRoot)) return; + + const expected = new Set(folderNames.map((name) => `${name}.zip`)); + for (const entry of readdirSync(staticDownloadsRoot, {withFileTypes: true})) { + if (!entry.isFile() || !entry.name.endsWith('.zip')) continue; + if (expected.has(entry.name)) continue; + + rmSync(join(staticDownloadsRoot, entry.name), {force: true}); + console.log( + `Removed orphaned static/downloads/${entry.name} (no matching downloads/ folder)`, + ); + } +} + /** Platform-aware archive command. From the IXP repo. */ function archiveCommand(zipPath) { if (process.platform !== 'win32') { @@ -91,6 +120,10 @@ for (const name of looseFiles) { ); } +// Before the early exit below: an emptied downloads/ means every ZIP present is +// an orphan, and that is exactly the case that must still be cleaned. +pruneOrphanedZips(downloadFolders); + if (downloadFolders.length === 0) { console.log('No download folders found. Skipping ZIP generation.'); process.exit(0); diff --git a/tools/extract-pptx-assets.py b/tools/extract-pptx-assets.py new file mode 100755 index 0000000..c57072e --- /dev/null +++ b/tools/extract-pptx-assets.py @@ -0,0 +1,417 @@ +#!/usr/bin/env python3 +"""Extract screenshots, text and callout geometry from a facilitator deck. + +Ran as a per-repo script copy in two workshops before it lived here. Those two +copies diverged, and the older one silently dropped every image that had been +dropped into a PowerPoint content placeholder rather than pasted as a picture -- +18 of 33 screenshots on one deck, with no error. That fix is at `is_picture` +below, and it is the reason this file is shared rather than copied. + +The two things that were genuinely per-deck in those copies -- which slides +belong to which docs page, and the role keywords -- are now a `--slide-map` +JSON file supplied by the caller. See `schemas/pptx-slide-map.schema.json`. + +Needs `python-pptx` and `Pillow`. The `workshop-kit extract-pptx` wrapper +checks for both before invoking this. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +from PIL import Image +from pptx import Presentation +from pptx.enum.shapes import MSO_SHAPE_TYPE + + +EMU_PER_INCH = 914400 +UNASSIGNED = "unassigned" + + +def inches(value: int | None) -> float | None: + if value is None: + return None + return round(value / EMU_PER_INCH, 3) + + +def shape_rect(shape: Any) -> dict[str, float | None]: + return { + "x_in": inches(getattr(shape, "left", None)), + "y_in": inches(getattr(shape, "top", None)), + "w_in": inches(getattr(shape, "width", None)), + "h_in": inches(getattr(shape, "height", None)), + } + + +def parse_slide_selector(entries: Any, context: str) -> list[tuple[int, int]]: + """`[2, "3-18"]` -> `[(2, 2), (3, 18)]`. Ranges are inclusive, as printed.""" + if entries is None: + return [] + if not isinstance(entries, list): + raise ValueError(f"{context}: \"slides\" must be a list") + + ranges: list[tuple[int, int]] = [] + for entry in entries: + if isinstance(entry, bool): + raise ValueError(f"{context}: {entry!r} is not a slide number") + if isinstance(entry, int): + ranges.append((entry, entry)) + continue + if isinstance(entry, str): + match = re.fullmatch(r"\s*(\d+)\s*-\s*(\d+)\s*", entry) + if match: + start, end = int(match.group(1)), int(match.group(2)) + if start > end: + raise ValueError(f"{context}: range \"{entry}\" runs backwards") + ranges.append((start, end)) + continue + if entry.strip().isdigit(): + number = int(entry.strip()) + ranges.append((number, number)) + continue + raise ValueError(f"{context}: {entry!r} is not a slide number or \"N-M\" range") + return ranges + + +class SlideMap: + """Deck-specific knowledge, loaded from JSON rather than compiled in. + + Without a map every slide lands in one output folder. That is the right + default for a first pass over an unfamiliar deck: you cannot write the + slide-to-page table until you have looked at what came out. + """ + + def __init__(self, data: dict[str, Any] | None, default_page: str) -> None: + data = data or {} + self.deck = data.get("deck") + self.default_page = default_page + self.pages: list[tuple[str, list[tuple[int, int]]]] = [] + self.roles: list[tuple[str, list[tuple[int, int]], list[str]]] = [] + + for index, entry in enumerate(data.get("pages", [])): + page = entry.get("page") + if not page: + raise ValueError(f"pages[{index}]: \"page\" is required") + self.pages.append((page, parse_slide_selector(entry.get("slides"), f"pages[{index}]"))) + + for index, entry in enumerate(data.get("roles", [])): + role = entry.get("role") + if not role: + raise ValueError(f"roles[{index}]: \"role\" is required") + match = entry.get("match", []) + if not isinstance(match, list) or any(not isinstance(term, str) for term in match): + raise ValueError(f"roles[{index}]: \"match\" must be a list of strings") + slides = parse_slide_selector(entry.get("slides"), f"roles[{index}]") + if not slides and not match: + raise ValueError(f"roles[{index}]: needs \"slides\", \"match\", or both") + self.roles.append((role, slides, [term.lower() for term in match])) + + @staticmethod + def load(path: Path | None, default_page: str) -> "SlideMap": + if path is None: + return SlideMap(None, default_page) + try: + data = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise SystemExit(f"{path}: not valid JSON ({exc})") from exc + try: + return SlideMap(data, default_page) + except ValueError as exc: + raise SystemExit(f"{path}: {exc}") from exc + + def page_for_slide(self, slide_number: int) -> str: + for page, ranges in self.pages: + if any(start <= slide_number <= end for start, end in ranges): + return page + return self.default_page + + def likely_role(self, slide_number: int, text: str) -> str: + """First matching rule wins, so order in the file is the priority order.""" + lowered = text.lower() + for role, ranges, match in self.roles: + if any(start <= slide_number <= end for start, end in ranges): + return role + if match and any(term in lowered for term in match): + return role + return self.page_for_slide(slide_number) + + +def collect_shapes(shapes: Any, parent: str | None = None) -> list[tuple[Any, str | None]]: + collected: list[tuple[Any, str | None]] = [] + for shape in shapes: + collected.append((shape, parent)) + if shape.shape_type == MSO_SHAPE_TYPE.GROUP: + collected.extend(collect_shapes(shape.shapes, getattr(shape, "name", None))) + return collected + + +def text_from_shape(shape: Any) -> str: + if not getattr(shape, "has_text_frame", False): + return "" + parts: list[str] = [] + for paragraph in shape.text_frame.paragraphs: + text = "".join(run.text for run in paragraph.runs).strip() + if not text: + text = paragraph.text.strip() + if text: + parts.append(text) + return "\n".join(parts).strip() + + +def image_size(path: Path) -> dict[str, int] | None: + try: + with Image.open(path) as image: + return {"width_px": image.width, "height_px": image.height} + except Exception: + return None + + +def candidate_sensitive_values(text: str) -> list[str]: + patterns = [ + r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", + r"https?://[^\s)>\"]+", + r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b", + r"\b(?:tenant|org|folder|client|secret|token|key)[\w -]{0,24}[:=]\s*[^\s,;]+", + ] + values: list[str] = [] + for pattern in patterns: + values.extend(re.findall(pattern, text, flags=re.IGNORECASE)) + return sorted(set(values)) + + +def callout_markers(text_blocks: list[dict], picture_blocks: list[dict]) -> list[dict]: + """Numbered ovals in a lab deck are separate shapes with text ("1", "2", ...). + + They are lost the moment a screenshot is extracted on its own, so record each + one as a percentage offset inside the picture it overlaps. Those percentages + drop straight into the kit's `.workshop-click-marker` overlay in MDX. + """ + markers = [] + for block in text_blocks: + label = block["text"].strip() + if not re.fullmatch(r"[0-9]{1,2}[a-z]?", label): + continue + rect = block["coordinates"] + if None in (rect["x_in"], rect["y_in"], rect["w_in"], rect["h_in"]): + continue + cx = rect["x_in"] + rect["w_in"] / 2 + cy = rect["y_in"] + rect["h_in"] / 2 + best = None + for picture in picture_blocks: + pr = picture["coordinates"] + if None in (pr["x_in"], pr["y_in"], pr["w_in"], pr["h_in"]): + continue + if not (pr["x_in"] <= cx <= pr["x_in"] + pr["w_in"]): + continue + if not (pr["y_in"] <= cy <= pr["y_in"] + pr["h_in"]): + continue + best = picture + break + markers.append( + { + "label": label, + "shape_id": block["shape_id"], + "slide_center_in": {"x": round(cx, 3), "y": round(cy, 3)}, + "on_picture": best["filename"] if best else None, + "left_pct": ( + round((cx - best["coordinates"]["x_in"]) / best["coordinates"]["w_in"] * 100, 1) + if best + else None + ), + "top_pct": ( + round((cy - best["coordinates"]["y_in"]) / best["coordinates"]["h_in"] * 100, 1) + if best + else None + ), + } + ) + return markers + + +def extract( + pptx_path: Path, + output_root: Path, + manifest_path: Path, + slide_map: SlideMap, +) -> dict[str, Any]: + prs = Presentation(str(pptx_path)) + output_root.mkdir(parents=True, exist_ok=True) + + manifest: dict[str, Any] = { + "source_pptx": str(pptx_path), + "slide_map_deck": slide_map.deck, + "slide_width_in": inches(prs.slide_width), + "slide_height_in": inches(prs.slide_height), + "slides": [], + "sensitive_candidates": [], + "notes": [ + "Extracted with python-pptx from slide text frames and embedded picture shapes.", + "Coordinates are in inches from the top-left of the slide canvas.", + "Picture files are saved from embedded image blobs; no whole-slide renders are used by this extractor.", + "callout_markers pairs each numbered oval with the picture it overlaps, as percentage offsets for .workshop-click-marker.", + ], + } + + sensitive_seen: set[tuple[int, str]] = set() + + for slide_index, slide in enumerate(prs.slides, start=1): + flat_shapes = collect_shapes(slide.shapes) + text_blocks = [] + picture_blocks = [] + shape_notes = [] + + for shape, parent in flat_shapes: + text = text_from_shape(shape) + if text: + text_blocks.append( + { + "shape_id": shape.shape_id, + "name": shape.name, + "parent_group": parent, + "text": text, + "coordinates": shape_rect(shape), + } + ) + for value in candidate_sensitive_values(text): + key = (slide_index, value) + if key not in sensitive_seen: + sensitive_seen.add(key) + manifest["sensitive_candidates"].append( + {"slide": slide_index, "value": value, "source": "slide_text"} + ) + + # `hasattr(shape, "image")` is the whole point: an image dropped into + # a content placeholder is a PlaceholderPicture, whose shape_type is + # PLACEHOLDER, not PICTURE. Filtering on shape_type alone loses them. + is_picture = shape.shape_type == MSO_SHAPE_TYPE.PICTURE or hasattr(shape, "image") + if is_picture: + try: + image = shape.image + except (ValueError, AttributeError) as exc: + shape_notes.append( + { + "shape_id": shape.shape_id, + "name": shape.name, + "shape_type": str(shape.shape_type), + "parent_group": parent, + "coordinates": shape_rect(shape), + "note": f"Picture-shaped object has no embedded image blob: {exc}", + } + ) + continue + ext = image.ext or "bin" + digest = hashlib.sha1(image.blob).hexdigest()[:10] + page = slide_map.page_for_slide(slide_index) + filename = f"pptx-slide-{slide_index:02d}-picture-{shape.shape_id}-{digest}.{ext}" + out_dir = output_root / page + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / filename + out_path.write_bytes(image.blob) + picture_info = { + "shape_id": shape.shape_id, + "name": shape.name, + "parent_group": parent, + "filename": str(out_path.relative_to(output_root.parent)), + "extension": ext, + "blob_sha1": hashlib.sha1(image.blob).hexdigest(), + "coordinates": shape_rect(shape), + "display_size_in": { + "width": inches(shape.width), + "height": inches(shape.height), + }, + "intrinsic_size_px": image_size(out_path), + } + picture_blocks.append(picture_info) + + if not is_picture and not text: + name = getattr(shape, "name", "") + if any(token in name.lower() for token in ["callout", "arrow", "rectangle", "line", "freeform", "oval"]): + shape_notes.append( + { + "shape_id": shape.shape_id, + "name": name, + "shape_type": str(shape.shape_type), + "parent_group": parent, + "coordinates": shape_rect(shape), + "note": "Non-picture shape; may be a separate callout, box, arrow, or UI highlight.", + } + ) + + text_blocks.sort(key=lambda item: (item["coordinates"]["y_in"] or 0, item["coordinates"]["x_in"] or 0)) + picture_blocks.sort(key=lambda item: (item["coordinates"]["y_in"] or 0, item["coordinates"]["x_in"] or 0)) + + title = text_blocks[0]["text"] if text_blocks else "" + subtitle = text_blocks[1]["text"] if len(text_blocks) > 1 else "" + all_text = "\n".join(block["text"] for block in text_blocks) + + manifest["slides"].append( + { + "slide_number": slide_index, + "page_assignment": slide_map.page_for_slide(slide_index), + "likely_exercise_role": slide_map.likely_role(slide_index, all_text), + "title": title, + "subtitle": subtitle, + "ordered_text_blocks": text_blocks, + "embedded_pictures": picture_blocks, + "separate_shape_notes": shape_notes, + "callout_markers": callout_markers(text_blocks, picture_blocks), + } + ) + + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="workshop-kit extract-pptx", + description="Extract screenshots, text and callout geometry from a facilitator deck.", + ) + parser.add_argument("--pptx", required=True, type=Path) + parser.add_argument("--output-root", required=True, type=Path) + parser.add_argument("--manifest", required=True, type=Path) + parser.add_argument( + "--slide-map", + type=Path, + help="JSON mapping slides to docs pages and roles. Without it every slide " + "goes to one folder, which is the right first pass over a new deck.", + ) + parser.add_argument( + "--default-page", + default=UNASSIGNED, + help=f"Folder for slides no page rule claims (default: {UNASSIGNED}).", + ) + args = parser.parse_args() + + if not args.pptx.is_file(): + raise SystemExit(f"{args.pptx}: no such file") + + slide_map = SlideMap.load(args.slide_map, args.default_page) + manifest = extract(args.pptx, args.output_root, args.manifest, slide_map) + + picture_count = sum(len(slide["embedded_pictures"]) for slide in manifest["slides"]) + text_count = sum(len(slide["ordered_text_blocks"]) for slide in manifest["slides"]) + marker_count = sum(len(slide["callout_markers"]) for slide in manifest["slides"]) + print(f"Extracted {len(manifest['slides'])} slides, {text_count} text blocks, " + f"{picture_count} picture shapes, {marker_count} callout markers") + + # A deck run without a map, or with a map written against a different + # revision of the deck, dumps everything into one folder. Say so, because + # the run otherwise looks like a success. + unassigned = [s["slide_number"] for s in manifest["slides"] if s["page_assignment"] == args.default_page] + if unassigned: + preview = ", ".join(str(number) for number in unassigned[:12]) + more = f" (+{len(unassigned) - 12} more)" if len(unassigned) > 12 else "" + print(f"{len(unassigned)} slides landed in {args.default_page}/: {preview}{more}") + + print(f"Manifest: {args.manifest}") + print(f"Image root: {args.output_root}") + + +if __name__ == "__main__": + main()