Canvas test harness, TypeScript IMSCC validator in CI, real-Canvas fixtures, refreshed CLAUDE.md

.gitattributes

@@ -0,0 +1,6 @@
+# Shell and Ruby scripts must keep LF even on Windows checkouts (core.autocrlf=true would
+# otherwise give them CRLF and bash/ruby fail with "\r: command not found").
+*.sh text eol=lf
+*.rb text eol=lf
+# Canvas cartridges are zip archives.
+*.imscc binary

.gitignore

@@ -37,6 +37,8 @@ deploy-courses/
 # Local template uploads
 *.imscc
 *.docx
+# ...except the scrubbed, Canvas-made cartridges the tests run against
+!tests/fixtures/*.imscc
 
 # Design scratch + internal references (not for the public repo)
 ClassBuild Redesign/

README.md

@@ -101,7 +101,7 @@ services, setup components). Pages — `BuildPage`, `SyllabusPage`,
 
 Without an uploaded template, "Export for Canvas (.imscc)" writes a **native Canvas course export** — the same file layout Canvas's own exporter produces (`module_meta.xml`, `wiki_content/` pages, `non_cc_assessments/` quizzes, `assessment_meta.xml`, `assignment_groups.xml`, syllabus). This matters: a package carrying Canvas's `course_settings/canvas_export.txt` marker is read by Canvas's *native* importer, which silently ignores plain Common Cartridge manifest content, so a "generic" cartridge with that marker imports with no modules, pages or quizzes.
 
-Check any cartridge offline with `python tools/validate-imscc.py file.imscc` (it flags exactly that class of problem). To test imports end to end you need a Canvas with Instructure's QTIMigrationTool installed, otherwise quizzes are skipped silently.
+Check any cartridge offline with `npm run validate:imscc -- file.imscc` (it flags exactly that class of problem and runs in CI against real Canvas-made fixtures). To test an import end to end, use the Canvas harness in [`tools/canvas-test/`](tools/canvas-test/README.md) — it builds a dev Canvas (with Instructure's QTIMigrationTool; without it quizzes are skipped silently) and shows what Canvas actually built.
 
 Current coverage:
 - `tests/template-parser.test.ts` — module classification (verbatim / pattern / example-pattern), prefix detection (`Module N:`, `MN Instructor Notes:`, fully-locked `Module N Overview`), `(Example to Edit)` placeholder marker, `**EDIT**` markers, example-pattern content extraction.

package.json

@@ -33,7 +33,8 @@
     "preview": "vite preview",
     "test": "vitest run",
     "test:watch": "vitest",
-    "test:coverage": "vitest run --coverage"
+    "test:coverage": "vitest run --coverage",
+    "validate:imscc": "tsx tools/validate-imscc.ts"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.74.0",

src/services/export/validateImscc.ts

@@ -0,0 +1,264 @@
+/**
+ * Offline structural validation of a Common Cartridge (.imscc) destined for
+ * Canvas. Not imported by the app (so it is never bundled) — it is used by the
+ * test suite, by `tools/validate-imscc.ts`, and could back an in-app "check
+ * before download" later.
+ *
+ * Why this exists: Canvas happily reports "imported, no issues" for a cartridge
+ * that silently drops content. A package carrying `course_settings/canvas_export.txt`
+ * is classified as a *native Canvas export*, and Canvas's native converter reads
+ * only Canvas's own files (module_meta.xml, wiki_content/, non_cc_assessments/…)
+ * while ignoring plain manifest content. These rules catch that class of problem
+ * without needing a live Canvas.
+ *
+ * This is a structural check only — it cannot prove Canvas will accept a package.
+ * An import into a real Canvas (see tools/canvas-test/) remains the source of truth.
+ */
+import JSZip from 'jszip';
+
+export interface ValidationReport {
+  /** Hard errors: Canvas will drop or mangle content. */
+  problems: string[];
+  /** Things worth a look but not fatal. */
+  warnings: string[];
+  info: string[];
+  ok: boolean;
+}
+
+export type ImsccInput = Uint8Array | ArrayBuffer | Blob;
+
+const NATIVE_MARKER = 'course_settings/canvas_export.txt';
+const QTI_RESOURCE = 'imsqti_xmlv1p2/imscc_xmlv1p1/assessment';
+const RESOURCE_BACKED_ITEMS = new Set(['WikiPage', 'Quizzes::Quiz', 'DiscussionTopic', 'Attachment', 'Assignment']);
+const KNOWN_ITEM_TYPES = new Set([
+  ...RESOURCE_BACKED_ITEMS,
+  'ExternalUrl',
+  'ContextExternalTool',
+  'ContextModuleSubHeader',
+]);
+const KNOWN_RESOURCE_TYPE =
+  /^(webcontent|imsdt_xmlv1p\d|imsqti_xmlv1p\d\/imscc_xmlv1p\d\/assessment|associatedcontent\/imscc_xmlv1p\d\/learning-application-resource|imsbasiclti_xmlv1p\d|imswl_xmlv1p\d|imsqti_xmlv1p\d\/.*)$/;
+
+function local(el: Element): string {
+  return el.localName || el.tagName.replace(/^.*:/, '');
+}
+
+function descendants(root: Document | Element, name: string): Element[] {
+  return Array.from(root.getElementsByTagName('*')).filter((e) => local(e) === name);
+}
+
+function childText(el: Element, name: string): string | null {
+  const c = Array.from(el.children).find((x) => local(x) === name);
+  return c ? (c.textContent ?? '') : null;
+}
+
+function parseXml(text: string): Document | null {
+  try {
+    const doc = new DOMParser().parseFromString(text, 'text/xml');
+    if (!doc || doc.getElementsByTagName('parsererror').length > 0) return null;
+    return doc;
+  } catch {
+    return null;
+  }
+}
+
+export async function validateImscc(input: ImsccInput): Promise<ValidationReport> {
+  const problems: string[] = [];
+  const warnings: string[] = [];
+  const info: string[] = [];
+
+  let zip: JSZip;
+  try {
+    zip = await JSZip.loadAsync(input as Blob);
+  } catch (e) {
+    return { problems: [`not a readable zip archive: ${(e as Error).message}`], warnings, info, ok: false };
+  }
+
+  const names = Object.keys(zip.files).filter((n) => !zip.files[n].dir);
+  const nameSet = new Set(names);
+  const read = (n: string) => zip.file(n)!.async('string');
+
+  // 1. Archive basics
+  if (!nameSet.has('imsmanifest.xml')) {
+    return { problems: ['imsmanifest.xml missing at archive root'], warnings, info, ok: false };
+  }
+  const backslashed = names.filter((n) => n.includes('\\'));
+  if (backslashed.length) problems.push(`backslash paths in archive: ${backslashed.slice(0, 3).join(', ')}`);
+
+  // 2. Every XML part parses
+  let xmlOk = 0;
+  for (const n of names) {
+    if (/\.(xml|qti)$/.test(n)) {
+      const text = await read(n);
+      // DOM parsers are lenient about a bare "&" (e.g. a title like "R&D"), but Canvas's
+      // XML parser is not — check for it explicitly.
+      if (/&(?!(?:amp|lt|gt|quot|apos|#\d+|#x[0-9a-fA-F]+);)/.test(text)) {
+        problems.push(`unescaped "&" in XML: ${n}`);
+      } else if (parseXml(text)) xmlOk++;
+      else problems.push(`not well-formed XML: ${n}`);
+    }
+  }
+  info.push(`${xmlOk} XML files parse cleanly; ${names.length} entries total`);
+
+  // 3. Manifest integrity
+  const manifestDoc = parseXml(await read('imsmanifest.xml'));
+  if (!manifestDoc) return { problems: [...problems, 'imsmanifest.xml is not well-formed'], warnings, info, ok: false };
+  const man = manifestDoc.documentElement;
+  info.push(`manifest namespace: ${man.getAttribute('xmlns') ?? '(none)'}`);
+
+  const resources = new Map<string, Element>();
+  for (const r of descendants(man, 'resource')) resources.set(r.getAttribute('identifier') ?? '', r);
+
+  const allIds: string[] = [];
+  for (const e of descendants(man, 'resource')) allIds.push(e.getAttribute('identifier') ?? '');
+  for (const e of descendants(man, 'item')) if (e.getAttribute('identifier')) allIds.push(e.getAttribute('identifier')!);
+  const counts = new Map<string, number>();
+  for (const id of allIds) counts.set(id, (counts.get(id) ?? 0) + 1);
+  const dupIds = [...counts].filter(([, c]) => c > 1).map(([id]) => id);
+  if (dupIds.length) problems.push(`duplicate identifiers in manifest: ${dupIds.slice(0, 5).join(', ')}`);
+
+  const claimed = new Set<string>();
+  const missingFiles: string[] = [];
+  for (const [rid, r] of resources) {
+    const hrefs = [r.getAttribute('href'), ...descendants(r, 'file').map((f) => f.getAttribute('href'))].filter(
+      (h): h is string => !!h,
+    );
+    for (const h of hrefs) {
+      claimed.add(h);
+      if (!nameSet.has(h)) missingFiles.push(`${rid}→${h}`);
+    }
+  }
+  if (missingFiles.length) {
+    problems.push(
+      `${missingFiles.length} manifest references to files not in the archive, e.g. ${missingFiles.slice(0, 3).join(', ')}`,
+    );
+  }
+
+  // Organization items must point at real resources (empty = no resource, e.g. a sub-header).
+  const orgBad: string[] = [];
+  let orgItems = 0;
+  for (const item of descendants(man, 'item')) {
+    orgItems++;
+    const ref = item.getAttribute('identifierref');
+    if (ref && !resources.has(ref)) orgBad.push(ref);
+  }
+  if (orgBad.length) problems.push(`item identifierref pointing at unknown resources: ${orgBad.slice(0, 5).join(', ')}`);
+  info.push(`${resources.size} resources, ${orgItems} organization items`);
+
+  const unclaimed = names.filter((n) => n !== 'imsmanifest.xml' && !claimed.has(n));
+  if (unclaimed.length) {
+    warnings.push(
+      `${unclaimed.length} archive files not claimed by any resource (Canvas ignores them): ${unclaimed.slice(0, 4).join(', ')}`,
+    );
+  }
+
+  // 4. Resource types
+  const typeCounts = new Map<string, number>();
+  for (const r of resources.values()) {
+    const t = r.getAttribute('type') ?? '';
+    typeCounts.set(t, (typeCounts.get(t) ?? 0) + 1);
+    if (!KNOWN_RESOURCE_TYPE.test(t)) warnings.push(`unrecognised resource type for ${r.getAttribute('identifier')}: ${t}`);
+  }
+  info.push('resource types: ' + [...typeCounts].map(([k, v]) => `${k} x${v}`).join(', '));
+
+  // 5. QTI sanity
+  let qtiFiles = 0;
+  for (const n of names.filter((x) => /\.(xml|qti)$/.test(x))) {
+    const text = await read(n);
+    if (!text.includes('<questestinterop')) continue;
+    qtiFiles++;
+    const doc = parseXml(text);
+    if (!doc) continue;
+    const items = descendants(doc, 'item');
+    if (items.length === 0) problems.push(`QTI file with no items: ${n}`);
+    for (const it of items) {
+      if (descendants(it, 'response_lid').length === 0) warnings.push(`${n}: item ${it.getAttribute('ident')} has no response_lid`);
+      if (descendants(it, 'varequal').length === 0) {
+        warnings.push(`${n}: item ${it.getAttribute('ident')} has no varequal (no correct answer wired)`);
+      }
+    }
+  }
+  info.push(`${qtiFiles} QTI assessment files`);
+
+  // 6. Native-Canvas-format rules
+  if (nameSet.has(NATIVE_MARKER)) {
+    info.push('native Canvas export marker present: applying native-format rules');
+
+    if (!nameSet.has('course_settings/module_meta.xml')) {
+      problems.push(
+        'native marker present but course_settings/module_meta.xml is missing: Canvas will create NO modules',
+      );
+    } else {
+      const mm = parseXml(await read('course_settings/module_meta.xml'));
+      if (mm) {
+        const orgIds = new Set(descendants(man, 'item').map((e) => e.getAttribute('identifier')));
+        const modItems = descendants(mm, 'item');
+        for (const it of modItems) {
+          const ref = childText(it, 'identifierref');
+          const ctype = childText(it, 'content_type') ?? '';
+          if (RESOURCE_BACKED_ITEMS.has(ctype) && (!ref || !resources.has(ref))) {
+            problems.push(`module_meta ${ctype} item points at unknown resource ${ref}`);
+          }
+          if (!orgIds.has(it.getAttribute('identifier'))) {
+            warnings.push(`module_meta item ${it.getAttribute('identifier')} is not in the manifest organization`);
+          }
+          if (!KNOWN_ITEM_TYPES.has(ctype)) warnings.push(`unusual module item content_type ${ctype}`);
+        }
+        info.push(`${descendants(mm, 'module').length} modules, ${modItems.length} module items in module_meta.xml`);
+      }
+    }
+
+    for (const n of names.filter((x) => x.startsWith('wiki_content/') && x.endsWith('.html'))) {
+      const body = await read(n);
+      if (!claimed.has(n)) {
+        problems.push(
+          `${n}: not listed in the manifest, but the native importer scans wiki_content/ directly: it would be imported as a stray page`,
+        );
+      }
+      if (!body.includes('name="identifier"')) {
+        problems.push(`${n}: wiki page has no <meta name="identifier">: Canvas cannot match it`);
+      }
+      if (/<script|<style/i.test(body)) warnings.push(`${n}: <script>/<style> will be stripped by Canvas`);
+    }
+
+    for (const [rid, r] of resources) {
+      if (r.getAttribute('type') !== QTI_RESOURCE) continue;
+      const dep = descendants(r, 'dependency')[0]?.getAttribute('identifierref');
+      const metaRes = dep ? resources.get(dep) : undefined;
+      if (!metaRes) {
+        problems.push(`quiz resource ${rid} has no assessment_meta dependency`);
+        continue;
+      }
+      const mfiles = descendants(metaRes, 'file').map((f) => f.getAttribute('href') ?? '');
+      if (!mfiles.some((f) => f.startsWith('non_cc_assessments/') && f.endsWith('.xml.qti'))) {
+        problems.push(
+          `quiz ${rid}: native importer needs non_cc_assessments/<id>.xml.qti in its meta resource - quiz would import with no questions`,
+        );
+      }
+      if (!mfiles.some((f) => f.endsWith('assessment_meta.xml'))) {
+        problems.push(`quiz ${rid}: meta resource lacks assessment_meta.xml`);
+      }
+    }
+    for (const n of names.filter((x) => x.startsWith('non_cc_assessments/') && x.endsWith('.xml.qti'))) {
+      const t = await read(n);
+      if (t.includes('<item ') && !t.includes('question_type')) problems.push(`${n}: items lack the Canvas question_type metadata`);
+    }
+  }
+
+  // 7. External hosts referenced from pages (need internet in Canvas)
+  const external = new Map<string, number>();
+  for (const n of names.filter((x) => x.endsWith('.html'))) {
+    for (const m of (await read(n)).matchAll(/(?:src|href)=["'](https?:\/\/[^"']+)/g)) {
+      const host = m[1].split('/')[2];
+      external.set(host, (external.get(host) ?? 0) + 1);
+    }
+  }
+  if (external.size) {
+    warnings.push(
+      'external hosts referenced from pages (need internet in Canvas): ' +
+        [...external].map(([h, c]) => `${h} x${c}`).join(', '),
+    );
+  }
+
+  return { problems, warnings, info, ok: problems.length === 0 };
+}

tests/build-demo-cartridge.test.ts

@@ -0,0 +1,56 @@
+import { it } from 'vitest';
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { assembleImscc } from '../src/services/export/imsccExporter';
+import { parseImsccTemplate } from '../src/services/template/parser';
+import { assembleTemplateImscc } from '../src/services/template/templateImsccExporter';
+import { demoSyllabus, demoChapters } from '../src/fixtures/demoCourse';
+import type { Syllabus, GeneratedChapter } from '../src/types/course';
+
+/**
+ * NOT a real test — a generator for the Canvas test harness (tools/canvas-test).
+ * It only runs when CCT_BUILD is set, so it is skipped in normal `npm test` / CI.
+ * It runs under vitest because the exporters need a browser-like environment
+ * (Blob/DOMParser) that plain Node does not provide.
+ *
+ *   tools/canvas-test/build-demo-cartridge.sh
+ *
+ * writes output/demo-course-native.imscc (no-template export of the demo course) and
+ * output/demo-template-export.imscc (template export, using the Canvas-made template
+ * fixture and canned chapter content — no LLM).
+ */
+const out = (name: string) => `output/${name}`;
+
+it.runIf(process.env.CCT_BUILD)('writes the demo cartridges for the Canvas test harness', async () => {
+  mkdirSync('output', { recursive: true });
+
+  const native = await assembleImscc(demoSyllabus, demoChapters, { themeId: 'press', curriculumCsv: 'a,b\n1,2\n' });
+  writeFileSync(out('demo-course-native.imscc'), Buffer.from(await native.arrayBuffer()));
+
+  const tplBytes = new Uint8Array(readFileSync('tests/fixtures/canvas-instructor-template.imscc'));
+  const template = await parseImsccTemplate({ file: tplBytes, name: 'Instructor Template' });
+  const syllabus: Syllabus = {
+    courseTitle: 'Cell Biology (Template Test)',
+    courseOverview: 'A short course on cells.',
+    chapters: [1, 2, 3].map((n) => ({
+      number: n,
+      title: `Module ${n}: ${['The Cell', 'Genes', 'Proteins'][n - 1]}`,
+      narrative: 'n',
+      keyConcepts: ['c'],
+      widgets: [],
+      scienceAnnotations: [],
+      spacingConnections: [],
+    })),
+  };
+  const chapters = syllabus.chapters.map((c) => ({
+    number: c.number,
+    title: c.title,
+    htmlContent: '',
+    templateContent: {
+      moduleOverviewHtml: `<h2>${c.title} overview</h2><p>What you will learn in module ${c.number}.</p>`,
+      instructorNotes: [{ title: `Pacing for ${c.title}`, htmlContent: `<p>Warm up for module ${c.number}.</p>` }],
+      discussion: { title: `Big question ${c.number}`, promptHtml: `<p>Discuss module ${c.number}.</p>` },
+    },
+  })) as GeneratedChapter[];
+  const tpl = await assembleTemplateImscc({ syllabus, chapters, template, templateBlob: tplBytes });
+  writeFileSync(out('demo-template-export.imscc'), Buffer.from(await tpl.arrayBuffer()));
+});