E2E coverage for every student probeable workflow, plus the six defects it exposed

.github/workflows/e2e.yml

@@ -18,6 +18,15 @@ on:
       - 'playwright.config.ts'
       - 'package.json'
       - '.github/workflows/e2e.yml'
+  pull_request:
+    paths:
+      - 'purplex/**'
+      - 'tests/**'
+      - 'e2e/**'
+      - 'requirements.txt'
+      - 'playwright.config.ts'
+      - 'package.json'
+      - '.github/workflows/e2e.yml'
   workflow_dispatch:
 
 env:

e2e/helpers/selectors.ts

@@ -240,3 +240,63 @@ export const common = {
   /** Dialog overlay (ConfirmDialog uses .dialog-overlay) */
   dialogOverlay: '.dialog-overlay',
 } as const;
+
+// ---------------------------------------------------------------------------
+// Probe panel (activities/inputs/shared/ProbePanel.vue)
+// ---------------------------------------------------------------------------
+export const probe = {
+  /** Panel root */
+  panel: '.probe-panel',
+  /** Input for one parameter (Enter triggers a probe) */
+  paramInput: (name: string) => `input#probe-${name}.param-input`,
+  /** Result slot ('?' until a result; .has-result / .is-cached) */
+  result: '.probe-panel .fn-result',
+  /** Probe button ('Probe' | 'Probing' | 'Already probed', .is-duplicate) */
+  button: 'button.probe-btn',
+  /** Remaining-probe badge (.status-explore | .status-available | .status-exhausted) */
+  badge: 'span.probe-count-badge',
+  /** Status hint, rendered only when probes are exhausted */
+  hint: '.probe-panel .probe-status-hint',
+  /** Oracle / request error line */
+  error: '.probe-panel .probe-error',
+  /** Per-parameter field errors */
+  fieldErrors: 'ul.probe-field-errors',
+  /** One parameter's field error */
+  fieldError: (name: string) => `li#probe-err-${name}`,
+  /** 'History (N)' button, rendered only when history is non-empty */
+  historyButton: '.probe-panel .history-btn',
+  /** History modal (teleported to body) */
+  historyModal: '.modal-overlay .modal-content[role="dialog"]',
+  /** Rows in the history modal */
+  historyRow: '.modal-content[role="dialog"] .modal-row',
+  /** History modal close button */
+  close: '.modal-content[role="dialog"] .modal-close',
+} as const;
+
+// ---------------------------------------------------------------------------
+// Code submission feedback (activities/feedback/CodeSubmissionFeedback.vue)
+// ---------------------------------------------------------------------------
+export const codeFeedback = {
+  /** Summary bar with pass/fail/locked counts */
+  summaryBar: '.feedback-container .test-summary-bar',
+  /** '✓ N Passing' */
+  passing: '.test-summary-bar .count-item.passing',
+  /** '✗ N Failing' */
+  failing: '.test-summary-bar .count-item.failing',
+  /** '🔒 N more tests locked' (only when locked > 0) */
+  locked: '.test-summary-bar .count-item.locked',
+  /** Failing test rows (.test-call, .expected, .actual, .actual.is-error) */
+  failingItem: 'article.test-item.failing',
+  /** Passing test rows (inside the collapsed passing group) */
+  passingItem: 'article.test-item.passing',
+  /** Note in the failing group explaining locked tests */
+  lockedNote: 'p.locked-note',
+  /** Attempts dropdown trigger (.attempt-text 'n/total', .attempt-score-badge) */
+  attemptTrigger: '.attempt-dropdown-trigger',
+  /** Attempts dropdown panel */
+  attemptPanel: '.attempt-dropdown-panel[role="menu"]',
+  /** One attempt in the dropdown (.attempt-tests 'passed/total') */
+  attemptItem: '.attempt-item-minimal',
+  /** Debug button on a test row */
+  debugButton: '.debug-btn',
+} as const;

e2e/helpers/submission.ts

@@ -0,0 +1,87 @@
+import { Page, expect } from '@playwright/test';
+import { setAceEditorValue } from './ace-editor';
+import { codeFeedback } from './selectors';
+
+/**
+ * Code-submission helpers for probeable_code problems.
+ *
+ * In the E2E stack (CELERY_EAGER=true) POST /api/submit/ grades the whole
+ * submission synchronously in the Docker sandbox, then the SSE stream reports
+ * completion. ProblemSet.vue clears the previous feedback before it sends the
+ * request, so once the submit response is back, a visible summary bar can only
+ * belong to the new result.
+ */
+
+const SUBMIT_URL = /\/api\/submit\/$/;
+
+/**
+ * Put `code` in the editor, submit it, and wait until the graded result's
+ * summary bar is visible. Throws if the submit request is not accepted.
+ */
+export async function submitCodeAndWait(
+  page: Page,
+  code: string,
+  timeout = 60_000,
+): Promise<void> {
+  await setAceEditorValue(page, '#codeEditor', code);
+
+  const submitButton = page.locator('#submitButton');
+  await expect(submitButton).toBeEnabled();
+
+  const responsePromise = page.waitForResponse(
+    (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST',
+    { timeout },
+  );
+  await submitButton.click();
+  const response = await responsePromise;
+  if (!response.ok()) {
+    throw new Error(
+      `submitCodeAndWait: POST /api/submit/ returned ${response.status()}: ${await response.text()}`,
+    );
+  }
+
+  await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout });
+}
+
+/**
+ * Submit code that the API rejects as invalid (e.g. a syntax error) and return
+ * the message the user sees.
+ *
+ * The probeable_code handler rejects uncompilable code with 400
+ * {"error": "Syntax error: <msg> (line N)"} before any submission exists.
+ * ProblemSet.vue's submit() catch shows it as a warning toast titled
+ * "Invalid Request" whose .notification-details is the server's error text.
+ * This asserts that toast and returns the details text.
+ */
+export async function submitSyntaxErrorAndReadMessage(page: Page, code: string): Promise<string> {
+  await setAceEditorValue(page, '#codeEditor', code);
+
+  const submitButton = page.locator('#submitButton');
+  await expect(submitButton).toBeEnabled();
+
+  const responsePromise = page.waitForResponse(
+    (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST',
+  );
+  await submitButton.click();
+  const response = await responsePromise;
+  if (response.status() !== 400) {
+    throw new Error(
+      `submitSyntaxErrorAndReadMessage: expected 400 from POST /api/submit/, got ${response.status()}`,
+    );
+  }
+
+  const body = await response.json();
+  if (typeof body?.error !== 'string' || body.error === '') {
+    throw new Error(`submitSyntaxErrorAndReadMessage: 400 without an error message: ${JSON.stringify(body)}`);
+  }
+
+  // Match the toast to this response (an older toast may still be on screen).
+  // Toasts auto-dismiss after 4-5 s, so read the text as soon as it shows.
+  const toast = page
+    .locator('.notification-container .notification.warning')
+    .filter({ has: page.locator('.notification-details', { hasText: body.error }) })
+    .last();
+  await expect(toast).toBeVisible();
+  await expect(toast.locator('.notification-message')).toHaveText('Invalid Request');
+  return ((await toast.locator('.notification-details').textContent()) ?? '').trim();
+}

e2e/helpers/users.ts

@@ -0,0 +1,159 @@
+import { Page } from '@playwright/test';
+
+/**
+ * Fresh, per-test student users.
+ *
+ * Probe budgets and probe history live in Redis per (problem, user) with a
+ * 7-day TTL and no reset, and submissions accumulate per user. Any test that
+ * touches a budget, history or submissions must therefore run as a user no
+ * other test (or earlier run) has used.
+ *
+ * Mock Firebase accepts any email: the backend (users_app/mock_firebase.py)
+ * creates a student for an unknown email using the uid from the token, and
+ * the frontend MockFirebase restores its session from localStorage.
+ */
+export interface FreshUser {
+  uid: string;
+  email: string;
+  displayName: string;
+}
+
+/** Create a unique user identity. Nothing is sent to the server. */
+export function freshStudent(label: string): FreshUser {
+  const safeLabel = label.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
+  const id = `e2e-${safeLabel}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  return {
+    uid: `mock-uid-${id}`,
+    email: `${id}@test.local`,
+    displayName: `E2E ${label}`,
+  };
+}
+
+/**
+ * Inject this user into localStorage so the Vue app boots as them.
+ * Mirrors injectAuth in auth.ts: Vuex `user`, `mockFirebaseAuth`, and the
+ * cookie consent flag. Must be called before navigating to the target route.
+ */
+export async function injectUser(page: Page, user: FreshUser): Promise<void> {
+  await page.goto('/', { waitUntil: 'commit' });
+
+  await page.evaluate(
+    ({ userJson, mockFirebaseAuthJson, consent }) => {
+      localStorage.setItem('user', userJson);
+      localStorage.setItem('mockFirebaseAuth', mockFirebaseAuthJson);
+      localStorage.setItem('purplex_cookie_consent', consent);
+    },
+    {
+      userJson: JSON.stringify({
+        uid: user.uid,
+        email: user.email,
+        displayName: user.displayName,
+        role: 'user',
+        isAdmin: false,
+      }),
+      mockFirebaseAuthJson: JSON.stringify({
+        uid: user.uid,
+        email: user.email,
+        displayName: user.displayName,
+      }),
+      consent: 'accepted',
+    },
+  );
+}
+
+/**
+ * Mint a mock token for this user. Same payload as the frontend's
+ * MockFirebase.createMockToken, which the backend decodes as
+ * MOCK.<base64 json>.development.
+ */
+function mockTokenFor(user: FreshUser): string {
+  const now = Math.floor(Date.now() / 1000);
+  const payload = {
+    uid: user.uid,
+    email: user.email,
+    name: user.displayName,
+    email_verified: true,
+    iat: now,
+    exp: now + 3600,
+  };
+  return `MOCK.${btoa(JSON.stringify(payload))}.development`;
+}
+
+/**
+ * Call the API as this user from inside the page (same origin, through the
+ * Vite proxy). The page must already be on the app origin.
+ */
+export async function apiAsUser(
+  page: Page,
+  user: FreshUser,
+  method: string,
+  path: string,
+  body?: object,
+): Promise<{ status: number; data: any }> {
+  const token = mockTokenFor(user);
+  return page.evaluate(
+    async ({ token, method, path, body }) => {
+      const res = await fetch(path, {
+        method,
+        headers: {
+          Authorization: `Bearer ${token}`,
+          'Content-Type': 'application/json',
+        },
+        body: body ? JSON.stringify(body) : undefined,
+      });
+      const text = await res.text();
+      let data: unknown = null;
+      if (text) {
+        try {
+          data = JSON.parse(text);
+        } catch {
+          data = text;
+        }
+      }
+      return { status: res.status, data };
+    },
+    { token, method, path, body },
+  );
+}
+
+/** Enroll this user in a course. Throws unless the API answers 200 or 201. */
+export async function enrollUser(
+  page: Page,
+  user: FreshUser,
+  courseId = 'CS101-2024',
+): Promise<void> {
+  const { status, data } = await apiAsUser(page, user, 'POST', '/api/courses/enroll/', {
+    course_id: courseId,
+  });
+  if (status !== 200 && status !== 201) {
+    throw new Error(
+      `enrollUser: ${user.email} -> ${courseId} returned ${status}: ${JSON.stringify(data)}`,
+    );
+  }
+}
+
+/** Create a fresh student, log in, enroll in CS101-2024, and open `path`. */
+export async function navigateAsFreshStudent(
+  page: Page,
+  label: string,
+  path: string,
+): Promise<FreshUser> {
+  const user = freshStudent(label);
+  await injectUser(page, user);
+  await enrollUser(page, user);
+  await page.goto(path, { waitUntil: 'networkidle' });
+  return user;
+}
+
+/**
+ * Grant this user AI-processing consent. Fresh users have none, and spec-type
+ * submissions (LLM code generation) are blocked without it.
+ */
+export async function grantAiConsent(page: Page, user: FreshUser): Promise<void> {
+  const { status, data } = await apiAsUser(page, user, 'POST', '/api/users/me/consents/', {
+    consent_type: 'ai_processing',
+  });
+  if (status !== 200 && status !== 201) {
+    throw new Error(`grantAiConsent: ${user.email} returned ${status}: ${JSON.stringify(data)}`);
+  }
+}