diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c3a55a28..096722be 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -18,6 +18,15 @@ on: - 'playwright.config.ts' - 'package.json' - '.github/workflows/e2e.yml' + pull_request: + paths: + - 'purplex/**' + - 'tests/**' + - 'e2e/**' + - 'requirements.txt' + - 'playwright.config.ts' + - 'package.json' + - '.github/workflows/e2e.yml' workflow_dispatch: env: diff --git a/e2e/helpers/selectors.ts b/e2e/helpers/selectors.ts index 1ad3cfbd..91905974 100644 --- a/e2e/helpers/selectors.ts +++ b/e2e/helpers/selectors.ts @@ -240,3 +240,63 @@ export const common = { /** Dialog overlay (ConfirmDialog uses .dialog-overlay) */ dialogOverlay: '.dialog-overlay', } as const; + +// --------------------------------------------------------------------------- +// Probe panel (activities/inputs/shared/ProbePanel.vue) +// --------------------------------------------------------------------------- +export const probe = { + /** Panel root */ + panel: '.probe-panel', + /** Input for one parameter (Enter triggers a probe) */ + paramInput: (name: string) => `input#probe-${name}.param-input`, + /** Result slot ('?' until a result; .has-result / .is-cached) */ + result: '.probe-panel .fn-result', + /** Probe button ('Probe' | 'Probing' | 'Already probed', .is-duplicate) */ + button: 'button.probe-btn', + /** Remaining-probe badge (.status-explore | .status-available | .status-exhausted) */ + badge: 'span.probe-count-badge', + /** Status hint, rendered only when probes are exhausted */ + hint: '.probe-panel .probe-status-hint', + /** Oracle / request error line */ + error: '.probe-panel .probe-error', + /** Per-parameter field errors */ + fieldErrors: 'ul.probe-field-errors', + /** One parameter's field error */ + fieldError: (name: string) => `li#probe-err-${name}`, + /** 'History (N)' button, rendered only when history is non-empty */ + historyButton: '.probe-panel .history-btn', + /** History modal (teleported to body) */ + historyModal: '.modal-overlay .modal-content[role="dialog"]', + /** Rows in the history modal */ + historyRow: '.modal-content[role="dialog"] .modal-row', + /** History modal close button */ + close: '.modal-content[role="dialog"] .modal-close', +} as const; + +// --------------------------------------------------------------------------- +// Code submission feedback (activities/feedback/CodeSubmissionFeedback.vue) +// --------------------------------------------------------------------------- +export const codeFeedback = { + /** Summary bar with pass/fail/locked counts */ + summaryBar: '.feedback-container .test-summary-bar', + /** 'โœ“ N Passing' */ + passing: '.test-summary-bar .count-item.passing', + /** 'โœ— N Failing' */ + failing: '.test-summary-bar .count-item.failing', + /** '๐Ÿ”’ N more tests locked' (only when locked > 0) */ + locked: '.test-summary-bar .count-item.locked', + /** Failing test rows (.test-call, .expected, .actual, .actual.is-error) */ + failingItem: 'article.test-item.failing', + /** Passing test rows (inside the collapsed passing group) */ + passingItem: 'article.test-item.passing', + /** Note in the failing group explaining locked tests */ + lockedNote: 'p.locked-note', + /** Attempts dropdown trigger (.attempt-text 'n/total', .attempt-score-badge) */ + attemptTrigger: '.attempt-dropdown-trigger', + /** Attempts dropdown panel */ + attemptPanel: '.attempt-dropdown-panel[role="menu"]', + /** One attempt in the dropdown (.attempt-tests 'passed/total') */ + attemptItem: '.attempt-item-minimal', + /** Debug button on a test row */ + debugButton: '.debug-btn', +} as const; diff --git a/e2e/helpers/submission.ts b/e2e/helpers/submission.ts new file mode 100644 index 00000000..b75b25ff --- /dev/null +++ b/e2e/helpers/submission.ts @@ -0,0 +1,87 @@ +import { Page, expect } from '@playwright/test'; +import { setAceEditorValue } from './ace-editor'; +import { codeFeedback } from './selectors'; + +/** + * Code-submission helpers for probeable_code problems. + * + * In the E2E stack (CELERY_EAGER=true) POST /api/submit/ grades the whole + * submission synchronously in the Docker sandbox, then the SSE stream reports + * completion. ProblemSet.vue clears the previous feedback before it sends the + * request, so once the submit response is back, a visible summary bar can only + * belong to the new result. + */ + +const SUBMIT_URL = /\/api\/submit\/$/; + +/** + * Put `code` in the editor, submit it, and wait until the graded result's + * summary bar is visible. Throws if the submit request is not accepted. + */ +export async function submitCodeAndWait( + page: Page, + code: string, + timeout = 60_000, +): Promise { + await setAceEditorValue(page, '#codeEditor', code); + + const submitButton = page.locator('#submitButton'); + await expect(submitButton).toBeEnabled(); + + const responsePromise = page.waitForResponse( + (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + { timeout }, + ); + await submitButton.click(); + const response = await responsePromise; + if (!response.ok()) { + throw new Error( + `submitCodeAndWait: POST /api/submit/ returned ${response.status()}: ${await response.text()}`, + ); + } + + await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout }); +} + +/** + * Submit code that the API rejects as invalid (e.g. a syntax error) and return + * the message the user sees. + * + * The probeable_code handler rejects uncompilable code with 400 + * {"error": "Syntax error: (line N)"} before any submission exists. + * ProblemSet.vue's submit() catch shows it as a warning toast titled + * "Invalid Request" whose .notification-details is the server's error text. + * This asserts that toast and returns the details text. + */ +export async function submitSyntaxErrorAndReadMessage(page: Page, code: string): Promise { + await setAceEditorValue(page, '#codeEditor', code); + + const submitButton = page.locator('#submitButton'); + await expect(submitButton).toBeEnabled(); + + const responsePromise = page.waitForResponse( + (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + ); + await submitButton.click(); + const response = await responsePromise; + if (response.status() !== 400) { + throw new Error( + `submitSyntaxErrorAndReadMessage: expected 400 from POST /api/submit/, got ${response.status()}`, + ); + } + + const body = await response.json(); + if (typeof body?.error !== 'string' || body.error === '') { + throw new Error(`submitSyntaxErrorAndReadMessage: 400 without an error message: ${JSON.stringify(body)}`); + } + + // Match the toast to this response (an older toast may still be on screen). + // Toasts auto-dismiss after 4-5 s, so read the text as soon as it shows. + const toast = page + .locator('.notification-container .notification.warning') + .filter({ has: page.locator('.notification-details', { hasText: body.error }) }) + .last(); + await expect(toast).toBeVisible(); + await expect(toast.locator('.notification-message')).toHaveText('Invalid Request'); + return ((await toast.locator('.notification-details').textContent()) ?? '').trim(); +} diff --git a/e2e/helpers/users.ts b/e2e/helpers/users.ts new file mode 100644 index 00000000..9c1980df --- /dev/null +++ b/e2e/helpers/users.ts @@ -0,0 +1,159 @@ +import { Page } from '@playwright/test'; + +/** + * Fresh, per-test student users. + * + * Probe budgets and probe history live in Redis per (problem, user) with a + * 7-day TTL and no reset, and submissions accumulate per user. Any test that + * touches a budget, history or submissions must therefore run as a user no + * other test (or earlier run) has used. + * + * Mock Firebase accepts any email: the backend (users_app/mock_firebase.py) + * creates a student for an unknown email using the uid from the token, and + * the frontend MockFirebase restores its session from localStorage. + */ +export interface FreshUser { + uid: string; + email: string; + displayName: string; +} + +/** Create a unique user identity. Nothing is sent to the server. */ +export function freshStudent(label: string): FreshUser { + const safeLabel = label.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); + const id = `e2e-${safeLabel}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + return { + uid: `mock-uid-${id}`, + email: `${id}@test.local`, + displayName: `E2E ${label}`, + }; +} + +/** + * Inject this user into localStorage so the Vue app boots as them. + * Mirrors injectAuth in auth.ts: Vuex `user`, `mockFirebaseAuth`, and the + * cookie consent flag. Must be called before navigating to the target route. + */ +export async function injectUser(page: Page, user: FreshUser): Promise { + await page.goto('/', { waitUntil: 'commit' }); + + await page.evaluate( + ({ userJson, mockFirebaseAuthJson, consent }) => { + localStorage.setItem('user', userJson); + localStorage.setItem('mockFirebaseAuth', mockFirebaseAuthJson); + localStorage.setItem('purplex_cookie_consent', consent); + }, + { + userJson: JSON.stringify({ + uid: user.uid, + email: user.email, + displayName: user.displayName, + role: 'user', + isAdmin: false, + }), + mockFirebaseAuthJson: JSON.stringify({ + uid: user.uid, + email: user.email, + displayName: user.displayName, + }), + consent: 'accepted', + }, + ); +} + +/** + * Mint a mock token for this user. Same payload as the frontend's + * MockFirebase.createMockToken, which the backend decodes as + * MOCK..development. + */ +function mockTokenFor(user: FreshUser): string { + const now = Math.floor(Date.now() / 1000); + const payload = { + uid: user.uid, + email: user.email, + name: user.displayName, + email_verified: true, + iat: now, + exp: now + 3600, + }; + return `MOCK.${btoa(JSON.stringify(payload))}.development`; +} + +/** + * Call the API as this user from inside the page (same origin, through the + * Vite proxy). The page must already be on the app origin. + */ +export async function apiAsUser( + page: Page, + user: FreshUser, + method: string, + path: string, + body?: object, +): Promise<{ status: number; data: any }> { + const token = mockTokenFor(user); + return page.evaluate( + async ({ token, method, path, body }) => { + const res = await fetch(path, { + method, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + let data: unknown = null; + if (text) { + try { + data = JSON.parse(text); + } catch { + data = text; + } + } + return { status: res.status, data }; + }, + { token, method, path, body }, + ); +} + +/** Enroll this user in a course. Throws unless the API answers 200 or 201. */ +export async function enrollUser( + page: Page, + user: FreshUser, + courseId = 'CS101-2024', +): Promise { + const { status, data } = await apiAsUser(page, user, 'POST', '/api/courses/enroll/', { + course_id: courseId, + }); + if (status !== 200 && status !== 201) { + throw new Error( + `enrollUser: ${user.email} -> ${courseId} returned ${status}: ${JSON.stringify(data)}`, + ); + } +} + +/** Create a fresh student, log in, enroll in CS101-2024, and open `path`. */ +export async function navigateAsFreshStudent( + page: Page, + label: string, + path: string, +): Promise { + const user = freshStudent(label); + await injectUser(page, user); + await enrollUser(page, user); + await page.goto(path, { waitUntil: 'networkidle' }); + return user; +} + +/** + * Grant this user AI-processing consent. Fresh users have none, and spec-type + * submissions (LLM code generation) are blocked without it. + */ +export async function grantAiConsent(page: Page, user: FreshUser): Promise { + const { status, data } = await apiAsUser(page, user, 'POST', '/api/users/me/consents/', { + consent_type: 'ai_processing', + }); + if (status !== 200 && status !== 201) { + throw new Error(`grantAiConsent: ${user.email} returned ${status}: ${JSON.stringify(data)}`); + } +} diff --git a/e2e/student/probeable-budget.spec.ts b/e2e/student/probeable-budget.spec.ts new file mode 100644 index 00000000..875c554b --- /dev/null +++ b/e2e/student/probeable-budget.spec.ts @@ -0,0 +1,365 @@ +/** + * E2E Tests: probe budgets on probeable_code problems. + * + * Problem set e2e-probeable (seed_e2e_data._seed_probeable_workflows): + * p=0 e2e-probeable-block add(a, b), probe_mode "block", max_probes 3 + * p=1 e2e-probeable-cooldown add(a, b), probe_mode "cooldown", max_probes 2, + * refill 2 probes after 1 submission + * p=6 e2e-probeable-cooldown-2 add(a, b), probe_mode "cooldown", max_probes 1, + * refill 1 probe after 2 submissions + * + * Budgets live in Redis per (problem, user) with no reset, so every test runs + * as a fresh student (e2e/helpers/users.ts). + * + * UI facts this spec relies on (ProbePanel.vue / useProbeState.ts): + * - The remaining-count badge is rendered inside the Probe button only while + * the current inputs are not a previous probe. After a probe the inputs + * keep their values, so the button switches to "Already probed"; the spec + * enters new values before it reads the badge. + * - A duplicate input is answered from the client-side history: no request. + */ + +import { test, expect, Page, Response } from '@playwright/test'; +import { navigateAsFreshStudent, freshStudent, injectUser, enrollUser } from '../helpers/users'; +import { submitCodeAndWait } from '../helpers/submission'; +import { probe } from '../helpers/selectors'; + +const SET_URL = '/courses/CS101-2024/problem-set/e2e-probeable'; +const BLOCK_URL = `${SET_URL}?p=0`; +const COOLDOWN_URL = `${SET_URL}?p=1`; +const COOLDOWN_2_URL = `${SET_URL}?p=6`; +const ADD_SOLUTION = 'def add(a, b):\n return a + b'; + +function isProbePost(resp: Response, slug: string): boolean { + return ( + resp.request().method() === 'POST' && + new URL(resp.url()).pathname === `/api/problems/${slug}/probe/` + ); +} + +/** Wait until the probe panel and its add(a, b) inputs are on screen. */ +async function waitForProbePanel(page: Page): Promise { + await expect(page.locator(probe.panel)).toBeVisible({ timeout: 15_000 }); + await expect(page.locator(probe.paramInput('a'))).toBeVisible(); + await expect(page.locator(probe.paramInput('b'))).toBeVisible(); +} + +async function fillArgs(page: Page, a: string, b: string): Promise { + await page.locator(probe.paramInput('a')).fill(a); + await page.locator(probe.paramInput('b')).fill(b); +} + +/** + * Probe add(a, b) through the UI, assert the inline result, and return the + * probe POST response body. + */ +async function probeAdd( + page: Page, + slug: string, + a: string, + b: string, + expected: string, +): Promise { + await fillArgs(page, a, b); + const button = page.locator(probe.button); + await expect(button).toBeEnabled(); + const responsePromise = page.waitForResponse((resp) => isProbePost(resp, slug)); + await button.click(); + const response = await responsePromise; + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.success).toBe(true); + expect(body.result).toBe(Number(expected)); + await expect(page.locator(probe.result)).toHaveText(expected); + await expect(page.locator(probe.result)).toHaveClass(/has-result/); + return body; +} + +/** Assert the badge reads `ยท N left` with the given status class. */ +async function expectBadge(page: Page, remaining: number, statusClass: string): Promise { + const badge = page.locator(probe.badge); + await expect(badge).toHaveText(`ยท ${remaining} left`); + await expect(badge).toHaveClass(new RegExp(`\\b${statusClass}\\b`)); +} + +// --------------------------------------------------------------------------- +// A8: block mode +// --------------------------------------------------------------------------- +test.describe('Probe budget: block mode', () => { + test('A8 block mode spends the budget and blocks probing, and a fresh load shows the exhausted state', async ({ + page, + }) => { + const slug = 'e2e-probeable-block'; + await navigateAsFreshStudent(page, 'a8-block', BLOCK_URL); + await waitForProbePanel(page); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (req.method() === 'POST' && new URL(req.url()).pathname === `/api/problems/${slug}/probe/`) { + probePosts.push(req.postData() ?? ''); + } + }); + + // Fresh user: full budget, no hint, no history. + await expectBadge(page, 3, 'status-available'); + await expect(page.locator(probe.hint)).toHaveCount(0); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + await expect(page.locator(probe.result)).toHaveText('?'); + + // Probe 1: add(1, 2) -> 3 + const first = await probeAdd(page, slug, '1', '2', '3'); + expect(first.probe_status.remaining).toBe(2); + expect(first.probe_status.can_probe).toBe(true); + + // Probe 2: add(2, 2) -> 4. The badge is visible once the inputs are new. + await fillArgs(page, '2', '2'); + await expectBadge(page, 2, 'status-available'); + const second = await probeAdd(page, slug, '2', '2', '4'); + expect(second.probe_status.remaining).toBe(1); + + // A duplicate of (1, 2) shows the cached result and sends nothing. + const postsBeforeDuplicate = probePosts.length; + await fillArgs(page, '1', '2'); + const button = page.locator(probe.button); + await expect(button).toHaveText(/Already probed/); + await expect(button).toHaveClass(/is-duplicate/); + await expect(button).toBeDisabled(); + await expect(page.locator(probe.result)).toHaveText('3'); + await expect(page.locator(probe.result)).toHaveClass(/is-cached/); + + // Confirming the duplicate from the keyboard must not probe either. + await page.locator(probe.paramInput('b')).press('Enter'); + + // The duplicate was free: one probe is still left. + await fillArgs(page, '3', '3'); + await expectBadge(page, 1, 'status-available'); + expect(probePosts.length).toBe(postsBeforeDuplicate); + + // Probe 3: add(3, 3) -> 6 exhausts the budget. + const third = await probeAdd(page, slug, '3', '3', '6'); + expect(third.probe_status.can_probe).toBe(false); + expect(third.probe_status.remaining).toBe(0); + expect(third.probe_status.mode).toBe('block'); + + // Exhausted state without a reload. + await expect(page.locator(probe.hint)).toHaveText('No probes remaining.'); + await expect(page.locator(probe.button)).toBeDisabled(); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(page.locator(probe.paramInput('b'))).toBeDisabled(); + await expect(page.locator(probe.historyButton)).toHaveText('History (3)'); + + // Server-side state survives a fresh load (second tab, same student). + // Checked before the in-page badge so it is verified even if that fails. + const freshTab = await page.context().newPage(); + try { + await freshTab.goto(BLOCK_URL, { waitUntil: 'networkidle' }); + await waitForProbePanel(freshTab); + await expectBadge(freshTab, 0, 'status-exhausted'); + await expect(freshTab.locator(probe.hint)).toHaveText('No probes remaining.'); + await expect(freshTab.locator(probe.button)).toBeDisabled(); + await expect(freshTab.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(freshTab.locator(probe.paramInput('b'))).toBeDisabled(); + await expect(freshTab.locator(probe.result)).toHaveText('6'); + + // Earlier results stay readable from the history after exhaustion. + await freshTab.locator(probe.historyButton).click(); + await expect(freshTab.locator(probe.historyModal)).toBeVisible(); + const rows = freshTab.locator(probe.historyRow); + await expect(rows).toHaveCount(3); + await expect(rows.locator('.modal-call')).toHaveText(['add(3, 3)', 'add(2, 2)', 'add(1, 2)']); + await expect(rows.locator('.modal-result')).toHaveText(['6', '4', '3']); + await freshTab.locator(probe.close).click(); + await expect(freshTab.locator(probe.historyModal)).toHaveCount(0); + } finally { + await freshTab.close(); + } + + // Only the three real probes reached the server. + expect(probePosts.length).toBe(3); + + // The page the student probed from must also show the exhausted count. + await expectBadge(page, 0, 'status-exhausted'); + }); + + test('A8 a duplicate probe confirmed with Enter sends no request and spends no budget', async ({ + page, + }) => { + const slug = 'e2e-probeable-block'; + await navigateAsFreshStudent(page, 'a8-dup-enter', BLOCK_URL); + await waitForProbePanel(page); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (req.method() === 'POST' && new URL(req.url()).pathname === `/api/problems/${slug}/probe/`) { + probePosts.push(req.postData() ?? ''); + } + }); + + const first = await probeAdd(page, slug, '1', '2', '3'); + expect(first.probe_status.remaining).toBe(2); + expect(probePosts.length).toBe(1); + + // The inputs still hold (1, 2): the panel shows the cached result. + const button = page.locator(probe.button); + await expect(button).toHaveText(/Already probed/); + await expect(button).toBeDisabled(); + await expect(page.locator(probe.result)).toHaveClass(/is-cached/); + await expect(page.locator(probe.result)).toHaveText('3'); + + // Enter in a parameter field is the keyboard path to "Probe". On a + // duplicate it must do nothing, like the disabled button. + await page.locator(probe.paramInput('b')).press('Enter'); + + // Budget is server-side: a fresh load shows whether anything was spent. + await page.reload({ waitUntil: 'networkidle' }); + await waitForProbePanel(page); + await expectBadge(page, 2, 'status-available'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + expect(probePosts.length).toBe(1); + }); + + test('A8b block budgets are per user: a second student starts with the full budget', async ({ + page, + browser, + }) => { + const slug = 'e2e-probeable-block'; + await navigateAsFreshStudent(page, 'a8b-first', BLOCK_URL); + await waitForProbePanel(page); + await expectBadge(page, 3, 'status-available'); + const first = await probeAdd(page, slug, '5', '6', '11'); + expect(first.probe_status.remaining).toBe(2); + await fillArgs(page, '7', '8'); + await expectBadge(page, 2, 'status-available'); + + // A different student, in a separate browser context, on the same problem. + const otherContext = await browser.newContext(); + try { + const otherPage = await otherContext.newPage(); + const other = freshStudent('a8b-second'); + await injectUser(otherPage, other); + await enrollUser(otherPage, other); + await otherPage.goto(BLOCK_URL, { waitUntil: 'networkidle' }); + await waitForProbePanel(otherPage); + await expectBadge(otherPage, 3, 'status-available'); + await expect(otherPage.locator(probe.hint)).toHaveCount(0); + await expect(otherPage.locator(probe.historyButton)).toHaveCount(0); + await expect(otherPage.locator(probe.result)).toHaveText('?'); + } finally { + await otherContext.close(); + } + + // The first student's budget is unchanged by the second student. + await page.reload({ waitUntil: 'networkidle' }); + await waitForProbePanel(page); + await expectBadge(page, 2, 'status-available'); + }); +}); + +// --------------------------------------------------------------------------- +// A9: cooldown mode +// --------------------------------------------------------------------------- +test.describe('Probe budget: cooldown mode', () => { + test('A9 cooldown mode refills the budget after a submission', async ({ page }) => { + // Grading runs the submission in the Docker sandbox. + test.setTimeout(90_000); + const slug = 'e2e-probeable-cooldown'; + const refillHint = 'No probes remaining. Submit 1 more time to unlock more probes.'; + await navigateAsFreshStudent(page, 'a9-cooldown', COOLDOWN_URL); + await waitForProbePanel(page); + + await expectBadge(page, 2, 'status-available'); + await expect(page.locator(probe.hint)).toHaveCount(0); + + const first = await probeAdd(page, slug, '1', '2', '3'); + expect(first.probe_status.remaining).toBe(1); + + await fillArgs(page, '2', '2'); + await expectBadge(page, 1, 'status-available'); + const second = await probeAdd(page, slug, '2', '2', '4'); + expect(second.probe_status.can_probe).toBe(false); + expect(second.probe_status.remaining).toBe(0); + expect(second.probe_status.mode).toBe('cooldown'); + expect(second.probe_status.submissions_to_next_refill).toBe(1); + + await expect(page.locator(probe.hint)).toHaveText(refillHint); + await expect(page.locator(probe.button)).toBeDisabled(); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(page.locator(probe.paramInput('b'))).toBeDisabled(); + + // One submission earns the refill. + await submitCodeAndWait(page, ADD_SOLUTION); + + // A fresh load of the same problem (second tab, same student) shows the + // server-side refill. Checked first so it is documented even if the + // in-page state below is stale. + const freshTab = await page.context().newPage(); + try { + await freshTab.goto(COOLDOWN_URL, { waitUntil: 'networkidle' }); + await waitForProbePanel(freshTab); + await expectBadge(freshTab, 2, 'status-available'); + await expect(freshTab.locator(probe.hint)).toHaveCount(0); + await expect(freshTab.locator(probe.paramInput('a'))).toBeEnabled(); + await expect(freshTab.locator(probe.paramInput('b'))).toBeEnabled(); + } finally { + await freshTab.close(); + } + + // Without a reload, the page the student submitted from must show the + // refilled budget too. + await expect(page.locator(probe.hint)).toHaveCount(0); + await expect(page.locator(probe.paramInput('a'))).toBeEnabled(); + await expect(page.locator(probe.paramInput('b'))).toBeEnabled(); + await fillArgs(page, '3', '3'); + await expectBadge(page, 2, 'status-available'); + + // A further probe works and spends from the refilled budget. + const third = await probeAdd(page, slug, '3', '3', '6'); + expect(third.probe_status.can_probe).toBe(true); + expect(third.probe_status.remaining).toBe(1); + await fillArgs(page, '4', '4'); + await expectBadge(page, 1, 'status-available'); + }); + + test('A9b the refill hint counts down the submissions still needed', async ({ page }) => { + // Two graded submissions in the Docker sandbox. + test.setTimeout(120_000); + const slug = 'e2e-probeable-cooldown-2'; + await navigateAsFreshStudent(page, 'a9b-cooldown-2', COOLDOWN_2_URL); + await waitForProbePanel(page); + + await expectBadge(page, 1, 'status-available'); + await expect(page.locator(probe.hint)).toHaveCount(0); + + const only = await probeAdd(page, slug, '1', '2', '3'); + expect(only.probe_status.can_probe).toBe(false); + expect(only.probe_status.remaining).toBe(0); + expect(only.probe_status.submissions_to_next_refill).toBe(2); + await expect(page.locator(probe.hint)).toHaveText( + 'No probes remaining. Submit 2 more times to unlock more probes.', + ); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + + // First submission: one more is still needed, and the budget stays empty. + await submitCodeAndWait(page, ADD_SOLUTION); + await expect(page.locator(probe.hint)).toHaveText( + 'No probes remaining. Submit 1 more time to unlock more probes.', + ); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(page.locator(probe.button)).toBeDisabled(); + await expectBadge(page, 0, 'status-exhausted'); + + // Second submission earns the refill. + await submitCodeAndWait(page, ADD_SOLUTION); + await expect(page.locator(probe.hint)).toHaveCount(0); + await expect(page.locator(probe.paramInput('a'))).toBeEnabled(); + await fillArgs(page, '2', '2'); + await expectBadge(page, 1, 'status-available'); + + const refilled = await probeAdd(page, slug, '2', '2', '4'); + expect(refilled.probe_status.remaining).toBe(0); + expect(refilled.probe_status.submissions_to_next_refill).toBe(2); + await expect(page.locator(probe.hint)).toHaveText( + 'No probes remaining. Submit 2 more times to unlock more probes.', + ); + }); +}); diff --git a/e2e/student/probeable-feedback.spec.ts b/e2e/student/probeable-feedback.spec.ts new file mode 100644 index 00000000..0018608c --- /dev/null +++ b/e2e/student/probeable-feedback.spec.ts @@ -0,0 +1,378 @@ +/** + * E2E: probeable_code submission feedback (workflows B14-B23). + * + * Problem: e2e-probeable-locked (problem set e2e-probeable, p=2), seeded by + * seed_e2e_data. clamp(n) = max(0, min(n, 10)), explore probe mode, four + * ordered tests: clamp(5) -> 5, clamp(0) -> 0, clamp(-3) -> 0, clamp(20) -> 10. + * + * Truncation rule (#165): the rows shown are the passing tests up to the + * first failure plus that first failing test; every later test is locked. + * The summary bar's pass/fail counts still cover all four tests. + * + * With CELERY_EAGER=true, POST /api/submit/ grades synchronously in the Docker + * sandbox. Every test uses a fresh student, so attempts never leak between + * tests or runs. + */ + +import { test, expect, Page } from '@playwright/test'; +import { FreshUser, apiAsUser, navigateAsFreshStudent } from '../helpers/users'; +import { submitCodeAndWait, submitSyntaxErrorAndReadMessage } from '../helpers/submission'; +import { setAceEditorValue } from '../helpers/ace-editor'; +import { codeFeedback } from '../helpers/selectors'; + +const CLAMP_URL = '/courses/CS101-2024/problem-set/e2e-probeable?p=2'; +const GRADING_TIMEOUT = 60_000; + +const SOLUTIONS = { + correct: 'def clamp(n):\n return max(0, min(n, 10))', + failsFirst: 'def clamp(n):\n return 0 if n == 5 else max(0, min(n, 10))', + identity: 'def clamp(n):\n return n', + failsLast: 'def clamp(n):\n return max(0, n)', + divideByZero: 'def clamp(n):\n return n // 0', + syntaxError: 'def clamp(n):\n return (n', +} as const; + +// Each test owns a fresh user, so they can run concurrently across workers. +test.describe.configure({ mode: 'parallel' }); + +// Grading runs in the Docker sandbox and some tests submit twice. +test.beforeEach(() => { + test.setTimeout(150_000); +}); + +async function openClamp(page: Page, label: string): Promise { + const user = await navigateAsFreshStudent(page, label, CLAMP_URL); + await expect(page.locator('.probeable-code-input')).toBeVisible(); + await expect(page.locator('#codeEditor .ace_editor')).toBeVisible(); + return user; +} + +async function submitAndGrade(page: Page, code: string): Promise { + await submitCodeAndWait(page, code, GRADING_TIMEOUT); +} + +async function expectSummary( + page: Page, + passing: number, + failing: number, + lockedText: string | null, +): Promise { + await expect(page.locator(codeFeedback.passing)).toHaveText(`โœ“ ${passing} Passing`); + await expect(page.locator(codeFeedback.failing)).toHaveText(`โœ— ${failing} Failing`); + if (lockedText === null) { + await expect(page.locator(codeFeedback.locked)).toHaveCount(0); + await expect(page.locator(codeFeedback.lockedNote)).toHaveCount(0); + } else { + await expect(page.locator(codeFeedback.locked)).toHaveText(lockedText); + await expect(page.locator(codeFeedback.lockedNote)).toBeVisible(); + await expect(page.locator(codeFeedback.lockedNote)).toHaveText( + 'Remaining tests unlock once this test passes. Keep probing to discover more behavior.', + ); + } +} + +async function expectSingleFailingRow( + page: Page, + call: string, + expected: string, + actual: string, +): Promise { + const failing = page.locator(codeFeedback.failingItem); + await expect(failing).toHaveCount(1); + await expect(failing.locator('.test-call')).toHaveText(call); + await expect(failing.locator('.expected')).toHaveText(expected); + await expect(failing.locator('.actual')).toHaveText(actual); + await expect(failing.locator('.actual')).not.toHaveClass(/is-error/); +} + +/** Expand the collapsed passing group and check its rows, in order. */ +async function expectPassingRows(page: Page, rows: string[]): Promise { + const header = page.locator('summary.test-group-header.passing'); + await expect(header).toContainText(`Passing Tests (${rows.length})`); + const items = page.locator(codeFeedback.passingItem); + await expect(items).toHaveCount(rows.length); + await header.click(); + await expect(items.first()).toBeVisible(); + await expect(items.locator('.test-call')).toHaveText(rows); +} + +async function expectAttemptBadge(page: Page, attemptText: string, score: string): Promise { + const trigger = page.locator(codeFeedback.attemptTrigger); + await expect(trigger.locator('.attempt-text')).toHaveText(attemptText); + await expect(trigger.locator('.attempt-score-badge')).toHaveText(new RegExp(`^\\S\\s*${score}%\\s*$`)); +} + +// --------------------------------------------------------------------------- +// B14: input rejected before grading +// --------------------------------------------------------------------------- + +test.describe('B14 rejected submissions', () => { + test('B14a too-short code keeps the submit button disabled', async ({ page }) => { + await openClamp(page, 'b14a'); + + await setAceEditorValue(page, '#codeEditor', 'x'); + await expect(page.locator('#submitButton')).toBeDisabled(); + }); + + test('B14b syntax error is reported with its line and creates no attempt', async ({ page }) => { + const user = await openClamp(page, 'b14b'); + + const message = await submitSyntaxErrorAndReadMessage(page, SOLUTIONS.syntaxError); + expect(message).toMatch(/^Syntax error: .+ \(line 2\)$/); + + // Server state: the user has no stored attempt on this problem. No + // problem-set or course filter, so an attempt stored anywhere would show. + const history = await apiAsUser( + page, + user, + 'GET', + '/api/submissions/history/e2e-probeable-locked/', + ); + expect(history.status).toBe(200); + expect(history.data.total_attempts).toBe(0); + expect(history.data.submissions).toEqual([]); + + // The feedback panel stays in its empty state: no results, no attempts. + const container = page.locator('.feedback-container'); + await expect(container.locator('.empty-state')).toHaveText( + /Submit your solution to see test results/, + ); + await expect(page.locator('.feedback-header')).toHaveCount(0); + await expect(page.locator(codeFeedback.summaryBar)).toHaveCount(0); + await expect(page.locator(codeFeedback.attemptTrigger)).toHaveCount(0); + }); +}); + +// --------------------------------------------------------------------------- +// B15: grading in progress +// --------------------------------------------------------------------------- + +test.describe('B15 grading in progress', () => { + test('B15 grading in progress disables submit and shows the running-tests panel', async ({ page }) => { + await openClamp(page, 'b15'); + await setAceEditorValue(page, '#codeEditor', SOLUTIONS.correct); + + // Hold the submit request until the in-flight state has been checked, so + // the assertions do not race the sandbox. + let release: () => void = () => {}; + const released = new Promise((resolve) => { + release = resolve; + }); + await page.route('**/api/submit/', async (route) => { + await released; + await route.continue(); + }); + + const submitButton = page.locator('#submitButton'); + const panel = page.locator('.feedback-container .generating-feedback-panel'); + await expect(submitButton).toBeEnabled(); + await expect(panel).toHaveCount(0); + + const responsePromise = page.waitForResponse( + (resp) => + /\/api\/submit\/$/.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + { timeout: GRADING_TIMEOUT }, + ); + await submitButton.click(); + + await expect(submitButton).toBeDisabled(); + await expect(submitButton.locator('.bouncing-dots')).toBeVisible(); + await expect(panel).toBeVisible(); + await expect(panel.locator('.generating-message')).toHaveText('Running tests...'); + await expect(page.locator(codeFeedback.summaryBar)).toHaveCount(0); + + release(); + expect((await responsePromise).status()).toBe(202); + + await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout: GRADING_TIMEOUT }); + await expect(panel).toHaveCount(0); + await expect(submitButton.locator('.bouncing-dots')).toHaveCount(0); + await expect(submitButton).toBeEnabled(); + await expectSummary(page, 4, 0, null); + }); +}); + +// --------------------------------------------------------------------------- +// B16-B19: graded result for a single submission +// --------------------------------------------------------------------------- + +test.describe('B16-B19 graded feedback and test locking', () => { + test('B16 all tests pass: full counts, success toast, 100% badge', async ({ page }) => { + await openClamp(page, 'b16'); + + await submitAndGrade(page, SOLUTIONS.correct); + + // The toast auto-dismisses after 5 s, so check it first. + await expect( + page.locator('.notification-container .notification.success .notification-message'), + ).toHaveText('Problem 3: 4/4 tests passed'); + await expectSummary(page, 4, 0, null); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + await expectPassingRows(page, [ + 'clamp(5) โ†’ 5', + 'clamp(0) โ†’ 0', + 'clamp(-3) โ†’ 0', + 'clamp(20) โ†’ 10', + ]); + await expectAttemptBadge(page, '1/1', '100'); + }); + + test('B17 an early failure locks every later test, even passing ones', async ({ page }) => { + await openClamp(page, 'b17'); + + await submitAndGrade(page, SOLUTIONS.failsFirst); + + await expectSingleFailingRow(page, 'clamp(5)', '5', '0'); + // The first test failed, so no passing rows come before it. + await expect(page.locator(codeFeedback.passingItem)).toHaveCount(0); + await expect(page.locator('summary.test-group-header.passing')).toHaveCount(0); + // Counts still cover the three locked tests, which pass. + await expectSummary(page, 3, 1, '๐Ÿ”’ 3 more tests locked'); + await expectAttemptBadge(page, '1/1', '75'); + }); + + test('B18 one locked test uses the singular label', async ({ page }) => { + await openClamp(page, 'b18'); + + await submitAndGrade(page, SOLUTIONS.identity); + + await expectSingleFailingRow(page, 'clamp(-3)', '0', '-3'); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0']); + }); + + test('B17b nothing is locked when only the last test fails', async ({ page }) => { + await openClamp(page, 'b17b'); + + await submitAndGrade(page, SOLUTIONS.failsLast); + + await expectSingleFailingRow(page, 'clamp(20)', '10', '20'); + await expectSummary(page, 3, 1, null); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0', 'clamp(-3) โ†’ 0']); + }); + + test('B19 a runtime error shows as an error row and locks the rest', async ({ page }) => { + await openClamp(page, 'b19'); + + await submitAndGrade(page, SOLUTIONS.divideByZero); + + const failing = page.locator(codeFeedback.failingItem); + await expect(failing).toHaveCount(1); + await expect(failing.locator('.test-call')).toHaveText('clamp(5)'); + await expect(failing.locator('.expected')).toHaveText('5'); + await expect(failing.locator('.actual')).toHaveClass(/is-error/); + await expect(failing.locator('.actual')).toHaveText('integer division or modulo by zero'); + await expect(page.locator(codeFeedback.passingItem)).toHaveCount(0); + await expectSummary(page, 0, 4, '๐Ÿ”’ 3 more tests locked'); + }); +}); + +// --------------------------------------------------------------------------- +// B20-B22: multiple attempts and restoring feedback +// --------------------------------------------------------------------------- + +test.describe('B20-B22 attempts and returning later', () => { + test('B20 resubmitting shows the new attempt as 2/2 at 100%', async ({ page }) => { + await openClamp(page, 'b20'); + + await submitAndGrade(page, SOLUTIONS.identity); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectAttemptBadge(page, '1/1', '50'); + + await submitAndGrade(page, SOLUTIONS.correct); + await expectSummary(page, 4, 0, null); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + await expectAttemptBadge(page, '2/2', '100'); + }); + + test('B21 switching attempts restores each attempt\'s truncated feedback', async ({ page }) => { + await openClamp(page, 'b21'); + + await submitAndGrade(page, SOLUTIONS.identity); + await submitAndGrade(page, SOLUTIONS.correct); + await expectAttemptBadge(page, '2/2', '100'); + + const trigger = page.locator(codeFeedback.attemptTrigger); + const panel = page.locator(codeFeedback.attemptPanel); + const attempt = (n: number) => + panel.locator(codeFeedback.attemptItem).filter({ + has: page.locator('.attempt-num', { hasText: new RegExp(`^${n}$`) }), + }); + + await trigger.click(); + await expect(panel).toBeVisible(); + await expect(panel.locator(codeFeedback.attemptItem)).toHaveCount(2); + await expect(attempt(1).locator('.attempt-tests')).toHaveText('2/4'); + await expect(attempt(2).locator('.attempt-tests')).toHaveText('4/4'); + + await attempt(1).click(); + await expect(panel).toBeHidden(); + await expectAttemptBadge(page, '1/2', '50'); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectSingleFailingRow(page, 'clamp(-3)', '0', '-3'); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0']); + + await trigger.click(); + await expect(panel).toBeVisible(); + await attempt(2).click(); + await expect(panel).toBeHidden(); + await expectAttemptBadge(page, '2/2', '100'); + await expectSummary(page, 4, 0, null); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + }); + + test('B22 reloading the page restores the truncated feedback', async ({ page }) => { + await openClamp(page, 'b22'); + + await submitAndGrade(page, SOLUTIONS.identity); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + + const lastSubmission = page.waitForResponse( + (resp) => + /\/api\/last-submission\/e2e-probeable-locked\/$/.test(new URL(resp.url()).pathname) && + resp.request().method() === 'GET', + ); + await page.reload(); + expect((await lastSubmission).status()).toBe(200); + + await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout: 15_000 }); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectSingleFailingRow(page, 'clamp(-3)', '0', '-3'); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0']); + await expectAttemptBadge(page, '1/1', '50'); + }); +}); + +// --------------------------------------------------------------------------- +// B23: debugger on a failing row +// --------------------------------------------------------------------------- + +test.describe('B23 debugger', () => { + test('B23 the debug button opens and closes the step-by-step debugger', async ({ page }) => { + // Never load the external Python Tutor page; serve an empty document. + await page.route('https://pythontutor.com/**', (route) => + route.fulfill({ status: 200, contentType: 'text/html', body: '' }), + ); + + await openClamp(page, 'b23'); + await submitAndGrade(page, SOLUTIONS.identity); + + const failing = page.locator(codeFeedback.failingItem); + await expect(failing).toHaveCount(1); + + const modal = page.locator('.modal-overlay[role="dialog"]').filter({ + has: page.locator('#modal-title', { hasText: 'Step-by-Step Debugger' }), + }); + await expect(modal).toHaveCount(0); + + await failing.locator(codeFeedback.debugButton).click(); + await expect(modal).toBeVisible(); + const iframe = modal.locator('iframe.debugger-iframe'); + await expect(iframe).toHaveAttribute('src', /^https:\/\/pythontutor\.com\//); + const src = (await iframe.getAttribute('src')) ?? ''; + expect(decodeURIComponent(src.replace(/\+/g, ' '))).toContain('clamp(-3)'); + + await modal.locator('.close-button').click(); + await expect(modal).toHaveCount(0); + }); +}); diff --git a/e2e/student/probeable-probing.spec.ts b/e2e/student/probeable-probing.spec.ts new file mode 100644 index 00000000..cd0ed0fc --- /dev/null +++ b/e2e/student/probeable-probing.spec.ts @@ -0,0 +1,361 @@ +/** + * E2E Tests: probing the oracle on probeable problems (workflow group A). + * + * Problem set e2e-probeable (seed_e2e_data._seed_probeable_workflows): + * p=2 e2e-probeable-locked clamp(n: int) -> max(0, min(n, 10)), explore mode + * p=3 e2e-probeable-errors recip(x: int) -> 10 // x, block mode, 3 probes + * p=4 e2e-probeable-hidden recip(x) with the signature hidden, explore mode + * + * Probe budgets and history live in Redis per (problem, user) with no reset, + * so every test runs as a fresh student. + */ + +import { test, expect, Page, Request } from '@playwright/test'; +import { apiAsUser, navigateAsFreshStudent, FreshUser } from '../helpers/users'; +import { probe } from '../helpers/selectors'; + +const SET_PATH = '/courses/CS101-2024/problem-set/e2e-probeable'; +const PROBE_POST_PATH = /^\/api\/problems\/[^/]+\/probe\/$/; + +function isProbePost(req: Request): boolean { + return req.method() === 'POST' && PROBE_POST_PATH.test(new URL(req.url()).pathname); +} + +/** Open problem `index` of e2e-probeable as a fresh student and wait for probe status. */ +async function openAsFreshStudent( + page: Page, + label: string, + index: number, + functionName: string, +): Promise { + const user = await navigateAsFreshStudent(page, label, `${SET_PATH}?p=${index}`); + await expect(page.locator(probe.panel)).toBeVisible(); + await expect(page.locator('.probe-panel .fn-name')).toHaveText(functionName); + // The badge renders once /probe/status/ has loaded + await expect(page.locator(probe.badge)).toBeVisible(); + return user; +} + +/** + * Fill one parameter, trigger a probe (button click or Enter in the field), + * and return the probe response body. Fails unless the API answers 200. + */ +async function probeWith( + page: Page, + param: string, + value: string, + via: 'click' | 'enter' = 'click', +): Promise { + const input = page.locator(probe.paramInput(param)); + await input.fill(value); + const responsePromise = page.waitForResponse((resp) => isProbePost(resp.request())); + if (via === 'enter') { + await input.press('Enter'); + } else { + await page.locator(probe.button).click(); + } + const response = await responsePromise; + expect(response.status()).toBe(200); + return response.json(); +} + +test.describe('A Probing the oracle', () => { + test('A1 first probe shows the result, the explore badge and History (1)', async ({ page }) => { + await openAsFreshStudent(page, 'a1-first-probe', 2, 'clamp'); + + const input = page.locator(probe.paramInput('n')); + const result = page.locator(probe.result); + const button = page.locator(probe.button); + + // Fresh state: empty input, no result, no history, unlimited badge + await expect(input).toHaveValue(''); + await expect(input).toHaveAttribute('placeholder', 'int'); + await expect(result).toHaveText('?'); + await expect(result).not.toHaveClass(/has-result/); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); + // No value yet: nothing to probe + await expect(button).toBeDisabled(); + + const body = await probeWith(page, 'n', '7'); + expect(body.success).toBe(true); + expect(body.result).toBe(7); + expect(body.probe_status.mode).toBe('explore'); + + await expect(result).toHaveText('7'); + await expect(result).toHaveClass(/has-result/); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + await expect(page.locator(probe.error)).toHaveCount(0); + + // A new value brings the Probe button back with the unlimited badge + await input.fill('8'); + await expect(button).toBeEnabled(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + }); + + test('A2 repeat probe with a new input (Enter key) replaces the result', async ({ page }) => { + await openAsFreshStudent(page, 'a2-repeat-probe', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await expect(page.locator(probe.result)).toHaveText('7'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + // Enter in the field triggers the probe + const body = await probeWith(page, 'n', '20', 'enter'); + expect(body.success).toBe(true); + expect(body.result).toBe(10); + + await expect(page.locator(probe.result)).toHaveText('10'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + }); + + test('A3 duplicate probe returns the cached result without a request', async ({ page }) => { + await openAsFreshStudent(page, 'a3-duplicate-probe', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await probeWith(page, 'n', '20'); + await expect(page.locator(probe.result)).toHaveText('10'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts.push(req.postData() ?? ''); + } + }); + + const input = page.locator(probe.paramInput('n')); + const result = page.locator(probe.result); + const button = page.locator(probe.button); + + await input.fill('7'); + await expect(result).toHaveClass(/is-cached/); + await expect(result).toHaveText('7'); + await expect(button).toHaveClass(/is-duplicate/); + await expect(button).toContainText('Already probed'); + await expect(button).toBeDisabled(); + await expect(page.locator(probe.badge)).toHaveCount(0); + + // Enter in the field is the other way to probe; it must not re-send either + await input.press('Enter'); + + // A real probe afterwards orders the request stream: if the duplicate had + // been sent, it would be recorded before this one. + const body = await probeWith(page, 'n', '3'); + expect(body.result).toBe(3); + expect(probePosts).toEqual([JSON.stringify({ input: { n: 3 } })]); + await expect(page.locator(probe.historyButton)).toHaveText('History (3)'); + }); + + test('A4 invalid type shows a field error and blocks probing until fixed', async ({ page }) => { + await openAsFreshStudent(page, 'a4-invalid-type', 2, 'clamp'); + + let probePosts = 0; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts += 1; + } + }); + + const input = page.locator(probe.paramInput('n')); + const button = page.locator(probe.button); + const fieldError = page.locator(probe.fieldError('n')); + + await input.fill('abc'); + await expect(fieldError).toHaveText('n: Must be a whole number (e.g. 5)'); + await expect(input).toHaveClass(/has-error/); + await expect(input).toHaveAttribute('aria-invalid', 'true'); + await expect(input).toHaveAttribute('aria-describedby', 'probe-err-n'); + await expect(button).toBeDisabled(); + // Enter with an invalid value sends nothing + await input.press('Enter'); + + await input.fill('3'); + await expect(fieldError).toHaveCount(0); + await expect(input).not.toHaveClass(/has-error/); + await expect(button).toBeEnabled(); + + const body = await probeWith(page, 'n', '3'); + expect(body.result).toBe(3); + expect(probePosts).toBe(1); + await expect(page.locator(probe.result)).toHaveText('3'); + }); + + test('A5 history modal lists every probe, most recent first, and closes', async ({ page }) => { + await openAsFreshStudent(page, 'a5-history-modal', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await probeWith(page, 'n', '20'); + await probeWith(page, 'n', '-3'); + await expect(page.locator(probe.result)).toHaveText('0'); + + const historyButton = page.locator(probe.historyButton); + await expect(historyButton).toHaveText('History (3)'); + await historyButton.click(); + + const modal = page.locator(probe.historyModal); + await expect(modal).toBeVisible(); + await expect(modal.locator('#history-modal-title')).toHaveText('Probe History'); + + const rows = page.locator(probe.historyRow); + await expect(rows).toHaveCount(3); + await expect(rows.locator('.modal-index')).toHaveText(['#3', '#2', '#1']); + await expect(rows.locator('.modal-call')).toHaveText(['clamp(-3)', 'clamp(20)', 'clamp(7)']); + await expect(rows.locator('.modal-result')).toHaveText(['0', '10', '7']); + + await page.locator(probe.close).click(); + await expect(page.locator('.modal-overlay')).toHaveCount(0); + await expect(historyButton).toHaveText('History (3)'); + }); + + test('A6 reload restores the probe history from the server', async ({ page }) => { + const user = await openAsFreshStudent(page, 'a6-reload', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await probeWith(page, 'n', '20'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + + const server = await apiAsUser( + page, + user, + 'GET', + '/api/problems/e2e-probeable-locked/probe/history/', + ); + expect(server.status).toBe(200); + expect(server.data.history.map((e: any) => [e.input, e.output])).toEqual([ + [{ n: 20 }, 10], + [{ n: 7 }, 7], + ]); + + const historyLoaded = page.waitForResponse( + (resp) => + new URL(resp.url()).pathname === '/api/problems/e2e-probeable-locked/probe/history/' && + resp.status() === 200, + ); + await page.reload(); + await historyLoaded; + await expect(page.locator(probe.panel)).toBeVisible(); + + // Both probes restored from the server, the latest result shown, inputs cleared + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + await expect(page.locator(probe.result)).toHaveText('10'); + await expect(page.locator(probe.paramInput('n'))).toHaveValue(''); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + + await page.locator(probe.historyButton).click(); + const rows = page.locator(probe.historyRow); + await expect(rows.locator('.modal-call')).toHaveText(['clamp(20)', 'clamp(7)']); + await expect(rows.locator('.modal-result')).toHaveText(['10', '7']); + await page.locator(probe.close).click(); + + // Re-entering a value probed before the reload is still recognised + await page.locator(probe.paramInput('n')).fill('7'); + await expect(page.locator(probe.result)).toHaveClass(/is-cached/); + await expect(page.locator(probe.result)).toHaveText('7'); + await expect(page.locator(probe.button)).toContainText('Already probed'); + }); + + test('A7 explore mode never runs out of probes', async ({ page }) => { + await openAsFreshStudent(page, 'a7-explore', 2, 'clamp'); + + const badge = page.locator(probe.badge); + await expect(badge).toHaveText('ยท โˆž'); + await expect(badge).toHaveClass(/status-explore/); + + for (let n = 1; n <= 15; n += 1) { + const body = await probeWith(page, 'n', String(n)); + expect(body.success).toBe(true); + expect(body.result).toBe(Math.min(n, 10)); + expect(body.probe_status.mode).toBe('explore'); + expect(body.probe_status.can_probe).toBe(true); + await expect(page.locator(probe.result)).toHaveText(String(Math.min(n, 10))); + } + + await expect(page.locator(probe.historyButton)).toHaveText('History (15)'); + + const input = page.locator(probe.paramInput('n')); + await input.fill('16'); + await expect(input).toBeEnabled(); + await expect(page.locator(probe.button)).toBeEnabled(); + await expect(badge).toHaveText('ยท โˆž'); + await expect(badge).toHaveClass(/status-explore/); + await expect(page.locator(probe.hint)).toHaveCount(0); + + const body = await probeWith(page, 'n', '16'); + expect(body.success).toBe(true); + expect(body.result).toBe(10); + await expect(page.locator(probe.historyButton)).toHaveText('History (16)'); + }); + + test('A10 hidden signature hides types and sanitises oracle errors', async ({ page }) => { + const user = await openAsFreshStudent(page, 'a10-hidden', 4, 'recip'); + + const input = page.locator(probe.paramInput('x')); + await expect(page.locator('.probe-panel .param-name')).toHaveText(['x']); + await expect(input).toHaveAttribute('placeholder', 'value โ€” quote text: "abc"'); + await expect(input).toHaveAttribute('aria-label', 'Enter value for x'); + + // The types never reach the browser: the problem payload the page loads + // carries no signature, untyped parameters and no oracle source. + const { status, data } = await apiAsUser(page, user, 'GET', '/api/problem-sets/e2e-probeable/'); + expect(status).toBe(200); + const hidden = data.problems.find((p: { slug: string }) => p.slug === 'e2e-probeable-hidden'); + expect(hidden.function_signature).toBeNull(); + expect(hidden.probe_config.function_signature).toBeNull(); + expect(hidden.probe_config.parameters).toEqual([{ name: 'x', type: '' }]); + expect(hidden.reference_solution).toBeNull(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + + const failed = await probeWith(page, 'x', '0'); + expect(failed.success).toBe(false); + const error = page.locator(probe.error); + await expect(error).toHaveText('These inputs could not be evaluated. Try different values.'); + await expect(error).not.toContainText('division'); + await expect(page.locator(probe.result)).toHaveText('?'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + + const ok = await probeWith(page, 'x', '5'); + expect(ok.success).toBe(true); + expect(ok.result).toBe(2); + await expect(page.locator(probe.result)).toHaveText('2'); + await expect(error).toHaveCount(0); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + }); + + test('A11 visible-signature oracle error is shown and not charged', async ({ page }) => { + await openAsFreshStudent(page, 'a11-oracle-error', 3, 'recip'); + + const badge = page.locator(probe.badge); + await expect(badge).toHaveText('ยท 3 left'); + await expect(badge).toHaveClass(/status-available/); + + const failed = await probeWith(page, 'x', '0'); + expect(failed.success).toBe(false); + expect(failed.probe_status.remaining).toBe(3); + // Exactly the exception text, nothing else (no traceback, no type name). + // The oracle runs under the server's interpreter (oracle_sandbox.py uses + // sys.executable): Python <= 3.13 says "integer division or modulo by + // zero" for `10 // 0`, 3.14 says "division by zero". + await expect(page.locator(probe.error)).toHaveText( + /^(integer division or modulo|division) by zero$/, + ); + await expect(badge).toHaveText('ยท 3 left'); + await expect(page.locator(probe.result)).toHaveText('?'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + + const ok = await probeWith(page, 'x', '5'); + expect(ok.success).toBe(true); + expect(ok.result).toBe(2); + expect(ok.probe_status.remaining).toBe(2); + await expect(page.locator(probe.result)).toHaveText('2'); + await expect(page.locator(probe.error)).toHaveCount(0); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + // The input still holds the probed value (duplicate state hides the badge); + // a new value shows the charged budget + await page.locator(probe.paramInput('x')).fill('6'); + await expect(badge).toHaveText('ยท 2 left'); + await expect(badge).toHaveClass(/status-available/); + }); +}); diff --git a/e2e/student/probeable-spec.spec.ts b/e2e/student/probeable-spec.spec.ts new file mode 100644 index 00000000..63d38e20 --- /dev/null +++ b/e2e/student/probeable-spec.spec.ts @@ -0,0 +1,250 @@ +/** + * E2E: Probeable Spec workflows (C24a-C24d). + * + * Problem: e2e-probeable-spec-open, index 5 of the e2e-probeable set. + * def square(x: int) -> int, reference `return x * x`, probe_mode explore. + * Test cases: square(3) = 9 (sample), square(-2) = 4 (hidden). + * segmentation_config {"enabled": False}, so a submission whose tests all + * pass is graded 'complete' (ProbeableSpecHandler: segmentation disabled -> + * complete) with no abstraction step. + * + * Grading in the E2E stack: CELERY_EAGER runs the EiPL pipeline inside + * POST /api/submit/, and USE_MOCK_OPENAI returns the reference solution as + * all 5 LLM variations, so every variation passes both tests. + * + * Fresh users have no AI processing consent. probeable_spec submissions are + * not consent-gated yet (#161); submit tests grant consent through the API up + * front so they stay valid once that gate exists. + * + * Every test runs as its own fresh student: probe budgets/history live in + * Redis per (problem, user) with no reset, and submissions accumulate. + */ + +import { test, expect, Page } from '@playwright/test'; +import { setAceEditorValue } from '../helpers/ace-editor'; +import { probe, feedback, problemSet } from '../helpers/selectors'; +import { + FreshUser, + apiAsUser, + enrollUser, + freshStudent, + grantAiConsent, + injectUser, +} from '../helpers/users'; + +const PROBLEM_SET = 'e2e-probeable'; +const PROBLEM_SLUG = 'e2e-probeable-spec-open'; +const SPEC_PATH = `/courses/CS101-2024/problem-set/${PROBLEM_SET}?p=5`; +const GRADE_TIMEOUT = 90_000; + +/** 60 characters of plain-language description. */ +const EXPLANATION = 'Returns the given whole number multiplied by itself squared.'; + +const SUBMIT_URL = /\/api\/submit\/$/; +const PROBE_URL = new RegExp(`/api/problems/${PROBLEM_SLUG}/probe/$`); + +/** + * Create a fresh student, enroll, optionally grant AI consent, and open the + * spec problem. Consent must be granted before navigation so the first submit + * is not intercepted by the consent flow. + */ +async function openSpecProblem( + page: Page, + label: string, + { aiConsent }: { aiConsent: boolean }, +): Promise { + const user = freshStudent(label); + await injectUser(page, user); + await enrollUser(page, user); + if (aiConsent) { + await grantAiConsent(page, user); + } + await page.goto(SPEC_PATH, { waitUntil: 'networkidle' }); + await expect(page.locator('.probeable-spec-input')).toBeVisible({ timeout: 15_000 }); + return user; +} + +/** Probe square(x=value) and wait for the oracle response. */ +async function probeX(page: Page, value: string): Promise { + await page.locator(probe.paramInput('x')).fill(value); + const button = page.locator(probe.button); + await expect(button).toBeEnabled(); + const responsePromise = page.waitForResponse( + (resp) => PROBE_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + ); + await button.click(); + const response = await responsePromise; + expect(response.status()).toBe(200); +} + +/** Put the explanation in #promptEditor, submit, and require a 2xx. */ +async function submitExplanation(page: Page, text: string): Promise { + await setAceEditorValue(page, problemSet.promptEditor, text); + const submitButton = page.locator(problemSet.submitButton); + await expect(submitButton).toBeEnabled(); + + const responsePromise = page.waitForResponse( + (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + { timeout: GRADE_TIMEOUT }, + ); + await submitButton.click(); + const response = await responsePromise; + expect(response.ok(), `POST /api/submit/ returned ${response.status()}`).toBe(true); +} + +test.describe('C24 Probeable Spec', () => { + test('C24a probe on a spec problem shows oracle results and history', async ({ page }) => { + await openSpecProblem(page, 'c24a', { aiConsent: false }); + + const panel = page.locator('.probeable-spec-input ' + probe.panel); + await expect(panel).toBeVisible(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); + await expect(page.locator(probe.result)).toHaveText('?'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + + await probeX(page, '3'); + await expect(page.locator(probe.result)).toHaveText('9'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + await probeX(page, '-2'); + await expect(page.locator(probe.result)).toHaveText('4'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + // The input still holds the last probed value, so the button reports it + // as already probed (no badge while duplicate). + await expect(page.locator(probe.button)).toHaveText(/Already probed/); + await expect(page.locator(probe.button)).toHaveClass(/is-duplicate/); + await expect(page.locator(probe.button)).toBeDisabled(); + // A new value re-enables probing; explore mode keeps the badge unlimited. + await page.locator(probe.paramInput('x')).fill('7'); + await expect(page.locator(probe.button)).toBeEnabled(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); + + await page.locator(probe.historyButton).click(); + await expect(page.locator(probe.historyModal)).toBeVisible(); + const rows = page.locator(probe.historyRow); + await expect(rows).toHaveCount(2); + // Newest first. + await expect(rows.nth(0).locator('.modal-index')).toHaveText('#2'); + await expect(rows.nth(0).locator('.modal-call')).toHaveText('square(-2)'); + await expect(rows.nth(0).locator('.modal-result')).toHaveText('4'); + await expect(rows.nth(1).locator('.modal-index')).toHaveText('#1'); + await expect(rows.nth(1).locator('.modal-call')).toHaveText('square(3)'); + await expect(rows.nth(1).locator('.modal-result')).toHaveText('9'); + await page.locator(probe.close).click(); + await expect(page.locator(probe.historyModal)).toHaveCount(0); + }); + + test('C24b explanation submit grades 100% with both tests shown', async ({ page }) => { + // One mock-LLM grading: 5 variations x 2 tests in the Docker sandbox. + test.setTimeout(GRADE_TIMEOUT + 30_000); + expect(EXPLANATION.length).toBe(60); + await openSpecProblem(page, 'c24b', { aiConsent: true }); + + await submitExplanation(page, EXPLANATION); + + // All 5 mock variations are the reference solution and segmentation is + // disabled for this problem, so the grade is complete at 100%. + await expect(page.locator('.notification-container .notification.success')).toContainText( + 'Problem 6: Variations: 5/5 passing', + { timeout: GRADE_TIMEOUT }, + ); + + const container = page.locator(feedback.container); + await expect(container.locator(feedback.header)).toBeVisible({ timeout: GRADE_TIMEOUT }); + await expect(container.locator('.attempt-text')).toHaveText('1/1'); + await expect(container.locator(feedback.scoreBadge)).toHaveText('100%'); + await expect(container.locator('.explanation-text')).toHaveText(EXPLANATION); + + const correctnessCard = container.locator(feedback.metricCard).first(); + await expect(correctnessCard.locator('.status-metric')).toHaveText('5/5 pass'); + await expect(correctnessCard.locator('.status-label')).toHaveText('Clear'); + + // Test rows live in the correctness modal. probeable_spec is outside the + // #165 truncation rule, so every variation lists both tests, the hidden + // one included. + await correctnessCard.click(); + const modal = page.locator('.modal-content.correctness-modal'); + await expect(modal).toBeVisible(); + await expect(modal.locator('.badge-level')).toHaveText('5/5 Versions Pass'); + await expect(modal.locator('.version-pill')).toHaveCount(5); + + for (let i = 0; i < 5; i++) { + await modal.locator(`#version-tab-${i}`).click(); + const panel = modal.locator(`#version-panel-${i}`); + await expect(panel).toBeVisible(); + await expect(panel.locator('.details-tests .section-meta')).toHaveText('2/2'); + const tests = panel.locator('.tests-list .test-item'); + await expect(tests).toHaveCount(2); + await expect(panel.locator('.tests-list .test-item.passing')).toHaveCount(2); + await expect(tests.nth(0).locator('.test-call')).toHaveText('square(3)'); + await expect(tests.nth(1).locator('.test-call')).toHaveText('square(-2)'); + } + }); + + test('C24c description length bounds gate the submit button', async ({ page }) => { + await openSpecProblem(page, 'c24c', { aiConsent: false }); + const submitButton = page.locator(problemSet.submitButton); + + // Empty on load. + await expect(submitButton).toBeDisabled(); + + await setAceEditorValue(page, problemSet.promptEditor, 'short'); + await expect(submitButton).toBeDisabled(); + + // Minimum is 10 characters after trimming. + await setAceEditorValue(page, problemSet.promptEditor, ' 123456789 '); + await expect(submitButton).toBeDisabled(); + await setAceEditorValue(page, problemSet.promptEditor, '1234567890'); + await expect(submitButton).toBeEnabled(); + + // Maximum is 1000 characters. + await setAceEditorValue(page, problemSet.promptEditor, 'a'.repeat(1000)); + await expect(submitButton).toBeEnabled(); + await setAceEditorValue(page, problemSet.promptEditor, 'a'.repeat(1001)); + await expect(submitButton).toBeDisabled(); + }); + + test('C24d resubmitting shows 2 attempts', async ({ page }) => { + // Two mock-LLM gradings, each 5 variations x 2 tests in the Docker sandbox. + test.setTimeout(2 * GRADE_TIMEOUT + 30_000); + const user = await openSpecProblem(page, 'c24d', { aiConsent: true }); + const container = page.locator(feedback.container); + + await submitExplanation(page, EXPLANATION); + await expect(container.locator('.attempt-text')).toHaveText('1/1', { timeout: GRADE_TIMEOUT }); + + await submitExplanation(page, 'It multiplies the integer argument by itself and returns it.'); + await expect(container.locator('.attempt-text')).toHaveText('2/2', { timeout: GRADE_TIMEOUT }); + await expect(container.locator(feedback.scoreBadge)).toHaveText('100%'); + + await container.locator(feedback.attemptSelector).click(); + const panel = container.locator('.attempt-dropdown-panel[role="menu"]'); + await expect(panel).toBeVisible(); + const items = panel.locator('.attempt-item-minimal'); + await expect(items).toHaveCount(2); + await expect(items.locator('.attempt-num')).toHaveText(['2', '1']); + await expect(items.locator('.attempt-score-minimal')).toHaveText(['100%', '100%']); + + const { status, data } = await apiAsUser( + page, + user, + 'GET', + `/api/submissions/history/${PROBLEM_SLUG}/?problem_set_slug=${PROBLEM_SET}&course_id=CS101-2024`, + ); + expect(status).toBe(200); + expect(data.submissions).toHaveLength(2); + for (const submission of data.submissions) { + expect(submission.score).toBe(100); + expect(submission.completion_status).toBe('complete'); + expect(submission.variations_count).toBe(5); + // History counts test executions across all 5 variations: 5 x 2 tests. + expect(submission.tests_passed).toBe(10); + expect(submission.total_tests).toBe(10); + } + + // The dropdown shows each attempt's tests_passed/total_tests from history. + await expect(items.locator('.attempt-tests')).toHaveText(['10/10', '10/10']); + }); +}); diff --git a/e2e/student/submission-probeable.spec.ts b/e2e/student/submission-probeable.spec.ts index aeca7af4..ce3df4e4 100644 --- a/e2e/student/submission-probeable.spec.ts +++ b/e2e/student/submission-probeable.spec.ts @@ -6,63 +6,61 @@ * * Probeable Code has two phases: * 1. Probe phase: students query the oracle with inputs to discover behavior + * (POST /api/problems//probe/ answers synchronously) * 2. Code phase: students write implementation code and submit * - * Both probe execution and final submission are ASYNCHRONOUS. + * The E2E stack runs Celery eagerly (CELERY_EAGER=true), so POST /api/submit/ + * grades the submission synchronously in the local Docker sandbox + * (DOCKER_POOL_ENABLED=false) and the SSE stream reports completion at once. + * + * Probe history lives in Redis per (problem, user) with no reset, so tests + * that probe or submit run as a fresh student (e2e/helpers/users.ts). * * Seeded data: * - CS101-2024 course with e2e-code problem set (soft past deadline) - * - e2e-probeable-code-1: mystery(lst) -- actually sorted(lst) + * - e2e-probeable-code-1: mystery(lst) -- actually sorted(lst) (e2e-code, p=1) * function_signature: "def mystery(lst: list[int]) -> list[int]" - * probe_mode: "explore", max_probes: 10 + * probe_mode: "explore", one test case: mystery([3, 1, 2]) == [1, 2, 3] * - e2e-probeable-code-typed: double(n) -- n * 2 (e2e-code, p=2) * function_signature: "def double(n: int) -> int" * used to check client-side probe input type validation (#135) * - e2e-refute-1: f(x) = x * 2, claim "always returns positive" * (in e2e-mixed which has hard past deadline -- locked) * - * Prerequisites: - * 1. Django dev server on :8000 with USE_MOCK_FIREBASE=true - * 2. Celery worker running (with Docker-in-Docker for code execution) - * 3. Vite dev server on :5173 - * 4. python manage.py create_test_users && python manage.py seed_e2e_data + * Prerequisites (same as .github/workflows/e2e.yml): + * 1. Django dev server on :8000 with USE_MOCK_FIREBASE=true, + * CELERY_EAGER=true, DOCKER_POOL_ENABLED=false, and the + * purplex/python-sandbox image available to the local Docker daemon + * 2. Vite dev server on :5173 + * 3. python manage.py create_test_users && python manage.py seed_e2e_data */ import { test, expect, Page } from '@playwright/test'; import { navigateAs } from '../helpers/navigation'; import { apiAs } from '../helpers/api'; +import { apiAsUser, enrollUser, freshStudent, injectUser, navigateAsFreshStudent } from '../helpers/users'; +import { submitCodeAndWait } from '../helpers/submission'; +import { codeFeedback, probe } from '../helpers/selectors'; -const ASYNC_TIMEOUT = 45000; +const PROBEABLE_PATH = '/courses/CS101-2024/problem-set/e2e-code?p=1'; +const TYPED_PROBEABLE_PATH = '/courses/CS101-2024/problem-set/e2e-code?p=2'; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- -/** Navigate to e2e-code and select e2e-probeable-code-1. */ -async function goToProbeableProblem(page: Page) { - await navigateAs( - page, - 'student', - '/courses/CS101-2024/problem-set/e2e-code?p=1', - ); - // Wait for the probeable code input to appear +/** Wait for the probeable code input to render. */ +async function waitForProbeableInput(page: Page) { await page.locator('.probeable-code-input, .probe-panel, #codeEditor').first().waitFor({ state: 'visible', timeout: 15000, }); } -/** Navigate to e2e-code and select e2e-probeable-code-typed. */ -async function goToTypedProbeableProblem(page: Page) { - await navigateAs( - page, - 'student', - '/courses/CS101-2024/problem-set/e2e-code?p=2', - ); - await page.locator('.probeable-code-input, .probe-panel, #codeEditor').first().waitFor({ - state: 'visible', - timeout: 15000, - }); +/** Navigate to e2e-code and select e2e-probeable-code-1 as the seeded student. */ +async function goToProbeableProblem(page: Page) { + await navigateAs(page, 'student', PROBEABLE_PATH); + await waitForProbeableInput(page); } /** Type code into the Probeable Code Ace editor. */ @@ -92,26 +90,23 @@ test.describe('Probeable Code Submission', () => { }); test('probe oracle via API returns sorted result', async ({ page }) => { - // Direct API test for probe functionality -- more reliable than UI probe - await navigateAs(page, 'student', '/home'); + const user = freshStudent('probe-api'); + await injectUser(page, user); + await enrollUser(page, user); - const { status, data } = await apiAs( + const { status, data } = await apiAsUser( page, - 'student', + user, 'POST', '/api/problems/e2e-probeable-code-1/probe/', - { input: { lst: [3, 1, 2] } }, + { input: { lst: [3, 1, 2] }, course_id: 'CS101-2024' }, ); - if (status === 200) { - // Probe succeeded -- oracle should return sorted list - expect(data.success).toBeTruthy(); - expect(data.result).toEqual([1, 2, 3]); - } else { - // 401/403 means mock auth is not reaching the API, or probe requires - // the student to be enrolled and have the right problem set context - expect([401, 403, 400]).toContain(status); - } + expect(status).toBe(200); + expect(data.success).toBe(true); + expect(data.error).toBeNull(); + expect(data.result).toEqual([1, 2, 3]); + expect(data.probe_status.mode).toBe('explore'); }); test('code editor is available for writing implementation', async ({ page }) => { @@ -126,25 +121,24 @@ test.describe('Probeable Code Submission', () => { await expect(submitBtn).toBeVisible(); }); - test('submitting code solution triggers async evaluation', async ({ page }) => { - await goToProbeableProblem(page); - - // Write a solution (the mystery function is sorted()) - const solution = 'def mystery(lst):\n return sorted(lst)'; - await typeInCodeEditor(page, solution); - - const submitBtn = page.locator('#submitButton'); - await expect(submitBtn).toBeEnabled({ timeout: 5000 }); - await submitBtn.click(); - - // Should enter loading state - await expect(submitBtn).toBeDisabled({ timeout: 5000 }); - - // Wait for results (async via SSE) - const feedbackOrLoading = page.locator( - '.feedback-container .feedback-header, .feedback-container .generating-feedback-panel', - ).first(); - await feedbackOrLoading.waitFor({ state: 'visible', timeout: ASYNC_TIMEOUT }); + test('submitting a correct solution grades it and shows the passing feedback', async ({ page }) => { + // submitCodeAndWait waits up to 60s for grading in the Docker sandbox. + test.setTimeout(90_000); + await navigateAsFreshStudent(page, 'probeable-submit', PROBEABLE_PATH); + await waitForProbeableInput(page); + + // The mystery function is sorted(); its single test case must pass + await submitCodeAndWait(page, 'def mystery(lst):\n return sorted(lst)'); + + await expect(page.locator(codeFeedback.passing)).toHaveText('โœ“ 1 Passing'); + await expect(page.locator(codeFeedback.failing)).toHaveText('โœ— 0 Failing'); + await expect(page.locator(codeFeedback.locked)).toHaveCount(0); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + await expect(page.locator(`${codeFeedback.attemptTrigger} .attempt-text`)).toHaveText('1/1'); + const scoreBadge = page.locator(`${codeFeedback.attemptTrigger} .attempt-score-badge`); + await expect(scoreBadge).toHaveText('โœ“ 100%'); + await expect(scoreBadge).toHaveAttribute('aria-label', 'Score: 100 percent'); + await expect(scoreBadge).toHaveClass(/score-perfect/); }); test('submit button requires minimum code length', async ({ page }) => { @@ -159,40 +153,45 @@ test.describe('Probeable Code Submission', () => { }); test('probe history populates after executing probes via UI', async ({ page }) => { - await goToProbeableProblem(page); + await navigateAsFreshStudent(page, 'probe-history-ui', PROBEABLE_PATH); + await expect(page.locator(probe.panel)).toBeVisible(); + // The badge renders once /probe/status/ has loaded + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); - // The probe panel has input fields for function parameters // e2e-probeable-code-1 has signature: mystery(lst: list[int]) -> list[int] - const paramInput = page.locator('.param-input').first(); - if (!(await paramInput.isVisible().catch(() => false))) { - // Probe panel may not have rendered param inputs -- skip gracefully - test.skip(); - return; - } - - await paramInput.fill('[3, 1, 2]'); - - // Find and click the probe/execute button - const probeBtn = page.locator('button').filter({ hasText: /probe|execute|run/i }).first(); - if (!(await probeBtn.isVisible().catch(() => false))) { - test.skip(); - return; - } - - // Probe button may be disabled if Docker execution is unavailable - const isEnabled = await probeBtn.isEnabled().catch(() => false); - if (!isEnabled) { - test.skip(); - return; + const lstInput = page.locator(probe.paramInput('lst')); + const probeBtn = page.locator(probe.button); + const probes: Array<[string, number[]]> = [ + ['[3, 1, 2]', [1, 2, 3]], + ['[9, 4]', [4, 9]], + ]; + + for (const [value, expected] of probes) { + await lstInput.fill(value); + await expect(probeBtn).toBeEnabled(); + const responsePromise = page.waitForResponse( + (resp) => + resp.request().method() === 'POST' && + new URL(resp.url()).pathname === '/api/problems/e2e-probeable-code-1/probe/', + ); + await probeBtn.click(); + const response = await responsePromise; + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.success).toBe(true); + expect(body.result).toEqual(expected); + await expect(page.locator(probe.result)).toHaveText(JSON.stringify(expected)); } - await probeBtn.click(); + const historyBtn = page.locator(probe.historyButton); + await expect(historyBtn).toHaveText('History (2)'); + await historyBtn.click(); - // Wait for a probe history entry to appear - const historyItem = page.locator('.probe-history-item, .probe-result').first(); - await historyItem.waitFor({ state: 'visible', timeout: 15000 }).catch(() => { - // Probe may have failed -- acceptable in test environments - }); + const rows = page.locator(probe.historyRow); + await expect(rows).toHaveCount(2); + await expect(rows.locator('.modal-call')).toHaveText(['mystery([9,4])', 'mystery([3,1,2])']); + await expect(rows.locator('.modal-result')).toHaveText(['[4,9]', '[1,2,3]']); }); }); @@ -202,25 +201,22 @@ test.describe('Probeable Code Submission', () => { test.describe('Probe Input Type Validation', () => { test('non-integer input for an int parameter shows an error and disables Probe', async ({ page }) => { - await goToTypedProbeableProblem(page); + // Fresh student: a probe history containing the typed value would put the + // button in its 'Already probed' state + await navigateAsFreshStudent(page, 'probe-typed', TYPED_PROBEABLE_PATH); + await waitForProbeableInput(page); // e2e-probeable-code-typed has signature: double(n: int) -> int - const nInput = page.locator('#probe-n'); - if (!(await nInput.isVisible().catch(() => false))) { - test.skip(); - return; - } - if (!(await nInput.isEnabled().catch(() => false))) { - // Probing may be unavailable (e.g. exhausted budget) -- skip gracefully - test.skip(); - return; - } + const nInput = page.locator(probe.paramInput('n')); + await expect(nInput).toBeVisible(); + await expect(nInput).toBeEnabled(); - const fieldErrors = page.locator('.probe-field-errors'); - const probeBtn = page.locator('.probe-btn'); + const fieldErrors = page.locator(probe.fieldErrors); + const probeBtn = page.locator(probe.button); await nInput.fill('abc'); await expect(fieldErrors).toBeVisible(); + await expect(page.locator(probe.fieldError('n'))).toHaveText('n: Must be a whole number (e.g. 5)'); await expect(nInput).toHaveClass(/has-error/); await expect(probeBtn).toBeDisabled(); @@ -243,19 +239,14 @@ test.describe('Refute Problem (Deadline Enforcement)', () => { '/courses/CS101-2024/problem-set/e2e-mixed?p=6', ); - // Wait for the page to load - await page.waitForLoadState('networkidle'); - - // Should see the locked deadline banner or lock icon - const lockedBanner = page.locator('.deadline-locked'); - const lockText = page.getByText(/Submissions Closed|submissions closed/i).first(); - - const isLocked = - (await lockedBanner.isVisible().catch(() => false)) || - (await lockText.isVisible().catch(() => false)); - - // The hard past deadline should block the UI - expect(isLocked).toBeTruthy(); + // The hard past deadline shows the locked banner. + const banner = page.locator('.deadline-banner.deadline-locked'); + await expect(banner).toBeVisible(); + await expect(banner.locator('.deadline-icon')).toHaveText('๐Ÿ”’'); + await expect(banner.locator('.deadline-text strong')).toHaveText('Submissions Closed'); + await expect(banner.locator('.deadline-text')).toHaveText( + /^Submissions Closed โ€” This problem set closed on \S.*$/, + ); }); test('API rejects submission on hard-deadline-locked problem set', async ({ page }) => { @@ -268,12 +259,9 @@ test.describe('Refute Problem (Deadline Enforcement)', () => { raw_input: JSON.stringify({ x: -5 }), }); - // Should be rejected due to hard past deadline - if (status === 200) { - // Some implementations return 200 with an error in the body - expect(data).toBeTruthy(); - } else { - expect([400, 403, 422]).toContain(status); - } + // SubmissionValidationService._check_deadline rejects it before grading. + expect(status).toBe(400); + expect(data.error).toMatch(/^Submission deadline has passed\. This problem set closed on .+\.$/); + expect(data.code).toBe('validation_error'); }); }); diff --git a/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue b/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue index 969c5694..bf01991a 100644 --- a/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue +++ b/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue @@ -155,7 +155,7 @@ const { formatFunctionCall, formatOutput, updateProbeInput, -} = useProbeState(() => props.problem) +} = useProbeState(() => props.problem, { isSubmitting: () => props.disabled }) const { t } = useI18n() diff --git a/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue b/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue index 4d9663b1..b2503e31 100644 --- a/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue +++ b/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue @@ -157,7 +157,7 @@ const { formatFunctionCall, formatOutput, updateProbeInput, -} = useProbeState(() => props.problem) +} = useProbeState(() => props.problem, { isSubmitting: () => props.disabled }) const { t } = useI18n() diff --git a/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue b/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue index 68bfaba4..2687eaf7 100644 --- a/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue +++ b/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue @@ -29,7 +29,7 @@ :aria-describedby="inputErrors[param.name] ? `probe-err-${param.name}` : undefined" :disabled="!canProbe || isExecuting" @input="$emit('update-input', param.name, ($event.target as HTMLInputElement).value)" - @keydown.enter="!isExecuting && canProbe && hasValidInputs && $emit('execute-probe')" + @keydown.enter="!isExecuting && canProbe && hasValidInputs && !isDuplicate && $emit('execute-probe')" > ) @@ -71,13 +71,13 @@ - +