From e634d8bc63635686bbc76c6f8399e06b715ec843 Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 15 Oct 2026 19:50:57 -0400 Subject: [PATCH 1/4] E2E foundation for probeable workflows: seeded problem set, fresh-user helper, graded-wait helper, PR trigger Seed a new problem set e2e-probeable (new slugs only; get_or_create ignores changed defaults on seeded databases) with seven problems built for the student workflows: block mode with a budget of 3, two cooldown variants, a four-test clamp problem whose wrong solutions produce each feedback shape, visible- and hidden-signature oracles that divide by zero, and a spec problem with two tests. Soft deadline 30 days out. Probe budgets and history live in Redis per user with no reset, so e2e/helpers/users.ts creates a fresh enrolled student per test (mock Firebase accepts any email) and grants AI consent where a submission needs it. e2e/helpers/submission.ts waits on the real graded result (eager Celery grades inside the request) and surfaces a 400 as the toast text. selectors.ts gains probe and feedback groups. The e2e workflow only ran on pushes to master; it now also runs on pull requests with the same path filter. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/e2e.yml | 9 + e2e/helpers/selectors.ts | 60 +++++ e2e/helpers/submission.ts | 87 +++++++ e2e/helpers/users.ts | 159 ++++++++++++ .../management/commands/seed_e2e_data.py | 230 +++++++++++++++++- 5 files changed, 542 insertions(+), 3 deletions(-) create mode 100644 e2e/helpers/submission.ts create mode 100644 e2e/helpers/users.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c3a55a2..096722b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -18,6 +18,15 @@ on: - 'playwright.config.ts' - 'package.json' - '.github/workflows/e2e.yml' + pull_request: + paths: + - 'purplex/**' + - 'tests/**' + - 'e2e/**' + - 'requirements.txt' + - 'playwright.config.ts' + - 'package.json' + - '.github/workflows/e2e.yml' workflow_dispatch: env: diff --git a/e2e/helpers/selectors.ts b/e2e/helpers/selectors.ts index 1ad3cfb..9190597 100644 --- a/e2e/helpers/selectors.ts +++ b/e2e/helpers/selectors.ts @@ -240,3 +240,63 @@ export const common = { /** Dialog overlay (ConfirmDialog uses .dialog-overlay) */ dialogOverlay: '.dialog-overlay', } as const; + +// --------------------------------------------------------------------------- +// Probe panel (activities/inputs/shared/ProbePanel.vue) +// --------------------------------------------------------------------------- +export const probe = { + /** Panel root */ + panel: '.probe-panel', + /** Input for one parameter (Enter triggers a probe) */ + paramInput: (name: string) => `input#probe-${name}.param-input`, + /** Result slot ('?' until a result; .has-result / .is-cached) */ + result: '.probe-panel .fn-result', + /** Probe button ('Probe' | 'Probing' | 'Already probed', .is-duplicate) */ + button: 'button.probe-btn', + /** Remaining-probe badge (.status-explore | .status-available | .status-exhausted) */ + badge: 'span.probe-count-badge', + /** Status hint, rendered only when probes are exhausted */ + hint: '.probe-panel .probe-status-hint', + /** Oracle / request error line */ + error: '.probe-panel .probe-error', + /** Per-parameter field errors */ + fieldErrors: 'ul.probe-field-errors', + /** One parameter's field error */ + fieldError: (name: string) => `li#probe-err-${name}`, + /** 'History (N)' button, rendered only when history is non-empty */ + historyButton: '.probe-panel .history-btn', + /** History modal (teleported to body) */ + historyModal: '.modal-overlay .modal-content[role="dialog"]', + /** Rows in the history modal */ + historyRow: '.modal-content[role="dialog"] .modal-row', + /** History modal close button */ + close: '.modal-content[role="dialog"] .modal-close', +} as const; + +// --------------------------------------------------------------------------- +// Code submission feedback (activities/feedback/CodeSubmissionFeedback.vue) +// --------------------------------------------------------------------------- +export const codeFeedback = { + /** Summary bar with pass/fail/locked counts */ + summaryBar: '.feedback-container .test-summary-bar', + /** 'โœ“ N Passing' */ + passing: '.test-summary-bar .count-item.passing', + /** 'โœ— N Failing' */ + failing: '.test-summary-bar .count-item.failing', + /** '๐Ÿ”’ N more tests locked' (only when locked > 0) */ + locked: '.test-summary-bar .count-item.locked', + /** Failing test rows (.test-call, .expected, .actual, .actual.is-error) */ + failingItem: 'article.test-item.failing', + /** Passing test rows (inside the collapsed passing group) */ + passingItem: 'article.test-item.passing', + /** Note in the failing group explaining locked tests */ + lockedNote: 'p.locked-note', + /** Attempts dropdown trigger (.attempt-text 'n/total', .attempt-score-badge) */ + attemptTrigger: '.attempt-dropdown-trigger', + /** Attempts dropdown panel */ + attemptPanel: '.attempt-dropdown-panel[role="menu"]', + /** One attempt in the dropdown (.attempt-tests 'passed/total') */ + attemptItem: '.attempt-item-minimal', + /** Debug button on a test row */ + debugButton: '.debug-btn', +} as const; diff --git a/e2e/helpers/submission.ts b/e2e/helpers/submission.ts new file mode 100644 index 0000000..b75b25f --- /dev/null +++ b/e2e/helpers/submission.ts @@ -0,0 +1,87 @@ +import { Page, expect } from '@playwright/test'; +import { setAceEditorValue } from './ace-editor'; +import { codeFeedback } from './selectors'; + +/** + * Code-submission helpers for probeable_code problems. + * + * In the E2E stack (CELERY_EAGER=true) POST /api/submit/ grades the whole + * submission synchronously in the Docker sandbox, then the SSE stream reports + * completion. ProblemSet.vue clears the previous feedback before it sends the + * request, so once the submit response is back, a visible summary bar can only + * belong to the new result. + */ + +const SUBMIT_URL = /\/api\/submit\/$/; + +/** + * Put `code` in the editor, submit it, and wait until the graded result's + * summary bar is visible. Throws if the submit request is not accepted. + */ +export async function submitCodeAndWait( + page: Page, + code: string, + timeout = 60_000, +): Promise { + await setAceEditorValue(page, '#codeEditor', code); + + const submitButton = page.locator('#submitButton'); + await expect(submitButton).toBeEnabled(); + + const responsePromise = page.waitForResponse( + (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + { timeout }, + ); + await submitButton.click(); + const response = await responsePromise; + if (!response.ok()) { + throw new Error( + `submitCodeAndWait: POST /api/submit/ returned ${response.status()}: ${await response.text()}`, + ); + } + + await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout }); +} + +/** + * Submit code that the API rejects as invalid (e.g. a syntax error) and return + * the message the user sees. + * + * The probeable_code handler rejects uncompilable code with 400 + * {"error": "Syntax error: (line N)"} before any submission exists. + * ProblemSet.vue's submit() catch shows it as a warning toast titled + * "Invalid Request" whose .notification-details is the server's error text. + * This asserts that toast and returns the details text. + */ +export async function submitSyntaxErrorAndReadMessage(page: Page, code: string): Promise { + await setAceEditorValue(page, '#codeEditor', code); + + const submitButton = page.locator('#submitButton'); + await expect(submitButton).toBeEnabled(); + + const responsePromise = page.waitForResponse( + (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + ); + await submitButton.click(); + const response = await responsePromise; + if (response.status() !== 400) { + throw new Error( + `submitSyntaxErrorAndReadMessage: expected 400 from POST /api/submit/, got ${response.status()}`, + ); + } + + const body = await response.json(); + if (typeof body?.error !== 'string' || body.error === '') { + throw new Error(`submitSyntaxErrorAndReadMessage: 400 without an error message: ${JSON.stringify(body)}`); + } + + // Match the toast to this response (an older toast may still be on screen). + // Toasts auto-dismiss after 4-5 s, so read the text as soon as it shows. + const toast = page + .locator('.notification-container .notification.warning') + .filter({ has: page.locator('.notification-details', { hasText: body.error }) }) + .last(); + await expect(toast).toBeVisible(); + await expect(toast.locator('.notification-message')).toHaveText('Invalid Request'); + return ((await toast.locator('.notification-details').textContent()) ?? '').trim(); +} diff --git a/e2e/helpers/users.ts b/e2e/helpers/users.ts new file mode 100644 index 0000000..9c1980d --- /dev/null +++ b/e2e/helpers/users.ts @@ -0,0 +1,159 @@ +import { Page } from '@playwright/test'; + +/** + * Fresh, per-test student users. + * + * Probe budgets and probe history live in Redis per (problem, user) with a + * 7-day TTL and no reset, and submissions accumulate per user. Any test that + * touches a budget, history or submissions must therefore run as a user no + * other test (or earlier run) has used. + * + * Mock Firebase accepts any email: the backend (users_app/mock_firebase.py) + * creates a student for an unknown email using the uid from the token, and + * the frontend MockFirebase restores its session from localStorage. + */ +export interface FreshUser { + uid: string; + email: string; + displayName: string; +} + +/** Create a unique user identity. Nothing is sent to the server. */ +export function freshStudent(label: string): FreshUser { + const safeLabel = label.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, ''); + const id = `e2e-${safeLabel}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + return { + uid: `mock-uid-${id}`, + email: `${id}@test.local`, + displayName: `E2E ${label}`, + }; +} + +/** + * Inject this user into localStorage so the Vue app boots as them. + * Mirrors injectAuth in auth.ts: Vuex `user`, `mockFirebaseAuth`, and the + * cookie consent flag. Must be called before navigating to the target route. + */ +export async function injectUser(page: Page, user: FreshUser): Promise { + await page.goto('/', { waitUntil: 'commit' }); + + await page.evaluate( + ({ userJson, mockFirebaseAuthJson, consent }) => { + localStorage.setItem('user', userJson); + localStorage.setItem('mockFirebaseAuth', mockFirebaseAuthJson); + localStorage.setItem('purplex_cookie_consent', consent); + }, + { + userJson: JSON.stringify({ + uid: user.uid, + email: user.email, + displayName: user.displayName, + role: 'user', + isAdmin: false, + }), + mockFirebaseAuthJson: JSON.stringify({ + uid: user.uid, + email: user.email, + displayName: user.displayName, + }), + consent: 'accepted', + }, + ); +} + +/** + * Mint a mock token for this user. Same payload as the frontend's + * MockFirebase.createMockToken, which the backend decodes as + * MOCK..development. + */ +function mockTokenFor(user: FreshUser): string { + const now = Math.floor(Date.now() / 1000); + const payload = { + uid: user.uid, + email: user.email, + name: user.displayName, + email_verified: true, + iat: now, + exp: now + 3600, + }; + return `MOCK.${btoa(JSON.stringify(payload))}.development`; +} + +/** + * Call the API as this user from inside the page (same origin, through the + * Vite proxy). The page must already be on the app origin. + */ +export async function apiAsUser( + page: Page, + user: FreshUser, + method: string, + path: string, + body?: object, +): Promise<{ status: number; data: any }> { + const token = mockTokenFor(user); + return page.evaluate( + async ({ token, method, path, body }) => { + const res = await fetch(path, { + method, + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + let data: unknown = null; + if (text) { + try { + data = JSON.parse(text); + } catch { + data = text; + } + } + return { status: res.status, data }; + }, + { token, method, path, body }, + ); +} + +/** Enroll this user in a course. Throws unless the API answers 200 or 201. */ +export async function enrollUser( + page: Page, + user: FreshUser, + courseId = 'CS101-2024', +): Promise { + const { status, data } = await apiAsUser(page, user, 'POST', '/api/courses/enroll/', { + course_id: courseId, + }); + if (status !== 200 && status !== 201) { + throw new Error( + `enrollUser: ${user.email} -> ${courseId} returned ${status}: ${JSON.stringify(data)}`, + ); + } +} + +/** Create a fresh student, log in, enroll in CS101-2024, and open `path`. */ +export async function navigateAsFreshStudent( + page: Page, + label: string, + path: string, +): Promise { + const user = freshStudent(label); + await injectUser(page, user); + await enrollUser(page, user); + await page.goto(path, { waitUntil: 'networkidle' }); + return user; +} + +/** + * Grant this user AI-processing consent. Fresh users have none, and spec-type + * submissions (LLM code generation) are blocked without it. + */ +export async function grantAiConsent(page: Page, user: FreshUser): Promise { + const { status, data } = await apiAsUser(page, user, 'POST', '/api/users/me/consents/', { + consent_type: 'ai_processing', + }); + if (status !== 200 && status !== 201) { + throw new Error(`grantAiConsent: ${user.email} returned ${status}: ${JSON.stringify(data)}`); + } +} diff --git a/purplex/problems_app/management/commands/seed_e2e_data.py b/purplex/problems_app/management/commands/seed_e2e_data.py index 1501d51..8643061 100644 --- a/purplex/problems_app/management/commands/seed_e2e_data.py +++ b/purplex/problems_app/management/commands/seed_e2e_data.py @@ -744,6 +744,23 @@ def handle(self, *args, **options): ) self._report("CourseProblemSet", "e2e-mixed -> CS101-2024 (past hard)", created) + # ------------------------------------------------------------------ + # 5b. Probeable workflows set (e2e-probeable), soft deadline in 30 days + # + # URL: /courses/CS101-2024/problem-set/e2e-probeable?p= + # p=0 e2e-probeable-block code, block mode, 3 probes + # p=1 e2e-probeable-cooldown code, cooldown 2 probes, +2 per submit + # p=2 e2e-probeable-locked code, explore, 4 tests (truncation) + # p=3 e2e-probeable-errors code, block 3, recip(0) raises + # p=4 e2e-probeable-hidden code, explore, hidden signature + # p=5 e2e-probeable-spec-open spec, explore, no segmentation + # p=6 e2e-probeable-cooldown-2 code, cooldown 1 probe, +1 per 2 submits + # + # These slugs are separate from the older probeable problems so their + # defaults can be relied on (get_or_create ignores changed defaults). + # ------------------------------------------------------------------ + self._seed_probeable_workflows(course, instructor, now) + # ------------------------------------------------------------------ # 6. Hints for e2e-eipl-1 # ------------------------------------------------------------------ @@ -893,9 +910,10 @@ def handle(self, *args, **options): self.style.SUCCESS( "\nE2E seed data ready!\n" f" Categories: 2\n" - f" Problems: {len(all_problems)} (7 by instructor + 4 by admin)\n" - f" Problem Sets: 5 (3 assigned to CS101-2024, 2 unassigned)\n" - f" CourseProblemSet linkages: 3\n" + f" Problems: {len(all_problems)} (7 by instructor + 4 by admin)" + " + 7 probeable workflow problems\n" + f" Problem Sets: 6 (4 assigned to CS101-2024, 2 unassigned)\n" + f" CourseProblemSet linkages: 4\n" f" Hints: 3 (variable_fade, subgoal_highlight, suggested_trace)\n" f" Submission: 1 (student -> e2e-mcq-1)\n" f" Consent: 2 (behavioral_tracking, ai_processing)\n" @@ -903,6 +921,212 @@ def handle(self, *args, **options): ) ) + def _seed_probeable_workflows(self, course, instructor, now): + """Seed the e2e-probeable problem set used by the probeable E2E specs. + + Test case ``inputs`` is the positional argument list, so ``add(1, 2)`` + is ``[1, 2]`` and ``clamp(5)`` is ``[5]``. + + Designed outcomes on e2e-probeable-locked (tests run in order + 5, 0, -3, 20; rows after the first failure are locked): + return n -> 2/4, 3 rows (1 failing), 1 locked + return max(0, n) -> 3/4, 4 rows (1 failing), 0 locked + return max(0, min(n, 10)) -> 4/4, 4 rows, 0 locked + return n // 0 -> 0/4, 1 error row, 3 locked + return 0 if n == 5 else -> 3/4, 1 failing row, 3 locked + """ + from datetime import timedelta + + from purplex.problems_app.models import ( + CourseProblemSet, + ProbeableCodeProblem, + ProbeableSpecProblem, + ProblemSet, + ProblemSetMembership, + TestCase, + ) + + add_signature = "def add(a: int, b: int) -> int" + add_reference = "def add(a, b):\n return a + b" + recip_signature = "def recip(x: int) -> int" + recip_reference = "def recip(x):\n return 10 // x" + + # (model, slug, defaults, test cases as (inputs, expected, is_sample)) + specs = [ + ( + ProbeableCodeProblem, + "e2e-probeable-block", + { + "title": "E2E Probeable: Block Mode", + "description": "Probe the function, then write it.", + "function_signature": add_signature, + "function_name": "add", + "reference_solution": add_reference, + "show_function_signature": True, + "probe_mode": "block", + "max_probes": 3, + "cooldown_attempts": 3, + "cooldown_refill": 5, + }, + [([1, 2], 3, True)], + ), + ( + ProbeableCodeProblem, + "e2e-probeable-cooldown", + { + "title": "E2E Probeable: Cooldown Mode", + "description": "Probe the function, then write it.", + "function_signature": add_signature, + "function_name": "add", + "reference_solution": add_reference, + "show_function_signature": True, + "probe_mode": "cooldown", + "max_probes": 2, + "cooldown_attempts": 1, + "cooldown_refill": 2, + }, + [([1, 2], 3, True)], + ), + ( + ProbeableCodeProblem, + "e2e-probeable-locked", + { + "title": "E2E Probeable: Locked Tests", + "description": "Probe the function, then write it.", + "function_signature": "def clamp(n: int) -> int", + "function_name": "clamp", + "reference_solution": "def clamp(n):\n return max(0, min(n, 10))", + "show_function_signature": True, + "probe_mode": "explore", + }, + [([5], 5, True), ([0], 0, False), ([-3], 0, False), ([20], 10, False)], + ), + ( + ProbeableCodeProblem, + "e2e-probeable-errors", + { + "title": "E2E Probeable: Oracle Errors", + "description": "Probe the function, then write it.", + "function_signature": recip_signature, + "function_name": "recip", + "reference_solution": recip_reference, + "show_function_signature": True, + "probe_mode": "block", + "max_probes": 3, + }, + [([2], 5, True)], + ), + ( + ProbeableCodeProblem, + "e2e-probeable-hidden", + { + "title": "E2E Probeable: Hidden Signature", + "description": "Probe the function, then write it.", + "function_signature": recip_signature, + "function_name": "recip", + "reference_solution": recip_reference, + "show_function_signature": False, + "probe_mode": "explore", + }, + [([2], 5, True)], + ), + ( + ProbeableSpecProblem, + "e2e-probeable-spec-open", + { + "title": "E2E Probeable: Explain It", + "description": "Probe the function, then explain it.", + "function_signature": "def square(x: int) -> int", + "function_name": "square", + "reference_solution": "def square(x):\n return x * x", + "show_function_signature": True, + "probe_mode": "explore", + # Plain pass/fail grade: no comprehension analysis. + "segmentation_config": {"enabled": False}, + "requires_highlevel_comprehension": False, + }, + [([3], 9, True), ([-2], 4, False)], + ), + ( + # Two submissions per refill, so the exhausted hint counts + # down through the plural and singular texts. + ProbeableCodeProblem, + "e2e-probeable-cooldown-2", + { + "title": "E2E Probeable: Two-Submission Cooldown", + "description": "Probe the function, then write it.", + "function_signature": add_signature, + "function_name": "add", + "reference_solution": add_reference, + "show_function_signature": True, + "probe_mode": "cooldown", + "max_probes": 1, + "cooldown_attempts": 2, + "cooldown_refill": 1, + }, + [([1, 2], 3, True)], + ), + ] + + problem_set, created = ProblemSet.objects.get_or_create( + slug="e2e-probeable", + defaults={ + "title": "E2E Probeable Workflows", + "description": "Probeable problems for E2E workflow tests", + "is_public": True, + "created_by": instructor, + }, + ) + self._report("ProblemSet", "e2e-probeable", created) + + for index, (model, slug, defaults, cases) in enumerate(specs): + problem, created = model.objects.get_or_create( + slug=slug, + defaults={ + **defaults, + "difficulty": "beginner", + "is_active": True, + "created_by": instructor, + }, + ) + self._report(model.__name__, slug, created) + + for order, (inputs, expected, is_sample) in enumerate(cases): + _, created = TestCase.objects.get_or_create( + problem=problem, + inputs=inputs, + expected_output=expected, + defaults={ + "is_sample": is_sample, + "is_hidden": not is_sample, + "order": order, + "description": ( + f"{problem.function_name}" + f"({', '.join(map(str, inputs))}) = {expected}" + ), + }, + ) + self._report(f"TestCase ({slug})", f"inputs={inputs}", created) + + _, created = ProblemSetMembership.objects.get_or_create( + problem_set=problem_set, + problem=problem, + defaults={"order": index}, + ) + self._report("ProblemSetMembership (probeable)", slug, created) + + _, created = CourseProblemSet.objects.update_or_create( + course=course, + problem_set=problem_set, + defaults={ + "order": 4, + "due_date": now + timedelta(days=30), + "deadline_type": "soft", + "is_required": True, + }, + ) + self._report("CourseProblemSet", "e2e-probeable -> CS101-2024", created) + def _report(self, model_name: str, identifier: str, created: bool): """Print whether an object was created or already existed.""" if created: From c5265cb775dc8165a073596b44158bc53631dfb7 Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 15 Oct 2026 19:51:05 -0400 Subject: [PATCH 2/4] E2E: cover every student-facing probeable workflow Four new specs, one fresh student per test, run green twice in a row with two workers against a shared stack: - probing: first/repeat/duplicate probes, invalid types, history modal, reload, explore mode, hidden-signature sanitising, visible oracle error not charged - budget: block mode countdown to exhaustion and the per-user budget, cooldown exhaustion, refill after a submission without a reload, and the plural refill hint counting down - feedback: too short, syntax error with no attempt stored, grading in progress, all pass, early failure with later passes locked, singular lock, nothing locked when only the last fails, runtime error row, resubmit, switching attempts, reload restoring the truncated result, debugger modal - spec: probing, explanation graded by the mock LLM with untruncated rows, length bounds, two attempts submission-probeable.spec.ts loses its runtime skips, swallowed waits and accept-any-status checks, and its header no longer claims a Celery worker and Docker-in-Docker. Co-Authored-By: Claude Fable 5.1 --- e2e/student/probeable-budget.spec.ts | 365 ++++++++++++++++++++++ e2e/student/probeable-feedback.spec.ts | 378 +++++++++++++++++++++++ e2e/student/probeable-probing.spec.ts | 361 ++++++++++++++++++++++ e2e/student/probeable-spec.spec.ts | 250 +++++++++++++++ e2e/student/submission-probeable.spec.ts | 234 +++++++------- 5 files changed, 1465 insertions(+), 123 deletions(-) create mode 100644 e2e/student/probeable-budget.spec.ts create mode 100644 e2e/student/probeable-feedback.spec.ts create mode 100644 e2e/student/probeable-probing.spec.ts create mode 100644 e2e/student/probeable-spec.spec.ts diff --git a/e2e/student/probeable-budget.spec.ts b/e2e/student/probeable-budget.spec.ts new file mode 100644 index 0000000..875c554 --- /dev/null +++ b/e2e/student/probeable-budget.spec.ts @@ -0,0 +1,365 @@ +/** + * E2E Tests: probe budgets on probeable_code problems. + * + * Problem set e2e-probeable (seed_e2e_data._seed_probeable_workflows): + * p=0 e2e-probeable-block add(a, b), probe_mode "block", max_probes 3 + * p=1 e2e-probeable-cooldown add(a, b), probe_mode "cooldown", max_probes 2, + * refill 2 probes after 1 submission + * p=6 e2e-probeable-cooldown-2 add(a, b), probe_mode "cooldown", max_probes 1, + * refill 1 probe after 2 submissions + * + * Budgets live in Redis per (problem, user) with no reset, so every test runs + * as a fresh student (e2e/helpers/users.ts). + * + * UI facts this spec relies on (ProbePanel.vue / useProbeState.ts): + * - The remaining-count badge is rendered inside the Probe button only while + * the current inputs are not a previous probe. After a probe the inputs + * keep their values, so the button switches to "Already probed"; the spec + * enters new values before it reads the badge. + * - A duplicate input is answered from the client-side history: no request. + */ + +import { test, expect, Page, Response } from '@playwright/test'; +import { navigateAsFreshStudent, freshStudent, injectUser, enrollUser } from '../helpers/users'; +import { submitCodeAndWait } from '../helpers/submission'; +import { probe } from '../helpers/selectors'; + +const SET_URL = '/courses/CS101-2024/problem-set/e2e-probeable'; +const BLOCK_URL = `${SET_URL}?p=0`; +const COOLDOWN_URL = `${SET_URL}?p=1`; +const COOLDOWN_2_URL = `${SET_URL}?p=6`; +const ADD_SOLUTION = 'def add(a, b):\n return a + b'; + +function isProbePost(resp: Response, slug: string): boolean { + return ( + resp.request().method() === 'POST' && + new URL(resp.url()).pathname === `/api/problems/${slug}/probe/` + ); +} + +/** Wait until the probe panel and its add(a, b) inputs are on screen. */ +async function waitForProbePanel(page: Page): Promise { + await expect(page.locator(probe.panel)).toBeVisible({ timeout: 15_000 }); + await expect(page.locator(probe.paramInput('a'))).toBeVisible(); + await expect(page.locator(probe.paramInput('b'))).toBeVisible(); +} + +async function fillArgs(page: Page, a: string, b: string): Promise { + await page.locator(probe.paramInput('a')).fill(a); + await page.locator(probe.paramInput('b')).fill(b); +} + +/** + * Probe add(a, b) through the UI, assert the inline result, and return the + * probe POST response body. + */ +async function probeAdd( + page: Page, + slug: string, + a: string, + b: string, + expected: string, +): Promise { + await fillArgs(page, a, b); + const button = page.locator(probe.button); + await expect(button).toBeEnabled(); + const responsePromise = page.waitForResponse((resp) => isProbePost(resp, slug)); + await button.click(); + const response = await responsePromise; + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.success).toBe(true); + expect(body.result).toBe(Number(expected)); + await expect(page.locator(probe.result)).toHaveText(expected); + await expect(page.locator(probe.result)).toHaveClass(/has-result/); + return body; +} + +/** Assert the badge reads `ยท N left` with the given status class. */ +async function expectBadge(page: Page, remaining: number, statusClass: string): Promise { + const badge = page.locator(probe.badge); + await expect(badge).toHaveText(`ยท ${remaining} left`); + await expect(badge).toHaveClass(new RegExp(`\\b${statusClass}\\b`)); +} + +// --------------------------------------------------------------------------- +// A8: block mode +// --------------------------------------------------------------------------- +test.describe('Probe budget: block mode', () => { + test('A8 block mode spends the budget and blocks probing, and a fresh load shows the exhausted state', async ({ + page, + }) => { + const slug = 'e2e-probeable-block'; + await navigateAsFreshStudent(page, 'a8-block', BLOCK_URL); + await waitForProbePanel(page); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (req.method() === 'POST' && new URL(req.url()).pathname === `/api/problems/${slug}/probe/`) { + probePosts.push(req.postData() ?? ''); + } + }); + + // Fresh user: full budget, no hint, no history. + await expectBadge(page, 3, 'status-available'); + await expect(page.locator(probe.hint)).toHaveCount(0); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + await expect(page.locator(probe.result)).toHaveText('?'); + + // Probe 1: add(1, 2) -> 3 + const first = await probeAdd(page, slug, '1', '2', '3'); + expect(first.probe_status.remaining).toBe(2); + expect(first.probe_status.can_probe).toBe(true); + + // Probe 2: add(2, 2) -> 4. The badge is visible once the inputs are new. + await fillArgs(page, '2', '2'); + await expectBadge(page, 2, 'status-available'); + const second = await probeAdd(page, slug, '2', '2', '4'); + expect(second.probe_status.remaining).toBe(1); + + // A duplicate of (1, 2) shows the cached result and sends nothing. + const postsBeforeDuplicate = probePosts.length; + await fillArgs(page, '1', '2'); + const button = page.locator(probe.button); + await expect(button).toHaveText(/Already probed/); + await expect(button).toHaveClass(/is-duplicate/); + await expect(button).toBeDisabled(); + await expect(page.locator(probe.result)).toHaveText('3'); + await expect(page.locator(probe.result)).toHaveClass(/is-cached/); + + // Confirming the duplicate from the keyboard must not probe either. + await page.locator(probe.paramInput('b')).press('Enter'); + + // The duplicate was free: one probe is still left. + await fillArgs(page, '3', '3'); + await expectBadge(page, 1, 'status-available'); + expect(probePosts.length).toBe(postsBeforeDuplicate); + + // Probe 3: add(3, 3) -> 6 exhausts the budget. + const third = await probeAdd(page, slug, '3', '3', '6'); + expect(third.probe_status.can_probe).toBe(false); + expect(third.probe_status.remaining).toBe(0); + expect(third.probe_status.mode).toBe('block'); + + // Exhausted state without a reload. + await expect(page.locator(probe.hint)).toHaveText('No probes remaining.'); + await expect(page.locator(probe.button)).toBeDisabled(); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(page.locator(probe.paramInput('b'))).toBeDisabled(); + await expect(page.locator(probe.historyButton)).toHaveText('History (3)'); + + // Server-side state survives a fresh load (second tab, same student). + // Checked before the in-page badge so it is verified even if that fails. + const freshTab = await page.context().newPage(); + try { + await freshTab.goto(BLOCK_URL, { waitUntil: 'networkidle' }); + await waitForProbePanel(freshTab); + await expectBadge(freshTab, 0, 'status-exhausted'); + await expect(freshTab.locator(probe.hint)).toHaveText('No probes remaining.'); + await expect(freshTab.locator(probe.button)).toBeDisabled(); + await expect(freshTab.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(freshTab.locator(probe.paramInput('b'))).toBeDisabled(); + await expect(freshTab.locator(probe.result)).toHaveText('6'); + + // Earlier results stay readable from the history after exhaustion. + await freshTab.locator(probe.historyButton).click(); + await expect(freshTab.locator(probe.historyModal)).toBeVisible(); + const rows = freshTab.locator(probe.historyRow); + await expect(rows).toHaveCount(3); + await expect(rows.locator('.modal-call')).toHaveText(['add(3, 3)', 'add(2, 2)', 'add(1, 2)']); + await expect(rows.locator('.modal-result')).toHaveText(['6', '4', '3']); + await freshTab.locator(probe.close).click(); + await expect(freshTab.locator(probe.historyModal)).toHaveCount(0); + } finally { + await freshTab.close(); + } + + // Only the three real probes reached the server. + expect(probePosts.length).toBe(3); + + // The page the student probed from must also show the exhausted count. + await expectBadge(page, 0, 'status-exhausted'); + }); + + test('A8 a duplicate probe confirmed with Enter sends no request and spends no budget', async ({ + page, + }) => { + const slug = 'e2e-probeable-block'; + await navigateAsFreshStudent(page, 'a8-dup-enter', BLOCK_URL); + await waitForProbePanel(page); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (req.method() === 'POST' && new URL(req.url()).pathname === `/api/problems/${slug}/probe/`) { + probePosts.push(req.postData() ?? ''); + } + }); + + const first = await probeAdd(page, slug, '1', '2', '3'); + expect(first.probe_status.remaining).toBe(2); + expect(probePosts.length).toBe(1); + + // The inputs still hold (1, 2): the panel shows the cached result. + const button = page.locator(probe.button); + await expect(button).toHaveText(/Already probed/); + await expect(button).toBeDisabled(); + await expect(page.locator(probe.result)).toHaveClass(/is-cached/); + await expect(page.locator(probe.result)).toHaveText('3'); + + // Enter in a parameter field is the keyboard path to "Probe". On a + // duplicate it must do nothing, like the disabled button. + await page.locator(probe.paramInput('b')).press('Enter'); + + // Budget is server-side: a fresh load shows whether anything was spent. + await page.reload({ waitUntil: 'networkidle' }); + await waitForProbePanel(page); + await expectBadge(page, 2, 'status-available'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + expect(probePosts.length).toBe(1); + }); + + test('A8b block budgets are per user: a second student starts with the full budget', async ({ + page, + browser, + }) => { + const slug = 'e2e-probeable-block'; + await navigateAsFreshStudent(page, 'a8b-first', BLOCK_URL); + await waitForProbePanel(page); + await expectBadge(page, 3, 'status-available'); + const first = await probeAdd(page, slug, '5', '6', '11'); + expect(first.probe_status.remaining).toBe(2); + await fillArgs(page, '7', '8'); + await expectBadge(page, 2, 'status-available'); + + // A different student, in a separate browser context, on the same problem. + const otherContext = await browser.newContext(); + try { + const otherPage = await otherContext.newPage(); + const other = freshStudent('a8b-second'); + await injectUser(otherPage, other); + await enrollUser(otherPage, other); + await otherPage.goto(BLOCK_URL, { waitUntil: 'networkidle' }); + await waitForProbePanel(otherPage); + await expectBadge(otherPage, 3, 'status-available'); + await expect(otherPage.locator(probe.hint)).toHaveCount(0); + await expect(otherPage.locator(probe.historyButton)).toHaveCount(0); + await expect(otherPage.locator(probe.result)).toHaveText('?'); + } finally { + await otherContext.close(); + } + + // The first student's budget is unchanged by the second student. + await page.reload({ waitUntil: 'networkidle' }); + await waitForProbePanel(page); + await expectBadge(page, 2, 'status-available'); + }); +}); + +// --------------------------------------------------------------------------- +// A9: cooldown mode +// --------------------------------------------------------------------------- +test.describe('Probe budget: cooldown mode', () => { + test('A9 cooldown mode refills the budget after a submission', async ({ page }) => { + // Grading runs the submission in the Docker sandbox. + test.setTimeout(90_000); + const slug = 'e2e-probeable-cooldown'; + const refillHint = 'No probes remaining. Submit 1 more time to unlock more probes.'; + await navigateAsFreshStudent(page, 'a9-cooldown', COOLDOWN_URL); + await waitForProbePanel(page); + + await expectBadge(page, 2, 'status-available'); + await expect(page.locator(probe.hint)).toHaveCount(0); + + const first = await probeAdd(page, slug, '1', '2', '3'); + expect(first.probe_status.remaining).toBe(1); + + await fillArgs(page, '2', '2'); + await expectBadge(page, 1, 'status-available'); + const second = await probeAdd(page, slug, '2', '2', '4'); + expect(second.probe_status.can_probe).toBe(false); + expect(second.probe_status.remaining).toBe(0); + expect(second.probe_status.mode).toBe('cooldown'); + expect(second.probe_status.submissions_to_next_refill).toBe(1); + + await expect(page.locator(probe.hint)).toHaveText(refillHint); + await expect(page.locator(probe.button)).toBeDisabled(); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(page.locator(probe.paramInput('b'))).toBeDisabled(); + + // One submission earns the refill. + await submitCodeAndWait(page, ADD_SOLUTION); + + // A fresh load of the same problem (second tab, same student) shows the + // server-side refill. Checked first so it is documented even if the + // in-page state below is stale. + const freshTab = await page.context().newPage(); + try { + await freshTab.goto(COOLDOWN_URL, { waitUntil: 'networkidle' }); + await waitForProbePanel(freshTab); + await expectBadge(freshTab, 2, 'status-available'); + await expect(freshTab.locator(probe.hint)).toHaveCount(0); + await expect(freshTab.locator(probe.paramInput('a'))).toBeEnabled(); + await expect(freshTab.locator(probe.paramInput('b'))).toBeEnabled(); + } finally { + await freshTab.close(); + } + + // Without a reload, the page the student submitted from must show the + // refilled budget too. + await expect(page.locator(probe.hint)).toHaveCount(0); + await expect(page.locator(probe.paramInput('a'))).toBeEnabled(); + await expect(page.locator(probe.paramInput('b'))).toBeEnabled(); + await fillArgs(page, '3', '3'); + await expectBadge(page, 2, 'status-available'); + + // A further probe works and spends from the refilled budget. + const third = await probeAdd(page, slug, '3', '3', '6'); + expect(third.probe_status.can_probe).toBe(true); + expect(third.probe_status.remaining).toBe(1); + await fillArgs(page, '4', '4'); + await expectBadge(page, 1, 'status-available'); + }); + + test('A9b the refill hint counts down the submissions still needed', async ({ page }) => { + // Two graded submissions in the Docker sandbox. + test.setTimeout(120_000); + const slug = 'e2e-probeable-cooldown-2'; + await navigateAsFreshStudent(page, 'a9b-cooldown-2', COOLDOWN_2_URL); + await waitForProbePanel(page); + + await expectBadge(page, 1, 'status-available'); + await expect(page.locator(probe.hint)).toHaveCount(0); + + const only = await probeAdd(page, slug, '1', '2', '3'); + expect(only.probe_status.can_probe).toBe(false); + expect(only.probe_status.remaining).toBe(0); + expect(only.probe_status.submissions_to_next_refill).toBe(2); + await expect(page.locator(probe.hint)).toHaveText( + 'No probes remaining. Submit 2 more times to unlock more probes.', + ); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + + // First submission: one more is still needed, and the budget stays empty. + await submitCodeAndWait(page, ADD_SOLUTION); + await expect(page.locator(probe.hint)).toHaveText( + 'No probes remaining. Submit 1 more time to unlock more probes.', + ); + await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expect(page.locator(probe.button)).toBeDisabled(); + await expectBadge(page, 0, 'status-exhausted'); + + // Second submission earns the refill. + await submitCodeAndWait(page, ADD_SOLUTION); + await expect(page.locator(probe.hint)).toHaveCount(0); + await expect(page.locator(probe.paramInput('a'))).toBeEnabled(); + await fillArgs(page, '2', '2'); + await expectBadge(page, 1, 'status-available'); + + const refilled = await probeAdd(page, slug, '2', '2', '4'); + expect(refilled.probe_status.remaining).toBe(0); + expect(refilled.probe_status.submissions_to_next_refill).toBe(2); + await expect(page.locator(probe.hint)).toHaveText( + 'No probes remaining. Submit 2 more times to unlock more probes.', + ); + }); +}); diff --git a/e2e/student/probeable-feedback.spec.ts b/e2e/student/probeable-feedback.spec.ts new file mode 100644 index 0000000..0018608 --- /dev/null +++ b/e2e/student/probeable-feedback.spec.ts @@ -0,0 +1,378 @@ +/** + * E2E: probeable_code submission feedback (workflows B14-B23). + * + * Problem: e2e-probeable-locked (problem set e2e-probeable, p=2), seeded by + * seed_e2e_data. clamp(n) = max(0, min(n, 10)), explore probe mode, four + * ordered tests: clamp(5) -> 5, clamp(0) -> 0, clamp(-3) -> 0, clamp(20) -> 10. + * + * Truncation rule (#165): the rows shown are the passing tests up to the + * first failure plus that first failing test; every later test is locked. + * The summary bar's pass/fail counts still cover all four tests. + * + * With CELERY_EAGER=true, POST /api/submit/ grades synchronously in the Docker + * sandbox. Every test uses a fresh student, so attempts never leak between + * tests or runs. + */ + +import { test, expect, Page } from '@playwright/test'; +import { FreshUser, apiAsUser, navigateAsFreshStudent } from '../helpers/users'; +import { submitCodeAndWait, submitSyntaxErrorAndReadMessage } from '../helpers/submission'; +import { setAceEditorValue } from '../helpers/ace-editor'; +import { codeFeedback } from '../helpers/selectors'; + +const CLAMP_URL = '/courses/CS101-2024/problem-set/e2e-probeable?p=2'; +const GRADING_TIMEOUT = 60_000; + +const SOLUTIONS = { + correct: 'def clamp(n):\n return max(0, min(n, 10))', + failsFirst: 'def clamp(n):\n return 0 if n == 5 else max(0, min(n, 10))', + identity: 'def clamp(n):\n return n', + failsLast: 'def clamp(n):\n return max(0, n)', + divideByZero: 'def clamp(n):\n return n // 0', + syntaxError: 'def clamp(n):\n return (n', +} as const; + +// Each test owns a fresh user, so they can run concurrently across workers. +test.describe.configure({ mode: 'parallel' }); + +// Grading runs in the Docker sandbox and some tests submit twice. +test.beforeEach(() => { + test.setTimeout(150_000); +}); + +async function openClamp(page: Page, label: string): Promise { + const user = await navigateAsFreshStudent(page, label, CLAMP_URL); + await expect(page.locator('.probeable-code-input')).toBeVisible(); + await expect(page.locator('#codeEditor .ace_editor')).toBeVisible(); + return user; +} + +async function submitAndGrade(page: Page, code: string): Promise { + await submitCodeAndWait(page, code, GRADING_TIMEOUT); +} + +async function expectSummary( + page: Page, + passing: number, + failing: number, + lockedText: string | null, +): Promise { + await expect(page.locator(codeFeedback.passing)).toHaveText(`โœ“ ${passing} Passing`); + await expect(page.locator(codeFeedback.failing)).toHaveText(`โœ— ${failing} Failing`); + if (lockedText === null) { + await expect(page.locator(codeFeedback.locked)).toHaveCount(0); + await expect(page.locator(codeFeedback.lockedNote)).toHaveCount(0); + } else { + await expect(page.locator(codeFeedback.locked)).toHaveText(lockedText); + await expect(page.locator(codeFeedback.lockedNote)).toBeVisible(); + await expect(page.locator(codeFeedback.lockedNote)).toHaveText( + 'Remaining tests unlock once this test passes. Keep probing to discover more behavior.', + ); + } +} + +async function expectSingleFailingRow( + page: Page, + call: string, + expected: string, + actual: string, +): Promise { + const failing = page.locator(codeFeedback.failingItem); + await expect(failing).toHaveCount(1); + await expect(failing.locator('.test-call')).toHaveText(call); + await expect(failing.locator('.expected')).toHaveText(expected); + await expect(failing.locator('.actual')).toHaveText(actual); + await expect(failing.locator('.actual')).not.toHaveClass(/is-error/); +} + +/** Expand the collapsed passing group and check its rows, in order. */ +async function expectPassingRows(page: Page, rows: string[]): Promise { + const header = page.locator('summary.test-group-header.passing'); + await expect(header).toContainText(`Passing Tests (${rows.length})`); + const items = page.locator(codeFeedback.passingItem); + await expect(items).toHaveCount(rows.length); + await header.click(); + await expect(items.first()).toBeVisible(); + await expect(items.locator('.test-call')).toHaveText(rows); +} + +async function expectAttemptBadge(page: Page, attemptText: string, score: string): Promise { + const trigger = page.locator(codeFeedback.attemptTrigger); + await expect(trigger.locator('.attempt-text')).toHaveText(attemptText); + await expect(trigger.locator('.attempt-score-badge')).toHaveText(new RegExp(`^\\S\\s*${score}%\\s*$`)); +} + +// --------------------------------------------------------------------------- +// B14: input rejected before grading +// --------------------------------------------------------------------------- + +test.describe('B14 rejected submissions', () => { + test('B14a too-short code keeps the submit button disabled', async ({ page }) => { + await openClamp(page, 'b14a'); + + await setAceEditorValue(page, '#codeEditor', 'x'); + await expect(page.locator('#submitButton')).toBeDisabled(); + }); + + test('B14b syntax error is reported with its line and creates no attempt', async ({ page }) => { + const user = await openClamp(page, 'b14b'); + + const message = await submitSyntaxErrorAndReadMessage(page, SOLUTIONS.syntaxError); + expect(message).toMatch(/^Syntax error: .+ \(line 2\)$/); + + // Server state: the user has no stored attempt on this problem. No + // problem-set or course filter, so an attempt stored anywhere would show. + const history = await apiAsUser( + page, + user, + 'GET', + '/api/submissions/history/e2e-probeable-locked/', + ); + expect(history.status).toBe(200); + expect(history.data.total_attempts).toBe(0); + expect(history.data.submissions).toEqual([]); + + // The feedback panel stays in its empty state: no results, no attempts. + const container = page.locator('.feedback-container'); + await expect(container.locator('.empty-state')).toHaveText( + /Submit your solution to see test results/, + ); + await expect(page.locator('.feedback-header')).toHaveCount(0); + await expect(page.locator(codeFeedback.summaryBar)).toHaveCount(0); + await expect(page.locator(codeFeedback.attemptTrigger)).toHaveCount(0); + }); +}); + +// --------------------------------------------------------------------------- +// B15: grading in progress +// --------------------------------------------------------------------------- + +test.describe('B15 grading in progress', () => { + test('B15 grading in progress disables submit and shows the running-tests panel', async ({ page }) => { + await openClamp(page, 'b15'); + await setAceEditorValue(page, '#codeEditor', SOLUTIONS.correct); + + // Hold the submit request until the in-flight state has been checked, so + // the assertions do not race the sandbox. + let release: () => void = () => {}; + const released = new Promise((resolve) => { + release = resolve; + }); + await page.route('**/api/submit/', async (route) => { + await released; + await route.continue(); + }); + + const submitButton = page.locator('#submitButton'); + const panel = page.locator('.feedback-container .generating-feedback-panel'); + await expect(submitButton).toBeEnabled(); + await expect(panel).toHaveCount(0); + + const responsePromise = page.waitForResponse( + (resp) => + /\/api\/submit\/$/.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + { timeout: GRADING_TIMEOUT }, + ); + await submitButton.click(); + + await expect(submitButton).toBeDisabled(); + await expect(submitButton.locator('.bouncing-dots')).toBeVisible(); + await expect(panel).toBeVisible(); + await expect(panel.locator('.generating-message')).toHaveText('Running tests...'); + await expect(page.locator(codeFeedback.summaryBar)).toHaveCount(0); + + release(); + expect((await responsePromise).status()).toBe(202); + + await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout: GRADING_TIMEOUT }); + await expect(panel).toHaveCount(0); + await expect(submitButton.locator('.bouncing-dots')).toHaveCount(0); + await expect(submitButton).toBeEnabled(); + await expectSummary(page, 4, 0, null); + }); +}); + +// --------------------------------------------------------------------------- +// B16-B19: graded result for a single submission +// --------------------------------------------------------------------------- + +test.describe('B16-B19 graded feedback and test locking', () => { + test('B16 all tests pass: full counts, success toast, 100% badge', async ({ page }) => { + await openClamp(page, 'b16'); + + await submitAndGrade(page, SOLUTIONS.correct); + + // The toast auto-dismisses after 5 s, so check it first. + await expect( + page.locator('.notification-container .notification.success .notification-message'), + ).toHaveText('Problem 3: 4/4 tests passed'); + await expectSummary(page, 4, 0, null); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + await expectPassingRows(page, [ + 'clamp(5) โ†’ 5', + 'clamp(0) โ†’ 0', + 'clamp(-3) โ†’ 0', + 'clamp(20) โ†’ 10', + ]); + await expectAttemptBadge(page, '1/1', '100'); + }); + + test('B17 an early failure locks every later test, even passing ones', async ({ page }) => { + await openClamp(page, 'b17'); + + await submitAndGrade(page, SOLUTIONS.failsFirst); + + await expectSingleFailingRow(page, 'clamp(5)', '5', '0'); + // The first test failed, so no passing rows come before it. + await expect(page.locator(codeFeedback.passingItem)).toHaveCount(0); + await expect(page.locator('summary.test-group-header.passing')).toHaveCount(0); + // Counts still cover the three locked tests, which pass. + await expectSummary(page, 3, 1, '๐Ÿ”’ 3 more tests locked'); + await expectAttemptBadge(page, '1/1', '75'); + }); + + test('B18 one locked test uses the singular label', async ({ page }) => { + await openClamp(page, 'b18'); + + await submitAndGrade(page, SOLUTIONS.identity); + + await expectSingleFailingRow(page, 'clamp(-3)', '0', '-3'); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0']); + }); + + test('B17b nothing is locked when only the last test fails', async ({ page }) => { + await openClamp(page, 'b17b'); + + await submitAndGrade(page, SOLUTIONS.failsLast); + + await expectSingleFailingRow(page, 'clamp(20)', '10', '20'); + await expectSummary(page, 3, 1, null); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0', 'clamp(-3) โ†’ 0']); + }); + + test('B19 a runtime error shows as an error row and locks the rest', async ({ page }) => { + await openClamp(page, 'b19'); + + await submitAndGrade(page, SOLUTIONS.divideByZero); + + const failing = page.locator(codeFeedback.failingItem); + await expect(failing).toHaveCount(1); + await expect(failing.locator('.test-call')).toHaveText('clamp(5)'); + await expect(failing.locator('.expected')).toHaveText('5'); + await expect(failing.locator('.actual')).toHaveClass(/is-error/); + await expect(failing.locator('.actual')).toHaveText('integer division or modulo by zero'); + await expect(page.locator(codeFeedback.passingItem)).toHaveCount(0); + await expectSummary(page, 0, 4, '๐Ÿ”’ 3 more tests locked'); + }); +}); + +// --------------------------------------------------------------------------- +// B20-B22: multiple attempts and restoring feedback +// --------------------------------------------------------------------------- + +test.describe('B20-B22 attempts and returning later', () => { + test('B20 resubmitting shows the new attempt as 2/2 at 100%', async ({ page }) => { + await openClamp(page, 'b20'); + + await submitAndGrade(page, SOLUTIONS.identity); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectAttemptBadge(page, '1/1', '50'); + + await submitAndGrade(page, SOLUTIONS.correct); + await expectSummary(page, 4, 0, null); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + await expectAttemptBadge(page, '2/2', '100'); + }); + + test('B21 switching attempts restores each attempt\'s truncated feedback', async ({ page }) => { + await openClamp(page, 'b21'); + + await submitAndGrade(page, SOLUTIONS.identity); + await submitAndGrade(page, SOLUTIONS.correct); + await expectAttemptBadge(page, '2/2', '100'); + + const trigger = page.locator(codeFeedback.attemptTrigger); + const panel = page.locator(codeFeedback.attemptPanel); + const attempt = (n: number) => + panel.locator(codeFeedback.attemptItem).filter({ + has: page.locator('.attempt-num', { hasText: new RegExp(`^${n}$`) }), + }); + + await trigger.click(); + await expect(panel).toBeVisible(); + await expect(panel.locator(codeFeedback.attemptItem)).toHaveCount(2); + await expect(attempt(1).locator('.attempt-tests')).toHaveText('2/4'); + await expect(attempt(2).locator('.attempt-tests')).toHaveText('4/4'); + + await attempt(1).click(); + await expect(panel).toBeHidden(); + await expectAttemptBadge(page, '1/2', '50'); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectSingleFailingRow(page, 'clamp(-3)', '0', '-3'); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0']); + + await trigger.click(); + await expect(panel).toBeVisible(); + await attempt(2).click(); + await expect(panel).toBeHidden(); + await expectAttemptBadge(page, '2/2', '100'); + await expectSummary(page, 4, 0, null); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + }); + + test('B22 reloading the page restores the truncated feedback', async ({ page }) => { + await openClamp(page, 'b22'); + + await submitAndGrade(page, SOLUTIONS.identity); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + + const lastSubmission = page.waitForResponse( + (resp) => + /\/api\/last-submission\/e2e-probeable-locked\/$/.test(new URL(resp.url()).pathname) && + resp.request().method() === 'GET', + ); + await page.reload(); + expect((await lastSubmission).status()).toBe(200); + + await expect(page.locator(codeFeedback.summaryBar)).toBeVisible({ timeout: 15_000 }); + await expectSummary(page, 2, 2, '๐Ÿ”’ 1 more test locked'); + await expectSingleFailingRow(page, 'clamp(-3)', '0', '-3'); + await expectPassingRows(page, ['clamp(5) โ†’ 5', 'clamp(0) โ†’ 0']); + await expectAttemptBadge(page, '1/1', '50'); + }); +}); + +// --------------------------------------------------------------------------- +// B23: debugger on a failing row +// --------------------------------------------------------------------------- + +test.describe('B23 debugger', () => { + test('B23 the debug button opens and closes the step-by-step debugger', async ({ page }) => { + // Never load the external Python Tutor page; serve an empty document. + await page.route('https://pythontutor.com/**', (route) => + route.fulfill({ status: 200, contentType: 'text/html', body: '' }), + ); + + await openClamp(page, 'b23'); + await submitAndGrade(page, SOLUTIONS.identity); + + const failing = page.locator(codeFeedback.failingItem); + await expect(failing).toHaveCount(1); + + const modal = page.locator('.modal-overlay[role="dialog"]').filter({ + has: page.locator('#modal-title', { hasText: 'Step-by-Step Debugger' }), + }); + await expect(modal).toHaveCount(0); + + await failing.locator(codeFeedback.debugButton).click(); + await expect(modal).toBeVisible(); + const iframe = modal.locator('iframe.debugger-iframe'); + await expect(iframe).toHaveAttribute('src', /^https:\/\/pythontutor\.com\//); + const src = (await iframe.getAttribute('src')) ?? ''; + expect(decodeURIComponent(src.replace(/\+/g, ' '))).toContain('clamp(-3)'); + + await modal.locator('.close-button').click(); + await expect(modal).toHaveCount(0); + }); +}); diff --git a/e2e/student/probeable-probing.spec.ts b/e2e/student/probeable-probing.spec.ts new file mode 100644 index 0000000..cd0ed0f --- /dev/null +++ b/e2e/student/probeable-probing.spec.ts @@ -0,0 +1,361 @@ +/** + * E2E Tests: probing the oracle on probeable problems (workflow group A). + * + * Problem set e2e-probeable (seed_e2e_data._seed_probeable_workflows): + * p=2 e2e-probeable-locked clamp(n: int) -> max(0, min(n, 10)), explore mode + * p=3 e2e-probeable-errors recip(x: int) -> 10 // x, block mode, 3 probes + * p=4 e2e-probeable-hidden recip(x) with the signature hidden, explore mode + * + * Probe budgets and history live in Redis per (problem, user) with no reset, + * so every test runs as a fresh student. + */ + +import { test, expect, Page, Request } from '@playwright/test'; +import { apiAsUser, navigateAsFreshStudent, FreshUser } from '../helpers/users'; +import { probe } from '../helpers/selectors'; + +const SET_PATH = '/courses/CS101-2024/problem-set/e2e-probeable'; +const PROBE_POST_PATH = /^\/api\/problems\/[^/]+\/probe\/$/; + +function isProbePost(req: Request): boolean { + return req.method() === 'POST' && PROBE_POST_PATH.test(new URL(req.url()).pathname); +} + +/** Open problem `index` of e2e-probeable as a fresh student and wait for probe status. */ +async function openAsFreshStudent( + page: Page, + label: string, + index: number, + functionName: string, +): Promise { + const user = await navigateAsFreshStudent(page, label, `${SET_PATH}?p=${index}`); + await expect(page.locator(probe.panel)).toBeVisible(); + await expect(page.locator('.probe-panel .fn-name')).toHaveText(functionName); + // The badge renders once /probe/status/ has loaded + await expect(page.locator(probe.badge)).toBeVisible(); + return user; +} + +/** + * Fill one parameter, trigger a probe (button click or Enter in the field), + * and return the probe response body. Fails unless the API answers 200. + */ +async function probeWith( + page: Page, + param: string, + value: string, + via: 'click' | 'enter' = 'click', +): Promise { + const input = page.locator(probe.paramInput(param)); + await input.fill(value); + const responsePromise = page.waitForResponse((resp) => isProbePost(resp.request())); + if (via === 'enter') { + await input.press('Enter'); + } else { + await page.locator(probe.button).click(); + } + const response = await responsePromise; + expect(response.status()).toBe(200); + return response.json(); +} + +test.describe('A Probing the oracle', () => { + test('A1 first probe shows the result, the explore badge and History (1)', async ({ page }) => { + await openAsFreshStudent(page, 'a1-first-probe', 2, 'clamp'); + + const input = page.locator(probe.paramInput('n')); + const result = page.locator(probe.result); + const button = page.locator(probe.button); + + // Fresh state: empty input, no result, no history, unlimited badge + await expect(input).toHaveValue(''); + await expect(input).toHaveAttribute('placeholder', 'int'); + await expect(result).toHaveText('?'); + await expect(result).not.toHaveClass(/has-result/); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); + // No value yet: nothing to probe + await expect(button).toBeDisabled(); + + const body = await probeWith(page, 'n', '7'); + expect(body.success).toBe(true); + expect(body.result).toBe(7); + expect(body.probe_status.mode).toBe('explore'); + + await expect(result).toHaveText('7'); + await expect(result).toHaveClass(/has-result/); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + await expect(page.locator(probe.error)).toHaveCount(0); + + // A new value brings the Probe button back with the unlimited badge + await input.fill('8'); + await expect(button).toBeEnabled(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + }); + + test('A2 repeat probe with a new input (Enter key) replaces the result', async ({ page }) => { + await openAsFreshStudent(page, 'a2-repeat-probe', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await expect(page.locator(probe.result)).toHaveText('7'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + // Enter in the field triggers the probe + const body = await probeWith(page, 'n', '20', 'enter'); + expect(body.success).toBe(true); + expect(body.result).toBe(10); + + await expect(page.locator(probe.result)).toHaveText('10'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + }); + + test('A3 duplicate probe returns the cached result without a request', async ({ page }) => { + await openAsFreshStudent(page, 'a3-duplicate-probe', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await probeWith(page, 'n', '20'); + await expect(page.locator(probe.result)).toHaveText('10'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts.push(req.postData() ?? ''); + } + }); + + const input = page.locator(probe.paramInput('n')); + const result = page.locator(probe.result); + const button = page.locator(probe.button); + + await input.fill('7'); + await expect(result).toHaveClass(/is-cached/); + await expect(result).toHaveText('7'); + await expect(button).toHaveClass(/is-duplicate/); + await expect(button).toContainText('Already probed'); + await expect(button).toBeDisabled(); + await expect(page.locator(probe.badge)).toHaveCount(0); + + // Enter in the field is the other way to probe; it must not re-send either + await input.press('Enter'); + + // A real probe afterwards orders the request stream: if the duplicate had + // been sent, it would be recorded before this one. + const body = await probeWith(page, 'n', '3'); + expect(body.result).toBe(3); + expect(probePosts).toEqual([JSON.stringify({ input: { n: 3 } })]); + await expect(page.locator(probe.historyButton)).toHaveText('History (3)'); + }); + + test('A4 invalid type shows a field error and blocks probing until fixed', async ({ page }) => { + await openAsFreshStudent(page, 'a4-invalid-type', 2, 'clamp'); + + let probePosts = 0; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts += 1; + } + }); + + const input = page.locator(probe.paramInput('n')); + const button = page.locator(probe.button); + const fieldError = page.locator(probe.fieldError('n')); + + await input.fill('abc'); + await expect(fieldError).toHaveText('n: Must be a whole number (e.g. 5)'); + await expect(input).toHaveClass(/has-error/); + await expect(input).toHaveAttribute('aria-invalid', 'true'); + await expect(input).toHaveAttribute('aria-describedby', 'probe-err-n'); + await expect(button).toBeDisabled(); + // Enter with an invalid value sends nothing + await input.press('Enter'); + + await input.fill('3'); + await expect(fieldError).toHaveCount(0); + await expect(input).not.toHaveClass(/has-error/); + await expect(button).toBeEnabled(); + + const body = await probeWith(page, 'n', '3'); + expect(body.result).toBe(3); + expect(probePosts).toBe(1); + await expect(page.locator(probe.result)).toHaveText('3'); + }); + + test('A5 history modal lists every probe, most recent first, and closes', async ({ page }) => { + await openAsFreshStudent(page, 'a5-history-modal', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await probeWith(page, 'n', '20'); + await probeWith(page, 'n', '-3'); + await expect(page.locator(probe.result)).toHaveText('0'); + + const historyButton = page.locator(probe.historyButton); + await expect(historyButton).toHaveText('History (3)'); + await historyButton.click(); + + const modal = page.locator(probe.historyModal); + await expect(modal).toBeVisible(); + await expect(modal.locator('#history-modal-title')).toHaveText('Probe History'); + + const rows = page.locator(probe.historyRow); + await expect(rows).toHaveCount(3); + await expect(rows.locator('.modal-index')).toHaveText(['#3', '#2', '#1']); + await expect(rows.locator('.modal-call')).toHaveText(['clamp(-3)', 'clamp(20)', 'clamp(7)']); + await expect(rows.locator('.modal-result')).toHaveText(['0', '10', '7']); + + await page.locator(probe.close).click(); + await expect(page.locator('.modal-overlay')).toHaveCount(0); + await expect(historyButton).toHaveText('History (3)'); + }); + + test('A6 reload restores the probe history from the server', async ({ page }) => { + const user = await openAsFreshStudent(page, 'a6-reload', 2, 'clamp'); + + await probeWith(page, 'n', '7'); + await probeWith(page, 'n', '20'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + + const server = await apiAsUser( + page, + user, + 'GET', + '/api/problems/e2e-probeable-locked/probe/history/', + ); + expect(server.status).toBe(200); + expect(server.data.history.map((e: any) => [e.input, e.output])).toEqual([ + [{ n: 20 }, 10], + [{ n: 7 }, 7], + ]); + + const historyLoaded = page.waitForResponse( + (resp) => + new URL(resp.url()).pathname === '/api/problems/e2e-probeable-locked/probe/history/' && + resp.status() === 200, + ); + await page.reload(); + await historyLoaded; + await expect(page.locator(probe.panel)).toBeVisible(); + + // Both probes restored from the server, the latest result shown, inputs cleared + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + await expect(page.locator(probe.result)).toHaveText('10'); + await expect(page.locator(probe.paramInput('n'))).toHaveValue(''); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + + await page.locator(probe.historyButton).click(); + const rows = page.locator(probe.historyRow); + await expect(rows.locator('.modal-call')).toHaveText(['clamp(20)', 'clamp(7)']); + await expect(rows.locator('.modal-result')).toHaveText(['10', '7']); + await page.locator(probe.close).click(); + + // Re-entering a value probed before the reload is still recognised + await page.locator(probe.paramInput('n')).fill('7'); + await expect(page.locator(probe.result)).toHaveClass(/is-cached/); + await expect(page.locator(probe.result)).toHaveText('7'); + await expect(page.locator(probe.button)).toContainText('Already probed'); + }); + + test('A7 explore mode never runs out of probes', async ({ page }) => { + await openAsFreshStudent(page, 'a7-explore', 2, 'clamp'); + + const badge = page.locator(probe.badge); + await expect(badge).toHaveText('ยท โˆž'); + await expect(badge).toHaveClass(/status-explore/); + + for (let n = 1; n <= 15; n += 1) { + const body = await probeWith(page, 'n', String(n)); + expect(body.success).toBe(true); + expect(body.result).toBe(Math.min(n, 10)); + expect(body.probe_status.mode).toBe('explore'); + expect(body.probe_status.can_probe).toBe(true); + await expect(page.locator(probe.result)).toHaveText(String(Math.min(n, 10))); + } + + await expect(page.locator(probe.historyButton)).toHaveText('History (15)'); + + const input = page.locator(probe.paramInput('n')); + await input.fill('16'); + await expect(input).toBeEnabled(); + await expect(page.locator(probe.button)).toBeEnabled(); + await expect(badge).toHaveText('ยท โˆž'); + await expect(badge).toHaveClass(/status-explore/); + await expect(page.locator(probe.hint)).toHaveCount(0); + + const body = await probeWith(page, 'n', '16'); + expect(body.success).toBe(true); + expect(body.result).toBe(10); + await expect(page.locator(probe.historyButton)).toHaveText('History (16)'); + }); + + test('A10 hidden signature hides types and sanitises oracle errors', async ({ page }) => { + const user = await openAsFreshStudent(page, 'a10-hidden', 4, 'recip'); + + const input = page.locator(probe.paramInput('x')); + await expect(page.locator('.probe-panel .param-name')).toHaveText(['x']); + await expect(input).toHaveAttribute('placeholder', 'value โ€” quote text: "abc"'); + await expect(input).toHaveAttribute('aria-label', 'Enter value for x'); + + // The types never reach the browser: the problem payload the page loads + // carries no signature, untyped parameters and no oracle source. + const { status, data } = await apiAsUser(page, user, 'GET', '/api/problem-sets/e2e-probeable/'); + expect(status).toBe(200); + const hidden = data.problems.find((p: { slug: string }) => p.slug === 'e2e-probeable-hidden'); + expect(hidden.function_signature).toBeNull(); + expect(hidden.probe_config.function_signature).toBeNull(); + expect(hidden.probe_config.parameters).toEqual([{ name: 'x', type: '' }]); + expect(hidden.reference_solution).toBeNull(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + + const failed = await probeWith(page, 'x', '0'); + expect(failed.success).toBe(false); + const error = page.locator(probe.error); + await expect(error).toHaveText('These inputs could not be evaluated. Try different values.'); + await expect(error).not.toContainText('division'); + await expect(page.locator(probe.result)).toHaveText('?'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + + const ok = await probeWith(page, 'x', '5'); + expect(ok.success).toBe(true); + expect(ok.result).toBe(2); + await expect(page.locator(probe.result)).toHaveText('2'); + await expect(error).toHaveCount(0); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + }); + + test('A11 visible-signature oracle error is shown and not charged', async ({ page }) => { + await openAsFreshStudent(page, 'a11-oracle-error', 3, 'recip'); + + const badge = page.locator(probe.badge); + await expect(badge).toHaveText('ยท 3 left'); + await expect(badge).toHaveClass(/status-available/); + + const failed = await probeWith(page, 'x', '0'); + expect(failed.success).toBe(false); + expect(failed.probe_status.remaining).toBe(3); + // Exactly the exception text, nothing else (no traceback, no type name). + // The oracle runs under the server's interpreter (oracle_sandbox.py uses + // sys.executable): Python <= 3.13 says "integer division or modulo by + // zero" for `10 // 0`, 3.14 says "division by zero". + await expect(page.locator(probe.error)).toHaveText( + /^(integer division or modulo|division) by zero$/, + ); + await expect(badge).toHaveText('ยท 3 left'); + await expect(page.locator(probe.result)).toHaveText('?'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + + const ok = await probeWith(page, 'x', '5'); + expect(ok.success).toBe(true); + expect(ok.result).toBe(2); + expect(ok.probe_status.remaining).toBe(2); + await expect(page.locator(probe.result)).toHaveText('2'); + await expect(page.locator(probe.error)).toHaveCount(0); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + // The input still holds the probed value (duplicate state hides the badge); + // a new value shows the charged budget + await page.locator(probe.paramInput('x')).fill('6'); + await expect(badge).toHaveText('ยท 2 left'); + await expect(badge).toHaveClass(/status-available/); + }); +}); diff --git a/e2e/student/probeable-spec.spec.ts b/e2e/student/probeable-spec.spec.ts new file mode 100644 index 0000000..63d38e2 --- /dev/null +++ b/e2e/student/probeable-spec.spec.ts @@ -0,0 +1,250 @@ +/** + * E2E: Probeable Spec workflows (C24a-C24d). + * + * Problem: e2e-probeable-spec-open, index 5 of the e2e-probeable set. + * def square(x: int) -> int, reference `return x * x`, probe_mode explore. + * Test cases: square(3) = 9 (sample), square(-2) = 4 (hidden). + * segmentation_config {"enabled": False}, so a submission whose tests all + * pass is graded 'complete' (ProbeableSpecHandler: segmentation disabled -> + * complete) with no abstraction step. + * + * Grading in the E2E stack: CELERY_EAGER runs the EiPL pipeline inside + * POST /api/submit/, and USE_MOCK_OPENAI returns the reference solution as + * all 5 LLM variations, so every variation passes both tests. + * + * Fresh users have no AI processing consent. probeable_spec submissions are + * not consent-gated yet (#161); submit tests grant consent through the API up + * front so they stay valid once that gate exists. + * + * Every test runs as its own fresh student: probe budgets/history live in + * Redis per (problem, user) with no reset, and submissions accumulate. + */ + +import { test, expect, Page } from '@playwright/test'; +import { setAceEditorValue } from '../helpers/ace-editor'; +import { probe, feedback, problemSet } from '../helpers/selectors'; +import { + FreshUser, + apiAsUser, + enrollUser, + freshStudent, + grantAiConsent, + injectUser, +} from '../helpers/users'; + +const PROBLEM_SET = 'e2e-probeable'; +const PROBLEM_SLUG = 'e2e-probeable-spec-open'; +const SPEC_PATH = `/courses/CS101-2024/problem-set/${PROBLEM_SET}?p=5`; +const GRADE_TIMEOUT = 90_000; + +/** 60 characters of plain-language description. */ +const EXPLANATION = 'Returns the given whole number multiplied by itself squared.'; + +const SUBMIT_URL = /\/api\/submit\/$/; +const PROBE_URL = new RegExp(`/api/problems/${PROBLEM_SLUG}/probe/$`); + +/** + * Create a fresh student, enroll, optionally grant AI consent, and open the + * spec problem. Consent must be granted before navigation so the first submit + * is not intercepted by the consent flow. + */ +async function openSpecProblem( + page: Page, + label: string, + { aiConsent }: { aiConsent: boolean }, +): Promise { + const user = freshStudent(label); + await injectUser(page, user); + await enrollUser(page, user); + if (aiConsent) { + await grantAiConsent(page, user); + } + await page.goto(SPEC_PATH, { waitUntil: 'networkidle' }); + await expect(page.locator('.probeable-spec-input')).toBeVisible({ timeout: 15_000 }); + return user; +} + +/** Probe square(x=value) and wait for the oracle response. */ +async function probeX(page: Page, value: string): Promise { + await page.locator(probe.paramInput('x')).fill(value); + const button = page.locator(probe.button); + await expect(button).toBeEnabled(); + const responsePromise = page.waitForResponse( + (resp) => PROBE_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + ); + await button.click(); + const response = await responsePromise; + expect(response.status()).toBe(200); +} + +/** Put the explanation in #promptEditor, submit, and require a 2xx. */ +async function submitExplanation(page: Page, text: string): Promise { + await setAceEditorValue(page, problemSet.promptEditor, text); + const submitButton = page.locator(problemSet.submitButton); + await expect(submitButton).toBeEnabled(); + + const responsePromise = page.waitForResponse( + (resp) => SUBMIT_URL.test(new URL(resp.url()).pathname) && resp.request().method() === 'POST', + { timeout: GRADE_TIMEOUT }, + ); + await submitButton.click(); + const response = await responsePromise; + expect(response.ok(), `POST /api/submit/ returned ${response.status()}`).toBe(true); +} + +test.describe('C24 Probeable Spec', () => { + test('C24a probe on a spec problem shows oracle results and history', async ({ page }) => { + await openSpecProblem(page, 'c24a', { aiConsent: false }); + + const panel = page.locator('.probeable-spec-input ' + probe.panel); + await expect(panel).toBeVisible(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); + await expect(page.locator(probe.result)).toHaveText('?'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + + await probeX(page, '3'); + await expect(page.locator(probe.result)).toHaveText('9'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + await probeX(page, '-2'); + await expect(page.locator(probe.result)).toHaveText('4'); + await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); + // The input still holds the last probed value, so the button reports it + // as already probed (no badge while duplicate). + await expect(page.locator(probe.button)).toHaveText(/Already probed/); + await expect(page.locator(probe.button)).toHaveClass(/is-duplicate/); + await expect(page.locator(probe.button)).toBeDisabled(); + // A new value re-enables probing; explore mode keeps the badge unlimited. + await page.locator(probe.paramInput('x')).fill('7'); + await expect(page.locator(probe.button)).toBeEnabled(); + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); + + await page.locator(probe.historyButton).click(); + await expect(page.locator(probe.historyModal)).toBeVisible(); + const rows = page.locator(probe.historyRow); + await expect(rows).toHaveCount(2); + // Newest first. + await expect(rows.nth(0).locator('.modal-index')).toHaveText('#2'); + await expect(rows.nth(0).locator('.modal-call')).toHaveText('square(-2)'); + await expect(rows.nth(0).locator('.modal-result')).toHaveText('4'); + await expect(rows.nth(1).locator('.modal-index')).toHaveText('#1'); + await expect(rows.nth(1).locator('.modal-call')).toHaveText('square(3)'); + await expect(rows.nth(1).locator('.modal-result')).toHaveText('9'); + await page.locator(probe.close).click(); + await expect(page.locator(probe.historyModal)).toHaveCount(0); + }); + + test('C24b explanation submit grades 100% with both tests shown', async ({ page }) => { + // One mock-LLM grading: 5 variations x 2 tests in the Docker sandbox. + test.setTimeout(GRADE_TIMEOUT + 30_000); + expect(EXPLANATION.length).toBe(60); + await openSpecProblem(page, 'c24b', { aiConsent: true }); + + await submitExplanation(page, EXPLANATION); + + // All 5 mock variations are the reference solution and segmentation is + // disabled for this problem, so the grade is complete at 100%. + await expect(page.locator('.notification-container .notification.success')).toContainText( + 'Problem 6: Variations: 5/5 passing', + { timeout: GRADE_TIMEOUT }, + ); + + const container = page.locator(feedback.container); + await expect(container.locator(feedback.header)).toBeVisible({ timeout: GRADE_TIMEOUT }); + await expect(container.locator('.attempt-text')).toHaveText('1/1'); + await expect(container.locator(feedback.scoreBadge)).toHaveText('100%'); + await expect(container.locator('.explanation-text')).toHaveText(EXPLANATION); + + const correctnessCard = container.locator(feedback.metricCard).first(); + await expect(correctnessCard.locator('.status-metric')).toHaveText('5/5 pass'); + await expect(correctnessCard.locator('.status-label')).toHaveText('Clear'); + + // Test rows live in the correctness modal. probeable_spec is outside the + // #165 truncation rule, so every variation lists both tests, the hidden + // one included. + await correctnessCard.click(); + const modal = page.locator('.modal-content.correctness-modal'); + await expect(modal).toBeVisible(); + await expect(modal.locator('.badge-level')).toHaveText('5/5 Versions Pass'); + await expect(modal.locator('.version-pill')).toHaveCount(5); + + for (let i = 0; i < 5; i++) { + await modal.locator(`#version-tab-${i}`).click(); + const panel = modal.locator(`#version-panel-${i}`); + await expect(panel).toBeVisible(); + await expect(panel.locator('.details-tests .section-meta')).toHaveText('2/2'); + const tests = panel.locator('.tests-list .test-item'); + await expect(tests).toHaveCount(2); + await expect(panel.locator('.tests-list .test-item.passing')).toHaveCount(2); + await expect(tests.nth(0).locator('.test-call')).toHaveText('square(3)'); + await expect(tests.nth(1).locator('.test-call')).toHaveText('square(-2)'); + } + }); + + test('C24c description length bounds gate the submit button', async ({ page }) => { + await openSpecProblem(page, 'c24c', { aiConsent: false }); + const submitButton = page.locator(problemSet.submitButton); + + // Empty on load. + await expect(submitButton).toBeDisabled(); + + await setAceEditorValue(page, problemSet.promptEditor, 'short'); + await expect(submitButton).toBeDisabled(); + + // Minimum is 10 characters after trimming. + await setAceEditorValue(page, problemSet.promptEditor, ' 123456789 '); + await expect(submitButton).toBeDisabled(); + await setAceEditorValue(page, problemSet.promptEditor, '1234567890'); + await expect(submitButton).toBeEnabled(); + + // Maximum is 1000 characters. + await setAceEditorValue(page, problemSet.promptEditor, 'a'.repeat(1000)); + await expect(submitButton).toBeEnabled(); + await setAceEditorValue(page, problemSet.promptEditor, 'a'.repeat(1001)); + await expect(submitButton).toBeDisabled(); + }); + + test('C24d resubmitting shows 2 attempts', async ({ page }) => { + // Two mock-LLM gradings, each 5 variations x 2 tests in the Docker sandbox. + test.setTimeout(2 * GRADE_TIMEOUT + 30_000); + const user = await openSpecProblem(page, 'c24d', { aiConsent: true }); + const container = page.locator(feedback.container); + + await submitExplanation(page, EXPLANATION); + await expect(container.locator('.attempt-text')).toHaveText('1/1', { timeout: GRADE_TIMEOUT }); + + await submitExplanation(page, 'It multiplies the integer argument by itself and returns it.'); + await expect(container.locator('.attempt-text')).toHaveText('2/2', { timeout: GRADE_TIMEOUT }); + await expect(container.locator(feedback.scoreBadge)).toHaveText('100%'); + + await container.locator(feedback.attemptSelector).click(); + const panel = container.locator('.attempt-dropdown-panel[role="menu"]'); + await expect(panel).toBeVisible(); + const items = panel.locator('.attempt-item-minimal'); + await expect(items).toHaveCount(2); + await expect(items.locator('.attempt-num')).toHaveText(['2', '1']); + await expect(items.locator('.attempt-score-minimal')).toHaveText(['100%', '100%']); + + const { status, data } = await apiAsUser( + page, + user, + 'GET', + `/api/submissions/history/${PROBLEM_SLUG}/?problem_set_slug=${PROBLEM_SET}&course_id=CS101-2024`, + ); + expect(status).toBe(200); + expect(data.submissions).toHaveLength(2); + for (const submission of data.submissions) { + expect(submission.score).toBe(100); + expect(submission.completion_status).toBe('complete'); + expect(submission.variations_count).toBe(5); + // History counts test executions across all 5 variations: 5 x 2 tests. + expect(submission.tests_passed).toBe(10); + expect(submission.total_tests).toBe(10); + } + + // The dropdown shows each attempt's tests_passed/total_tests from history. + await expect(items.locator('.attempt-tests')).toHaveText(['10/10', '10/10']); + }); +}); diff --git a/e2e/student/submission-probeable.spec.ts b/e2e/student/submission-probeable.spec.ts index aeca7af..ce3df4e 100644 --- a/e2e/student/submission-probeable.spec.ts +++ b/e2e/student/submission-probeable.spec.ts @@ -6,63 +6,61 @@ * * Probeable Code has two phases: * 1. Probe phase: students query the oracle with inputs to discover behavior + * (POST /api/problems//probe/ answers synchronously) * 2. Code phase: students write implementation code and submit * - * Both probe execution and final submission are ASYNCHRONOUS. + * The E2E stack runs Celery eagerly (CELERY_EAGER=true), so POST /api/submit/ + * grades the submission synchronously in the local Docker sandbox + * (DOCKER_POOL_ENABLED=false) and the SSE stream reports completion at once. + * + * Probe history lives in Redis per (problem, user) with no reset, so tests + * that probe or submit run as a fresh student (e2e/helpers/users.ts). * * Seeded data: * - CS101-2024 course with e2e-code problem set (soft past deadline) - * - e2e-probeable-code-1: mystery(lst) -- actually sorted(lst) + * - e2e-probeable-code-1: mystery(lst) -- actually sorted(lst) (e2e-code, p=1) * function_signature: "def mystery(lst: list[int]) -> list[int]" - * probe_mode: "explore", max_probes: 10 + * probe_mode: "explore", one test case: mystery([3, 1, 2]) == [1, 2, 3] * - e2e-probeable-code-typed: double(n) -- n * 2 (e2e-code, p=2) * function_signature: "def double(n: int) -> int" * used to check client-side probe input type validation (#135) * - e2e-refute-1: f(x) = x * 2, claim "always returns positive" * (in e2e-mixed which has hard past deadline -- locked) * - * Prerequisites: - * 1. Django dev server on :8000 with USE_MOCK_FIREBASE=true - * 2. Celery worker running (with Docker-in-Docker for code execution) - * 3. Vite dev server on :5173 - * 4. python manage.py create_test_users && python manage.py seed_e2e_data + * Prerequisites (same as .github/workflows/e2e.yml): + * 1. Django dev server on :8000 with USE_MOCK_FIREBASE=true, + * CELERY_EAGER=true, DOCKER_POOL_ENABLED=false, and the + * purplex/python-sandbox image available to the local Docker daemon + * 2. Vite dev server on :5173 + * 3. python manage.py create_test_users && python manage.py seed_e2e_data */ import { test, expect, Page } from '@playwright/test'; import { navigateAs } from '../helpers/navigation'; import { apiAs } from '../helpers/api'; +import { apiAsUser, enrollUser, freshStudent, injectUser, navigateAsFreshStudent } from '../helpers/users'; +import { submitCodeAndWait } from '../helpers/submission'; +import { codeFeedback, probe } from '../helpers/selectors'; -const ASYNC_TIMEOUT = 45000; +const PROBEABLE_PATH = '/courses/CS101-2024/problem-set/e2e-code?p=1'; +const TYPED_PROBEABLE_PATH = '/courses/CS101-2024/problem-set/e2e-code?p=2'; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- -/** Navigate to e2e-code and select e2e-probeable-code-1. */ -async function goToProbeableProblem(page: Page) { - await navigateAs( - page, - 'student', - '/courses/CS101-2024/problem-set/e2e-code?p=1', - ); - // Wait for the probeable code input to appear +/** Wait for the probeable code input to render. */ +async function waitForProbeableInput(page: Page) { await page.locator('.probeable-code-input, .probe-panel, #codeEditor').first().waitFor({ state: 'visible', timeout: 15000, }); } -/** Navigate to e2e-code and select e2e-probeable-code-typed. */ -async function goToTypedProbeableProblem(page: Page) { - await navigateAs( - page, - 'student', - '/courses/CS101-2024/problem-set/e2e-code?p=2', - ); - await page.locator('.probeable-code-input, .probe-panel, #codeEditor').first().waitFor({ - state: 'visible', - timeout: 15000, - }); +/** Navigate to e2e-code and select e2e-probeable-code-1 as the seeded student. */ +async function goToProbeableProblem(page: Page) { + await navigateAs(page, 'student', PROBEABLE_PATH); + await waitForProbeableInput(page); } /** Type code into the Probeable Code Ace editor. */ @@ -92,26 +90,23 @@ test.describe('Probeable Code Submission', () => { }); test('probe oracle via API returns sorted result', async ({ page }) => { - // Direct API test for probe functionality -- more reliable than UI probe - await navigateAs(page, 'student', '/home'); + const user = freshStudent('probe-api'); + await injectUser(page, user); + await enrollUser(page, user); - const { status, data } = await apiAs( + const { status, data } = await apiAsUser( page, - 'student', + user, 'POST', '/api/problems/e2e-probeable-code-1/probe/', - { input: { lst: [3, 1, 2] } }, + { input: { lst: [3, 1, 2] }, course_id: 'CS101-2024' }, ); - if (status === 200) { - // Probe succeeded -- oracle should return sorted list - expect(data.success).toBeTruthy(); - expect(data.result).toEqual([1, 2, 3]); - } else { - // 401/403 means mock auth is not reaching the API, or probe requires - // the student to be enrolled and have the right problem set context - expect([401, 403, 400]).toContain(status); - } + expect(status).toBe(200); + expect(data.success).toBe(true); + expect(data.error).toBeNull(); + expect(data.result).toEqual([1, 2, 3]); + expect(data.probe_status.mode).toBe('explore'); }); test('code editor is available for writing implementation', async ({ page }) => { @@ -126,25 +121,24 @@ test.describe('Probeable Code Submission', () => { await expect(submitBtn).toBeVisible(); }); - test('submitting code solution triggers async evaluation', async ({ page }) => { - await goToProbeableProblem(page); - - // Write a solution (the mystery function is sorted()) - const solution = 'def mystery(lst):\n return sorted(lst)'; - await typeInCodeEditor(page, solution); - - const submitBtn = page.locator('#submitButton'); - await expect(submitBtn).toBeEnabled({ timeout: 5000 }); - await submitBtn.click(); - - // Should enter loading state - await expect(submitBtn).toBeDisabled({ timeout: 5000 }); - - // Wait for results (async via SSE) - const feedbackOrLoading = page.locator( - '.feedback-container .feedback-header, .feedback-container .generating-feedback-panel', - ).first(); - await feedbackOrLoading.waitFor({ state: 'visible', timeout: ASYNC_TIMEOUT }); + test('submitting a correct solution grades it and shows the passing feedback', async ({ page }) => { + // submitCodeAndWait waits up to 60s for grading in the Docker sandbox. + test.setTimeout(90_000); + await navigateAsFreshStudent(page, 'probeable-submit', PROBEABLE_PATH); + await waitForProbeableInput(page); + + // The mystery function is sorted(); its single test case must pass + await submitCodeAndWait(page, 'def mystery(lst):\n return sorted(lst)'); + + await expect(page.locator(codeFeedback.passing)).toHaveText('โœ“ 1 Passing'); + await expect(page.locator(codeFeedback.failing)).toHaveText('โœ— 0 Failing'); + await expect(page.locator(codeFeedback.locked)).toHaveCount(0); + await expect(page.locator(codeFeedback.failingItem)).toHaveCount(0); + await expect(page.locator(`${codeFeedback.attemptTrigger} .attempt-text`)).toHaveText('1/1'); + const scoreBadge = page.locator(`${codeFeedback.attemptTrigger} .attempt-score-badge`); + await expect(scoreBadge).toHaveText('โœ“ 100%'); + await expect(scoreBadge).toHaveAttribute('aria-label', 'Score: 100 percent'); + await expect(scoreBadge).toHaveClass(/score-perfect/); }); test('submit button requires minimum code length', async ({ page }) => { @@ -159,40 +153,45 @@ test.describe('Probeable Code Submission', () => { }); test('probe history populates after executing probes via UI', async ({ page }) => { - await goToProbeableProblem(page); + await navigateAsFreshStudent(page, 'probe-history-ui', PROBEABLE_PATH); + await expect(page.locator(probe.panel)).toBeVisible(); + // The badge renders once /probe/status/ has loaded + await expect(page.locator(probe.badge)).toHaveText('ยท โˆž'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); - // The probe panel has input fields for function parameters // e2e-probeable-code-1 has signature: mystery(lst: list[int]) -> list[int] - const paramInput = page.locator('.param-input').first(); - if (!(await paramInput.isVisible().catch(() => false))) { - // Probe panel may not have rendered param inputs -- skip gracefully - test.skip(); - return; - } - - await paramInput.fill('[3, 1, 2]'); - - // Find and click the probe/execute button - const probeBtn = page.locator('button').filter({ hasText: /probe|execute|run/i }).first(); - if (!(await probeBtn.isVisible().catch(() => false))) { - test.skip(); - return; - } - - // Probe button may be disabled if Docker execution is unavailable - const isEnabled = await probeBtn.isEnabled().catch(() => false); - if (!isEnabled) { - test.skip(); - return; + const lstInput = page.locator(probe.paramInput('lst')); + const probeBtn = page.locator(probe.button); + const probes: Array<[string, number[]]> = [ + ['[3, 1, 2]', [1, 2, 3]], + ['[9, 4]', [4, 9]], + ]; + + for (const [value, expected] of probes) { + await lstInput.fill(value); + await expect(probeBtn).toBeEnabled(); + const responsePromise = page.waitForResponse( + (resp) => + resp.request().method() === 'POST' && + new URL(resp.url()).pathname === '/api/problems/e2e-probeable-code-1/probe/', + ); + await probeBtn.click(); + const response = await responsePromise; + expect(response.status()).toBe(200); + const body = await response.json(); + expect(body.success).toBe(true); + expect(body.result).toEqual(expected); + await expect(page.locator(probe.result)).toHaveText(JSON.stringify(expected)); } - await probeBtn.click(); + const historyBtn = page.locator(probe.historyButton); + await expect(historyBtn).toHaveText('History (2)'); + await historyBtn.click(); - // Wait for a probe history entry to appear - const historyItem = page.locator('.probe-history-item, .probe-result').first(); - await historyItem.waitFor({ state: 'visible', timeout: 15000 }).catch(() => { - // Probe may have failed -- acceptable in test environments - }); + const rows = page.locator(probe.historyRow); + await expect(rows).toHaveCount(2); + await expect(rows.locator('.modal-call')).toHaveText(['mystery([9,4])', 'mystery([3,1,2])']); + await expect(rows.locator('.modal-result')).toHaveText(['[4,9]', '[1,2,3]']); }); }); @@ -202,25 +201,22 @@ test.describe('Probeable Code Submission', () => { test.describe('Probe Input Type Validation', () => { test('non-integer input for an int parameter shows an error and disables Probe', async ({ page }) => { - await goToTypedProbeableProblem(page); + // Fresh student: a probe history containing the typed value would put the + // button in its 'Already probed' state + await navigateAsFreshStudent(page, 'probe-typed', TYPED_PROBEABLE_PATH); + await waitForProbeableInput(page); // e2e-probeable-code-typed has signature: double(n: int) -> int - const nInput = page.locator('#probe-n'); - if (!(await nInput.isVisible().catch(() => false))) { - test.skip(); - return; - } - if (!(await nInput.isEnabled().catch(() => false))) { - // Probing may be unavailable (e.g. exhausted budget) -- skip gracefully - test.skip(); - return; - } + const nInput = page.locator(probe.paramInput('n')); + await expect(nInput).toBeVisible(); + await expect(nInput).toBeEnabled(); - const fieldErrors = page.locator('.probe-field-errors'); - const probeBtn = page.locator('.probe-btn'); + const fieldErrors = page.locator(probe.fieldErrors); + const probeBtn = page.locator(probe.button); await nInput.fill('abc'); await expect(fieldErrors).toBeVisible(); + await expect(page.locator(probe.fieldError('n'))).toHaveText('n: Must be a whole number (e.g. 5)'); await expect(nInput).toHaveClass(/has-error/); await expect(probeBtn).toBeDisabled(); @@ -243,19 +239,14 @@ test.describe('Refute Problem (Deadline Enforcement)', () => { '/courses/CS101-2024/problem-set/e2e-mixed?p=6', ); - // Wait for the page to load - await page.waitForLoadState('networkidle'); - - // Should see the locked deadline banner or lock icon - const lockedBanner = page.locator('.deadline-locked'); - const lockText = page.getByText(/Submissions Closed|submissions closed/i).first(); - - const isLocked = - (await lockedBanner.isVisible().catch(() => false)) || - (await lockText.isVisible().catch(() => false)); - - // The hard past deadline should block the UI - expect(isLocked).toBeTruthy(); + // The hard past deadline shows the locked banner. + const banner = page.locator('.deadline-banner.deadline-locked'); + await expect(banner).toBeVisible(); + await expect(banner.locator('.deadline-icon')).toHaveText('๐Ÿ”’'); + await expect(banner.locator('.deadline-text strong')).toHaveText('Submissions Closed'); + await expect(banner.locator('.deadline-text')).toHaveText( + /^Submissions Closed โ€” This problem set closed on \S.*$/, + ); }); test('API rejects submission on hard-deadline-locked problem set', async ({ page }) => { @@ -268,12 +259,9 @@ test.describe('Refute Problem (Deadline Enforcement)', () => { raw_input: JSON.stringify({ x: -5 }), }); - // Should be rejected due to hard past deadline - if (status === 200) { - // Some implementations return 200 with an error in the body - expect(data).toBeTruthy(); - } else { - expect([400, 403, 422]).toContain(status); - } + // SubmissionValidationService._check_deadline rejects it before grading. + expect(status).toBe(400); + expect(data.error).toMatch(/^Submission deadline has passed\. This problem set closed on .+\.$/); + expect(data.code).toBe('validation_error'); }); }); From 78bd8ad387ffd39a33a8be1083b22da874c294cd Mon Sep 17 00:00:00 2001 From: David Smith Date: Thu, 15 Oct 2026 19:51:14 -0400 Subject: [PATCH 3/4] Fix four probe-panel and feedback defects the e2e workflows exposed - Pressing Enter on an already-probed input sent a new request and, in block or cooldown mode, spent a probe for a result the student already had. The Enter guard and executeProbe now honour the duplicate state. - After the probe that exhausted the budget the inputs stayed on a duplicate, so the button showed "Already probed" and the "0 left" badge, exhausted tooltip and hint never appeared. The duplicate branch now applies only while probes remain. - A cooldown refill earned by a submission was invisible until a reload: probe status was fetched only on mount. useProbeState re-fetches it when a submission finishes. - Switching to an earlier attempt rendered clamp([-3]) instead of clamp(-3): the call text formatter treated the positional-args list as one value. Extracted to functionCall.ts, following the backend's _format_function_call rules, with tests. - A 400 from /api/submit/ (syntax error) showed a generic "Error" toast because the submit catch branched on error.response, which the service layer never rethrows. submitError.ts reads status and message from either shape; the intended "Invalid Request" warning now shows. Co-Authored-By: Claude Fable 5.1 --- .../activities/inputs/ProbeableCodeInput.vue | 2 +- .../activities/inputs/ProbeableSpecInput.vue | 2 +- .../activities/inputs/shared/ProbePanel.vue | 12 +- .../__tests__/probeBudgetGuards.test.ts | 120 ++++++++++++++++++ .../activities/inputs/shared/useProbeState.ts | 22 +++- .../src/features/problems/ProblemSet.vue | 55 +------- .../problems/__tests__/functionCall.test.ts | 36 ++++++ .../problems/__tests__/submitError.test.ts | 53 ++++++++ .../src/features/problems/functionCall.ts | 92 ++++++++++++++ .../src/features/problems/submitError.ts | 54 ++++++++ .../client/src/services/submissionService.ts | 3 +- 11 files changed, 392 insertions(+), 59 deletions(-) create mode 100644 purplex/client/src/components/activities/inputs/shared/__tests__/probeBudgetGuards.test.ts create mode 100644 purplex/client/src/features/problems/__tests__/functionCall.test.ts create mode 100644 purplex/client/src/features/problems/__tests__/submitError.test.ts create mode 100644 purplex/client/src/features/problems/functionCall.ts create mode 100644 purplex/client/src/features/problems/submitError.ts diff --git a/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue b/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue index 969c569..bf01991 100644 --- a/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue +++ b/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue @@ -155,7 +155,7 @@ const { formatFunctionCall, formatOutput, updateProbeInput, -} = useProbeState(() => props.problem) +} = useProbeState(() => props.problem, { isSubmitting: () => props.disabled }) const { t } = useI18n() diff --git a/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue b/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue index 4d9663b..b2503e3 100644 --- a/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue +++ b/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue @@ -157,7 +157,7 @@ const { formatFunctionCall, formatOutput, updateProbeInput, -} = useProbeState(() => props.problem) +} = useProbeState(() => props.problem, { isSubmitting: () => props.disabled }) const { t } = useI18n() diff --git a/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue b/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue index 68bfaba..2687eaf 100644 --- a/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue +++ b/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue @@ -29,7 +29,7 @@ :aria-describedby="inputErrors[param.name] ? `probe-err-${param.name}` : undefined" :disabled="!canProbe || isExecuting" @input="$emit('update-input', param.name, ($event.target as HTMLInputElement).value)" - @keydown.enter="!isExecuting && canProbe && hasValidInputs && $emit('execute-probe')" + @keydown.enter="!isExecuting && canProbe && hasValidInputs && !isDuplicate && $emit('execute-probe')" > ) @@ -71,13 +71,13 @@ - +