diff --git a/e2e/helpers/probe.ts b/e2e/helpers/probe.ts new file mode 100644 index 0000000..912facd --- /dev/null +++ b/e2e/helpers/probe.ts @@ -0,0 +1,67 @@ +import { expect, Page } from '@playwright/test'; +import { probe } from './selectors'; + +/** + * Probe panel call line (activities/inputs/shared/ProbePanel.vue). + * + * The student edits only the argument list between `name(` and `)`: one + * contenteditable #probe-args holding Python-style arguments, positional or + * keyword (`7`, `n=7`, `["a", "b"], prefix="x"`). + */ + +/** + * Replace the argument list with `text` and wait until the editable holds it. + * + * Locator.fill selects the current contents and inserts the text as one input + * event, which a contenteditable accepts, brackets and quotes included. + */ +export async function fillProbeArgs(page: Page, text: string): Promise { + const line = page.locator(probe.argsLine); + await line.fill(text); + await expect(line).toHaveJSProperty('textContent', text); +} + +/** Text of the result slot after the arrow ('?' until a result). */ +export async function readProbeResult(page: Page): Promise { + return (await page.locator(probe.result).innerText()).trim(); +} + +/** Text of the remaining-probe badge, e.g. '· ∞' or '· 2 left'. */ +export async function readBadge(page: Page): Promise { + return (await page.locator(probe.badge).innerText()).trim(); +} + +/** + * Assert the error shown for one argument. `name` is a parameter name, or + * 'line' for a line-level error (unknown, duplicate, extra or empty argument). + */ +export async function expectArgError(page: Page, name: string, text: string | RegExp): Promise { + await expect(page.locator(probe.fieldError(name))).toHaveText(text); +} + +/** + * Assert whether the argument list can be edited. Locked means + * contenteditable="false" plus aria-disabled, as when probes are exhausted. + */ +export async function expectArgsEditable(page: Page, editable: boolean): Promise { + const line = page.locator(probe.argsLine); + if (editable) { + // The e2e project is Chromium-only, which supports plaintext-only; the 'true' + // fallback for other browsers is covered in Vitest (probeArgsLine.test.ts) + await expect(line).toHaveAttribute('contenteditable', 'plaintext-only'); + await expect(line).toBeEnabled(); + } else { + await expect(line).toHaveAttribute('contenteditable', 'false'); + await expect(line).toHaveAttribute('aria-disabled', 'true'); + await expect(line).toBeDisabled(); + } +} + +/** The placeholder actually painted in the empty call line (CSS :empty::before). */ +export async function readArgsPlaceholder(page: Page): Promise { + return page.locator(probe.argsLine).evaluate((el) => { + const content = getComputedStyle(el, '::before').content; + // 'none' when the line is not empty; otherwise a quoted CSS string + return content === 'none' ? '' : JSON.parse(content); + }); +} diff --git a/e2e/helpers/selectors.ts b/e2e/helpers/selectors.ts index 9190597..b2673b1 100644 --- a/e2e/helpers/selectors.ts +++ b/e2e/helpers/selectors.ts @@ -247,8 +247,16 @@ export const common = { export const probe = { /** Panel root */ panel: '.probe-panel', - /** Input for one parameter (Enter triggers a probe) */ - paramInput: (name: string) => `input#probe-${name}.param-input`, + /** Row holding the call and the History button (wraps on narrow screens) */ + row: '.probe-panel .probe-row', + /** Box around the call line: `name(` + arguments + `)` (.has-error / .is-disabled) */ + callShell: '.probe-panel .call-shell', + /** Editable argument list between the parentheses (Enter probes, Tab fills names) */ + argsLine: '#probe-args', + /** Closing parenthesis after the argument list */ + closeParen: '.probe-panel .call-shell .fn-paren:last-of-type', + /** 'Fill in argument names' button: inserts `name=, name=` */ + fillArgsButton: 'button.fill-args-btn', /** Result slot ('?' until a result; .has-result / .is-cached) */ result: '.probe-panel .fn-result', /** Probe button ('Probe' | 'Probing' | 'Already probed', .is-duplicate) */ @@ -263,6 +271,8 @@ export const probe = { fieldErrors: 'ul.probe-field-errors', /** One parameter's field error */ fieldError: (name: string) => `li#probe-err-${name}`, + /** Line-level error (unknown keyword, duplicate/empty argument, too many arguments) */ + lineError: 'li#probe-err-line', /** 'History (N)' button, rendered only when history is non-empty */ historyButton: '.probe-panel .history-btn', /** History modal (teleported to body) */ diff --git a/e2e/student/probeable-budget.spec.ts b/e2e/student/probeable-budget.spec.ts index 875c554..a15fab0 100644 --- a/e2e/student/probeable-budget.spec.ts +++ b/e2e/student/probeable-budget.spec.ts @@ -17,12 +17,15 @@ * keep their values, so the button switches to "Already probed"; the spec * enters new values before it reads the badge. * - A duplicate input is answered from the client-side history: no request. + * - The arguments are typed into one call line (#probe-args) as `a=1, b=2`; + * once probes are exhausted it is contenteditable="false" + aria-disabled. */ import { test, expect, Page, Response } from '@playwright/test'; import { navigateAsFreshStudent, freshStudent, injectUser, enrollUser } from '../helpers/users'; import { submitCodeAndWait } from '../helpers/submission'; import { probe } from '../helpers/selectors'; +import { expectArgsEditable, fillProbeArgs } from '../helpers/probe'; const SET_URL = '/courses/CS101-2024/problem-set/e2e-probeable'; const BLOCK_URL = `${SET_URL}?p=0`; @@ -37,16 +40,16 @@ function isProbePost(resp: Response, slug: string): boolean { ); } -/** Wait until the probe panel and its add(a, b) inputs are on screen. */ +/** Wait until the probe panel and its add(a, b) call line are on screen. */ async function waitForProbePanel(page: Page): Promise { await expect(page.locator(probe.panel)).toBeVisible({ timeout: 15_000 }); - await expect(page.locator(probe.paramInput('a'))).toBeVisible(); - await expect(page.locator(probe.paramInput('b'))).toBeVisible(); + await expect(page.locator('.probe-panel .fn-name')).toHaveText('add'); + await expect(page.locator(probe.argsLine)).toBeVisible(); } +/** Type add(a=, b=) into the call line. */ async function fillArgs(page: Page, a: string, b: string): Promise { - await page.locator(probe.paramInput('a')).fill(a); - await page.locator(probe.paramInput('b')).fill(b); + await fillProbeArgs(page, `a=${a}, b=${b}`); } /** @@ -128,7 +131,7 @@ test.describe('Probe budget: block mode', () => { await expect(page.locator(probe.result)).toHaveClass(/is-cached/); // Confirming the duplicate from the keyboard must not probe either. - await page.locator(probe.paramInput('b')).press('Enter'); + await page.locator(probe.argsLine).press('Enter'); // The duplicate was free: one probe is still left. await fillArgs(page, '3', '3'); @@ -144,8 +147,7 @@ test.describe('Probe budget: block mode', () => { // Exhausted state without a reload. await expect(page.locator(probe.hint)).toHaveText('No probes remaining.'); await expect(page.locator(probe.button)).toBeDisabled(); - await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); - await expect(page.locator(probe.paramInput('b'))).toBeDisabled(); + await expectArgsEditable(page, false); await expect(page.locator(probe.historyButton)).toHaveText('History (3)'); // Server-side state survives a fresh load (second tab, same student). @@ -157,8 +159,7 @@ test.describe('Probe budget: block mode', () => { await expectBadge(freshTab, 0, 'status-exhausted'); await expect(freshTab.locator(probe.hint)).toHaveText('No probes remaining.'); await expect(freshTab.locator(probe.button)).toBeDisabled(); - await expect(freshTab.locator(probe.paramInput('a'))).toBeDisabled(); - await expect(freshTab.locator(probe.paramInput('b'))).toBeDisabled(); + await expectArgsEditable(freshTab, false); await expect(freshTab.locator(probe.result)).toHaveText('6'); // Earlier results stay readable from the history after exhaustion. @@ -206,9 +207,9 @@ test.describe('Probe budget: block mode', () => { await expect(page.locator(probe.result)).toHaveClass(/is-cached/); await expect(page.locator(probe.result)).toHaveText('3'); - // Enter in a parameter field is the keyboard path to "Probe". On a + // Enter in the call line is the keyboard path to "Probe". On a // duplicate it must do nothing, like the disabled button. - await page.locator(probe.paramInput('b')).press('Enter'); + await page.locator(probe.argsLine).press('Enter'); // Budget is server-side: a fresh load shows whether anything was spent. await page.reload({ waitUntil: 'networkidle' }); @@ -283,8 +284,7 @@ test.describe('Probe budget: cooldown mode', () => { await expect(page.locator(probe.hint)).toHaveText(refillHint); await expect(page.locator(probe.button)).toBeDisabled(); - await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); - await expect(page.locator(probe.paramInput('b'))).toBeDisabled(); + await expectArgsEditable(page, false); // One submission earns the refill. await submitCodeAndWait(page, ADD_SOLUTION); @@ -298,8 +298,7 @@ test.describe('Probe budget: cooldown mode', () => { await waitForProbePanel(freshTab); await expectBadge(freshTab, 2, 'status-available'); await expect(freshTab.locator(probe.hint)).toHaveCount(0); - await expect(freshTab.locator(probe.paramInput('a'))).toBeEnabled(); - await expect(freshTab.locator(probe.paramInput('b'))).toBeEnabled(); + await expectArgsEditable(freshTab, true); } finally { await freshTab.close(); } @@ -307,8 +306,7 @@ test.describe('Probe budget: cooldown mode', () => { // Without a reload, the page the student submitted from must show the // refilled budget too. await expect(page.locator(probe.hint)).toHaveCount(0); - await expect(page.locator(probe.paramInput('a'))).toBeEnabled(); - await expect(page.locator(probe.paramInput('b'))).toBeEnabled(); + await expectArgsEditable(page, true); await fillArgs(page, '3', '3'); await expectBadge(page, 2, 'status-available'); @@ -337,21 +335,21 @@ test.describe('Probe budget: cooldown mode', () => { await expect(page.locator(probe.hint)).toHaveText( 'No probes remaining. Submit 2 more times to unlock more probes.', ); - await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expectArgsEditable(page, false); // First submission: one more is still needed, and the budget stays empty. await submitCodeAndWait(page, ADD_SOLUTION); await expect(page.locator(probe.hint)).toHaveText( 'No probes remaining. Submit 1 more time to unlock more probes.', ); - await expect(page.locator(probe.paramInput('a'))).toBeDisabled(); + await expectArgsEditable(page, false); await expect(page.locator(probe.button)).toBeDisabled(); await expectBadge(page, 0, 'status-exhausted'); // Second submission earns the refill. await submitCodeAndWait(page, ADD_SOLUTION); await expect(page.locator(probe.hint)).toHaveCount(0); - await expect(page.locator(probe.paramInput('a'))).toBeEnabled(); + await expectArgsEditable(page, true); await fillArgs(page, '2', '2'); await expectBadge(page, 1, 'status-available'); diff --git a/e2e/student/probeable-probing.spec.ts b/e2e/student/probeable-probing.spec.ts index cd0ed0f..4aa97be 100644 --- a/e2e/student/probeable-probing.spec.ts +++ b/e2e/student/probeable-probing.spec.ts @@ -5,14 +5,25 @@ * p=2 e2e-probeable-locked clamp(n: int) -> max(0, min(n, 10)), explore mode * p=3 e2e-probeable-errors recip(x: int) -> 10 // x, block mode, 3 probes * p=4 e2e-probeable-hidden recip(x) with the signature hidden, explore mode + * p=7 e2e-probeable-list tag(items: list[str], prefix: str), explore mode * * Probe budgets and history live in Redis per (problem, user) with no reset, * so every test runs as a fresh student. + * + * The probe panel is a call line: `name(` and `)` are fixed text and the + * student types the argument list (#probe-args), positional or keyword. */ import { test, expect, Page, Request } from '@playwright/test'; import { apiAsUser, navigateAsFreshStudent, FreshUser } from '../helpers/users'; import { probe } from '../helpers/selectors'; +import { + expectArgError, + expectArgsEditable, + fillProbeArgs, + readArgsPlaceholder, + readBadge, +} from '../helpers/probe'; const SET_PATH = '/courses/CS101-2024/problem-set/e2e-probeable'; const PROBE_POST_PATH = /^\/api\/problems\/[^/]+\/probe\/$/; @@ -37,20 +48,14 @@ async function openAsFreshStudent( } /** - * Fill one parameter, trigger a probe (button click or Enter in the field), - * and return the probe response body. Fails unless the API answers 200. + * Type the argument list, trigger a probe (button click or Enter in the call + * line), and return the probe response body. Fails unless the API answers 200. */ -async function probeWith( - page: Page, - param: string, - value: string, - via: 'click' | 'enter' = 'click', -): Promise { - const input = page.locator(probe.paramInput(param)); - await input.fill(value); +async function probeWith(page: Page, args: string, via: 'click' | 'enter' = 'click'): Promise { + await fillProbeArgs(page, args); const responsePromise = page.waitForResponse((resp) => isProbePost(resp.request())); if (via === 'enter') { - await input.press('Enter'); + await page.locator(probe.argsLine).press('Enter'); } else { await page.locator(probe.button).click(); } @@ -63,13 +68,17 @@ test.describe('A Probing the oracle', () => { test('A1 first probe shows the result, the explore badge and History (1)', async ({ page }) => { await openAsFreshStudent(page, 'a1-first-probe', 2, 'clamp'); - const input = page.locator(probe.paramInput('n')); + const line = page.locator(probe.argsLine); const result = page.locator(probe.result); const button = page.locator(probe.button); - // Fresh state: empty input, no result, no history, unlimited badge - await expect(input).toHaveValue(''); - await expect(input).toHaveAttribute('placeholder', 'int'); + // Fresh state: empty call line showing the parameter name and its declared + // type (the signature is visible here; A10 covers names only), no result, + // no history, unlimited badge + await expect(line).toHaveJSProperty('textContent', ''); + await expect(line).toHaveAttribute('role', 'textbox'); + await expect(line).toHaveAttribute('aria-label', 'Arguments for clamp'); + expect(await readArgsPlaceholder(page)).toBe('n: int'); await expect(result).toHaveText('?'); await expect(result).not.toHaveClass(/has-result/); await expect(page.locator(probe.historyButton)).toHaveCount(0); @@ -78,7 +87,7 @@ test.describe('A Probing the oracle', () => { // No value yet: nothing to probe await expect(button).toBeDisabled(); - const body = await probeWith(page, 'n', '7'); + const body = await probeWith(page, '7'); expect(body.success).toBe(true); expect(body.result).toBe(7); expect(body.probe_status.mode).toBe('explore'); @@ -89,7 +98,7 @@ test.describe('A Probing the oracle', () => { await expect(page.locator(probe.error)).toHaveCount(0); // A new value brings the Probe button back with the unlimited badge - await input.fill('8'); + await fillProbeArgs(page, '8'); await expect(button).toBeEnabled(); await expect(page.locator(probe.badge)).toHaveText('· ∞'); }); @@ -97,12 +106,12 @@ test.describe('A Probing the oracle', () => { test('A2 repeat probe with a new input (Enter key) replaces the result', async ({ page }) => { await openAsFreshStudent(page, 'a2-repeat-probe', 2, 'clamp'); - await probeWith(page, 'n', '7'); + await probeWith(page, '7'); await expect(page.locator(probe.result)).toHaveText('7'); await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); - // Enter in the field triggers the probe - const body = await probeWith(page, 'n', '20', 'enter'); + // Enter in the call line triggers the probe + const body = await probeWith(page, '20', 'enter'); expect(body.success).toBe(true); expect(body.result).toBe(10); @@ -113,8 +122,8 @@ test.describe('A Probing the oracle', () => { test('A3 duplicate probe returns the cached result without a request', async ({ page }) => { await openAsFreshStudent(page, 'a3-duplicate-probe', 2, 'clamp'); - await probeWith(page, 'n', '7'); - await probeWith(page, 'n', '20'); + await probeWith(page, '7'); + await probeWith(page, '20'); await expect(page.locator(probe.result)).toHaveText('10'); await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); @@ -125,11 +134,11 @@ test.describe('A Probing the oracle', () => { } }); - const input = page.locator(probe.paramInput('n')); + const line = page.locator(probe.argsLine); const result = page.locator(probe.result); const button = page.locator(probe.button); - await input.fill('7'); + await fillProbeArgs(page, '7'); await expect(result).toHaveClass(/is-cached/); await expect(result).toHaveText('7'); await expect(button).toHaveClass(/is-duplicate/); @@ -137,12 +146,12 @@ test.describe('A Probing the oracle', () => { await expect(button).toBeDisabled(); await expect(page.locator(probe.badge)).toHaveCount(0); - // Enter in the field is the other way to probe; it must not re-send either - await input.press('Enter'); + // Enter in the call line is the other way to probe; it must not re-send either + await line.press('Enter'); // A real probe afterwards orders the request stream: if the duplicate had // been sent, it would be recorded before this one. - const body = await probeWith(page, 'n', '3'); + const body = await probeWith(page, '3'); expect(body.result).toBe(3); expect(probePosts).toEqual([JSON.stringify({ input: { n: 3 } })]); await expect(page.locator(probe.historyButton)).toHaveText('History (3)'); @@ -158,25 +167,27 @@ test.describe('A Probing the oracle', () => { } }); - const input = page.locator(probe.paramInput('n')); + const line = page.locator(probe.argsLine); + const shell = page.locator(probe.callShell); const button = page.locator(probe.button); const fieldError = page.locator(probe.fieldError('n')); - await input.fill('abc'); - await expect(fieldError).toHaveText('n: Must be a whole number (e.g. 5)'); - await expect(input).toHaveClass(/has-error/); - await expect(input).toHaveAttribute('aria-invalid', 'true'); - await expect(input).toHaveAttribute('aria-describedby', 'probe-err-n'); + await fillProbeArgs(page, 'abc'); + await expectArgError(page, 'n', 'n: Must be a whole number (e.g. 5)'); + await expect(shell).toHaveClass(/has-error/); + await expect(line).toHaveAttribute('aria-invalid', 'true'); + await expect(line).toHaveAttribute('aria-describedby', 'probe-field-errors probe-args-help'); await expect(button).toBeDisabled(); // Enter with an invalid value sends nothing - await input.press('Enter'); + await line.press('Enter'); - await input.fill('3'); + await fillProbeArgs(page, '3'); await expect(fieldError).toHaveCount(0); - await expect(input).not.toHaveClass(/has-error/); + await expect(shell).not.toHaveClass(/has-error/); + await expect(line).not.toHaveAttribute('aria-invalid', 'true'); await expect(button).toBeEnabled(); - const body = await probeWith(page, 'n', '3'); + const body = await probeWith(page, '3'); expect(body.result).toBe(3); expect(probePosts).toBe(1); await expect(page.locator(probe.result)).toHaveText('3'); @@ -185,9 +196,9 @@ test.describe('A Probing the oracle', () => { test('A5 history modal lists every probe, most recent first, and closes', async ({ page }) => { await openAsFreshStudent(page, 'a5-history-modal', 2, 'clamp'); - await probeWith(page, 'n', '7'); - await probeWith(page, 'n', '20'); - await probeWith(page, 'n', '-3'); + await probeWith(page, '7'); + await probeWith(page, '20'); + await probeWith(page, '-3'); await expect(page.locator(probe.result)).toHaveText('0'); const historyButton = page.locator(probe.historyButton); @@ -212,8 +223,8 @@ test.describe('A Probing the oracle', () => { test('A6 reload restores the probe history from the server', async ({ page }) => { const user = await openAsFreshStudent(page, 'a6-reload', 2, 'clamp'); - await probeWith(page, 'n', '7'); - await probeWith(page, 'n', '20'); + await probeWith(page, '7'); + await probeWith(page, '20'); await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); const server = await apiAsUser( @@ -237,10 +248,10 @@ test.describe('A Probing the oracle', () => { await historyLoaded; await expect(page.locator(probe.panel)).toBeVisible(); - // Both probes restored from the server, the latest result shown, inputs cleared + // Both probes restored from the server, the latest result shown, call line cleared await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); await expect(page.locator(probe.result)).toHaveText('10'); - await expect(page.locator(probe.paramInput('n'))).toHaveValue(''); + await expect(page.locator(probe.argsLine)).toHaveJSProperty('textContent', ''); await expect(page.locator(probe.badge)).toHaveText('· ∞'); await page.locator(probe.historyButton).click(); @@ -250,7 +261,7 @@ test.describe('A Probing the oracle', () => { await page.locator(probe.close).click(); // Re-entering a value probed before the reload is still recognised - await page.locator(probe.paramInput('n')).fill('7'); + await fillProbeArgs(page, '7'); await expect(page.locator(probe.result)).toHaveClass(/is-cached/); await expect(page.locator(probe.result)).toHaveText('7'); await expect(page.locator(probe.button)).toContainText('Already probed'); @@ -264,7 +275,7 @@ test.describe('A Probing the oracle', () => { await expect(badge).toHaveClass(/status-explore/); for (let n = 1; n <= 15; n += 1) { - const body = await probeWith(page, 'n', String(n)); + const body = await probeWith(page, String(n)); expect(body.success).toBe(true); expect(body.result).toBe(Math.min(n, 10)); expect(body.probe_status.mode).toBe('explore'); @@ -274,15 +285,14 @@ test.describe('A Probing the oracle', () => { await expect(page.locator(probe.historyButton)).toHaveText('History (15)'); - const input = page.locator(probe.paramInput('n')); - await input.fill('16'); - await expect(input).toBeEnabled(); + await fillProbeArgs(page, '16'); + await expectArgsEditable(page, true); await expect(page.locator(probe.button)).toBeEnabled(); await expect(badge).toHaveText('· ∞'); await expect(badge).toHaveClass(/status-explore/); await expect(page.locator(probe.hint)).toHaveCount(0); - const body = await probeWith(page, 'n', '16'); + const body = await probeWith(page, '16'); expect(body.success).toBe(true); expect(body.result).toBe(10); await expect(page.locator(probe.historyButton)).toHaveText('History (16)'); @@ -291,10 +301,10 @@ test.describe('A Probing the oracle', () => { test('A10 hidden signature hides types and sanitises oracle errors', async ({ page }) => { const user = await openAsFreshStudent(page, 'a10-hidden', 4, 'recip'); - const input = page.locator(probe.paramInput('x')); - await expect(page.locator('.probe-panel .param-name')).toHaveText(['x']); - await expect(input).toHaveAttribute('placeholder', 'value — quote text: "abc"'); - await expect(input).toHaveAttribute('aria-label', 'Enter value for x'); + // Parameter names are not secret: the empty call line still shows them + const line = page.locator(probe.argsLine); + expect(await readArgsPlaceholder(page)).toBe('x'); + await expect(line).toHaveAttribute('aria-label', 'Arguments for recip'); // The types never reach the browser: the problem payload the page loads // carries no signature, untyped parameters and no oracle source. @@ -307,15 +317,28 @@ test.describe('A Probing the oracle', () => { expect(hidden.reference_solution).toBeNull(); await expect(page.locator(probe.badge)).toHaveText('· ∞'); - const failed = await probeWith(page, 'x', '0'); - expect(failed.success).toBe(false); + // Without the types the browser cannot pre-check 'abc', so only the backend + // can reject it: a 400 (not charged) whose generic message reaches the UI const error = page.locator(probe.error); + await fillProbeArgs(page, 'abc'); + await expect(page.locator(probe.fieldError('x'))).toHaveCount(0); + const rejectedPromise = page.waitForResponse((resp) => isProbePost(resp.request())); + await page.locator(probe.button).click(); + const rejected = await rejectedPromise; + expect(rejected.status()).toBe(400); + await expect(error).toHaveText("That value is not valid for parameter 'x'"); + await expect(error).not.toContainText('int'); + await expect(page.locator(probe.historyButton)).toHaveCount(0); + await expect(page.locator(probe.badge)).toHaveText('· ∞'); + + const failed = await probeWith(page, '0'); + expect(failed.success).toBe(false); await expect(error).toHaveText('These inputs could not be evaluated. Try different values.'); await expect(error).not.toContainText('division'); await expect(page.locator(probe.result)).toHaveText('?'); await expect(page.locator(probe.historyButton)).toHaveCount(0); - const ok = await probeWith(page, 'x', '5'); + const ok = await probeWith(page, '5'); expect(ok.success).toBe(true); expect(ok.result).toBe(2); await expect(page.locator(probe.result)).toHaveText('2'); @@ -330,7 +353,7 @@ test.describe('A Probing the oracle', () => { await expect(badge).toHaveText('· 3 left'); await expect(badge).toHaveClass(/status-available/); - const failed = await probeWith(page, 'x', '0'); + const failed = await probeWith(page, '0'); expect(failed.success).toBe(false); expect(failed.probe_status.remaining).toBe(3); // Exactly the exception text, nothing else (no traceback, no type name). @@ -344,7 +367,7 @@ test.describe('A Probing the oracle', () => { await expect(page.locator(probe.result)).toHaveText('?'); await expect(page.locator(probe.historyButton)).toHaveCount(0); - const ok = await probeWith(page, 'x', '5'); + const ok = await probeWith(page, '5'); expect(ok.success).toBe(true); expect(ok.result).toBe(2); expect(ok.probe_status.remaining).toBe(2); @@ -352,10 +375,244 @@ test.describe('A Probing the oracle', () => { await expect(page.locator(probe.error)).toHaveCount(0); await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); - // The input still holds the probed value (duplicate state hides the badge); - // a new value shows the charged budget - await page.locator(probe.paramInput('x')).fill('6'); + // The call line still holds the probed value (duplicate state hides the + // badge); a new value shows the charged budget + await fillProbeArgs(page, '6'); await expect(badge).toHaveText('· 2 left'); await expect(badge).toHaveClass(/status-available/); }); + + test('A12 list[str] given integers shows the item-type error and disables Probe', async ({ + page, + }) => { + await openAsFreshStudent(page, 'a12-list-items', 7, 'tag'); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts.push(req.postData() ?? ''); + } + }); + + const line = page.locator(probe.argsLine); + const button = page.locator(probe.button); + + // Integers are not str: the item type is checked, not just the list shape + await fillProbeArgs(page, '[1, 2, 3], "x"'); + await expectArgError(page, 'items', 'items: Every item must be str'); + await expect(page.locator(probe.fieldError('prefix'))).toHaveCount(0); + await expect(page.locator(probe.callShell)).toHaveClass(/has-error/); + await expect(button).toBeDisabled(); + // Enter with an invalid value sends nothing + await line.press('Enter'); + + await fillProbeArgs(page, '["a", "b"], "x"'); + await expect(page.locator(`${probe.fieldErrors} li`)).toHaveCount(0); + await expect(button).toBeEnabled(); + await expect.poll(() => readBadge(page)).toBe('· ∞'); + + const body = await probeWith(page, '["a", "b"], "x"'); + expect(body.success).toBe(true); + expect(body.result).toEqual(['xa', 'xb']); + await expect(page.locator(probe.result)).toHaveText('["xa","xb"]'); + expect(probePosts).toEqual([JSON.stringify({ input: { items: ['a', 'b'], prefix: 'x' } })]); + }); + + test('A13 keyword and positional arguments map onto the signature', async ({ page }) => { + await openAsFreshStudent(page, 'a13-arg-mapping', 2, 'clamp'); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts.push(req.postData() ?? ''); + } + }); + + const result = page.locator(probe.result); + const button = page.locator(probe.button); + const lineError = page.locator(probe.lineError); + + // Positional: the first argument is n + const body = await probeWith(page, '7'); + expect(body.result).toBe(7); + await expect(result).toHaveText('7'); + + // Keyword: n=7 is the same call, so the cached result answers it + await fillProbeArgs(page, 'n=7'); + await expect(lineError).toHaveCount(0); + await expect(result).toHaveClass(/is-cached/); + await expect(result).toHaveText('7'); + await expect(button).toContainText('Already probed'); + + await fillProbeArgs(page, 'n=7, n=8'); + await expectArgError(page, 'line', 'Argument n given twice'); + await expect(page.locator(probe.callShell)).toHaveClass(/has-error/); + await expect(button).toBeDisabled(); + + await fillProbeArgs(page, 'm=7'); + await expectArgError(page, 'line', 'Unknown argument m'); + await expect(button).toBeDisabled(); + + await fillProbeArgs(page, '7, 8'); + await expectArgError(page, 'line', 'Too many arguments for clamp'); + await expect(button).toBeDisabled(); + + // A clean line clears the line error + await fillProbeArgs(page, 'n=8'); + await expect(lineError).toHaveCount(0); + await expect(button).toBeEnabled(); + + // Only the one positional probe reached the server, as {"n": 7} + expect(probePosts).toEqual([JSON.stringify({ input: { n: 7 } })]); + }); + + test('A14 Tab on an empty line fills in the argument names', async ({ page }) => { + await openAsFreshStudent(page, 'a14-tab-template', 7, 'tag'); + + const line = page.locator(probe.argsLine); + await expect(line).toHaveJSProperty('textContent', ''); + await line.focus(); + await page.keyboard.press('Tab'); + + await expect(line).toHaveJSProperty('textContent', 'items=, prefix='); + await expect(line).toBeFocused(); + // The caret sits right after the first '=', collapsed + const caret = await line.evaluate((el) => { + const selection = window.getSelection(); + if (!selection || selection.rangeCount === 0) { + return null; + } + const range = selection.getRangeAt(0); + const before = document.createRange(); + before.selectNodeContents(el); + before.setEnd(range.startContainer, range.startOffset); + return { + inside: el.contains(range.startContainer), + collapsed: range.collapsed, + offset: before.toString().length, + }; + }); + expect(caret).toEqual({ inside: true, collapsed: true, offset: 'items='.length }); + + // Typing continues from the caret + await page.keyboard.type('["a", "b"]'); + await expect(line).toHaveJSProperty('textContent', 'items=["a", "b"], prefix='); + await page.keyboard.press('End'); + await page.keyboard.type('"x"'); + await expect(line).toHaveJSProperty('textContent', 'items=["a", "b"], prefix="x"'); + // Shift+Enter inserts nothing: the call is one logical line + await page.keyboard.press('Shift+Enter'); + await expect(line).toHaveJSProperty('textContent', 'items=["a", "b"], prefix="x"'); + await expect(line).toBeFocused(); + + const responsePromise = page.waitForResponse((resp) => isProbePost(resp.request())); + await page.keyboard.press('Enter'); + const response = await responsePromise; + expect(response.status()).toBe(200); + expect(response.request().postData()).toBe( + JSON.stringify({ input: { items: ['a', 'b'], prefix: 'x' } }), + ); + await expect(page.locator(probe.result)).toHaveText('["xa","xb"]'); + // Enter probes; it never adds a line break + await expect(line).toHaveJSProperty('textContent', 'items=["a", "b"], prefix="x"'); + // Focus comes back to the line, so the next probe needs no Tab back + await expect(line).toBeFocused(); + await expect(line).toHaveAttribute('contenteditable', 'plaintext-only'); + + // Tab on a non-empty line is ordinary focus navigation: it leaves the line + // and changes nothing, so the editable is never a keyboard trap + await page.keyboard.press('Tab'); + await expect(line).not.toBeFocused(); + await expect(line).toHaveJSProperty('textContent', 'items=["a", "b"], prefix="x"'); + }); + + test('A15 the closing parenthesis trails the arguments', async ({ page }) => { + await openAsFreshStudent(page, 'a15-close-paren', 7, 'tag'); + + const words = Array.from({ length: 40 }, (_, i) => `"word${String(i).padStart(2, '0')}"`); + const args = `[${words.join(', ')}], "x"`; + await fillProbeArgs(page, args); + await expect(page.locator(probe.button)).toBeEnabled(); + + const line = page.locator(probe.argsLine); + const lineRects = await line.evaluate((el) => + Array.from(el.getClientRects()).map((r) => ({ + top: r.top, + bottom: r.bottom, + right: r.right, + })), + ); + // The long argument list wraps over several lines + expect(lineRects.length).toBeGreaterThan(1); + const last = lineRects[lineRects.length - 1]; + + const paren = await page.locator(probe.closeParen).boundingBox(); + expect(paren).not.toBeNull(); + await expect(page.locator(probe.closeParen)).toHaveText(')'); + // ')' sits on the editable's last line, right after its last character + const parenMiddle = paren!.y + paren!.height / 2; + expect(parenMiddle).toBeGreaterThan(last.top); + expect(parenMiddle).toBeLessThan(last.bottom); + expect(paren!.x).toBeGreaterThanOrEqual(last.right - 1); + }); + + test('A16 the backend rejects a list[str] with integers too', async ({ page }) => { + const user = await openAsFreshStudent(page, 'a16-backend-list', 7, 'tag'); + + const { status, data } = await apiAsUser( + page, + user, + 'POST', + '/api/problems/e2e-probeable-list/probe/', + { input: { items: [1, 2], prefix: 'x' } }, + ); + expect(status).toBe(400); + expect(data.error).toBe("Parameter 'items' must be a list of str"); + }); + + test('A17 a quoted str argument is a Python string literal', async ({ page }) => { + await openAsFreshStudent(page, 'a17-str-quotes', 7, 'tag'); + + const probePosts: string[] = []; + page.on('request', (req) => { + if (isProbePost(req)) { + probePosts.push(req.postData() ?? ''); + } + }); + const button = page.locator(probe.button); + + // An unterminated quote is not a string: error, and Probe stays disabled + await fillProbeArgs(page, `['a'], "abc`); + await expectArgError(page, 'prefix', 'prefix: Not a valid Python value'); + await expect(button).toBeDisabled(); + await page.locator(probe.argsLine).press('Enter'); + + // An empty argument between commas is a line error, not a skipped parameter + await fillProbeArgs(page, `['a'],, 'x'`); + await expectArgError(page, 'line', 'Empty argument between commas'); + await expect(button).toBeDisabled(); + + // Single quotes delimit the str exactly as double quotes do + const body = await probeWith(page, `['a', 'b'], 'x'`, 'enter'); + expect(body.success).toBe(true); + expect(body.result).toEqual(['xa', 'xb']); + await expect(page.locator(probe.result)).toHaveText('["xa","xb"]'); + expect(probePosts).toEqual([JSON.stringify({ input: { items: ['a', 'b'], prefix: 'x' } })]); + }); + + test('A18 on a narrow screen History wraps below the call line', async ({ page }) => { + await page.setViewportSize({ width: 320, height: 800 }); + await openAsFreshStudent(page, 'a18-narrow', 7, 'tag'); + + await probeWith(page, '["a"], "x"'); + await expect(page.locator(probe.historyButton)).toHaveText('History (1)'); + + const row = await page.locator(probe.row).boundingBox(); + const shell = await page.locator(probe.callShell).boundingBox(); + const history = await page.locator(probe.historyButton).boundingBox(); + expect(row && shell && history).toBeTruthy(); + // The call line gets the full row width; History sits on a line of its own + expect(shell!.width).toBeGreaterThanOrEqual(row!.width - 1); + expect(history!.y).toBeGreaterThanOrEqual(shell!.y + shell!.height); + }); }); diff --git a/e2e/student/probeable-spec.spec.ts b/e2e/student/probeable-spec.spec.ts index 63d38e2..3d3f353 100644 --- a/e2e/student/probeable-spec.spec.ts +++ b/e2e/student/probeable-spec.spec.ts @@ -23,6 +23,7 @@ import { test, expect, Page } from '@playwright/test'; import { setAceEditorValue } from '../helpers/ace-editor'; import { probe, feedback, problemSet } from '../helpers/selectors'; +import { fillProbeArgs } from '../helpers/probe'; import { FreshUser, apiAsUser, @@ -64,9 +65,9 @@ async function openSpecProblem( return user; } -/** Probe square(x=value) and wait for the oracle response. */ +/** Probe square(value) through the call line and wait for the oracle response. */ async function probeX(page: Page, value: string): Promise { - await page.locator(probe.paramInput('x')).fill(value); + await fillProbeArgs(page, value); const button = page.locator(probe.button); await expect(button).toBeEnabled(); const responsePromise = page.waitForResponse( @@ -110,13 +111,13 @@ test.describe('C24 Probeable Spec', () => { await probeX(page, '-2'); await expect(page.locator(probe.result)).toHaveText('4'); await expect(page.locator(probe.historyButton)).toHaveText('History (2)'); - // The input still holds the last probed value, so the button reports it + // The call line still holds the last probed value, so the button reports it // as already probed (no badge while duplicate). await expect(page.locator(probe.button)).toHaveText(/Already probed/); await expect(page.locator(probe.button)).toHaveClass(/is-duplicate/); await expect(page.locator(probe.button)).toBeDisabled(); // A new value re-enables probing; explore mode keeps the badge unlimited. - await page.locator(probe.paramInput('x')).fill('7'); + await fillProbeArgs(page, 'x=7'); await expect(page.locator(probe.button)).toBeEnabled(); await expect(page.locator(probe.badge)).toHaveText('· ∞'); await expect(page.locator(probe.badge)).toHaveClass(/status-explore/); diff --git a/e2e/student/submission-probeable.spec.ts b/e2e/student/submission-probeable.spec.ts index ce3df4e..039363c 100644 --- a/e2e/student/submission-probeable.spec.ts +++ b/e2e/student/submission-probeable.spec.ts @@ -41,6 +41,7 @@ import { apiAs } from '../helpers/api'; import { apiAsUser, enrollUser, freshStudent, injectUser, navigateAsFreshStudent } from '../helpers/users'; import { submitCodeAndWait } from '../helpers/submission'; import { codeFeedback, probe } from '../helpers/selectors'; +import { expectArgError, expectArgsEditable, fillProbeArgs } from '../helpers/probe'; const PROBEABLE_PATH = '/courses/CS101-2024/problem-set/e2e-code?p=1'; const TYPED_PROBEABLE_PATH = '/courses/CS101-2024/problem-set/e2e-code?p=2'; @@ -160,7 +161,6 @@ test.describe('Probeable Code Submission', () => { await expect(page.locator(probe.historyButton)).toHaveCount(0); // e2e-probeable-code-1 has signature: mystery(lst: list[int]) -> list[int] - const lstInput = page.locator(probe.paramInput('lst')); const probeBtn = page.locator(probe.button); const probes: Array<[string, number[]]> = [ ['[3, 1, 2]', [1, 2, 3]], @@ -168,7 +168,7 @@ test.describe('Probeable Code Submission', () => { ]; for (const [value, expected] of probes) { - await lstInput.fill(value); + await fillProbeArgs(page, value); await expect(probeBtn).toBeEnabled(); const responsePromise = page.waitForResponse( (resp) => @@ -207,22 +207,22 @@ test.describe('Probe Input Type Validation', () => { await waitForProbeableInput(page); // e2e-probeable-code-typed has signature: double(n: int) -> int - const nInput = page.locator(probe.paramInput('n')); - await expect(nInput).toBeVisible(); - await expect(nInput).toBeEnabled(); + await expect(page.locator(probe.argsLine)).toBeVisible(); + await expectArgsEditable(page, true); const fieldErrors = page.locator(probe.fieldErrors); + const callShell = page.locator(probe.callShell); const probeBtn = page.locator(probe.button); - await nInput.fill('abc'); + await fillProbeArgs(page, 'abc'); await expect(fieldErrors).toBeVisible(); - await expect(page.locator(probe.fieldError('n'))).toHaveText('n: Must be a whole number (e.g. 5)'); - await expect(nInput).toHaveClass(/has-error/); + await expectArgError(page, 'n', 'n: Must be a whole number (e.g. 5)'); + await expect(callShell).toHaveClass(/has-error/); await expect(probeBtn).toBeDisabled(); - await nInput.fill('5'); + await fillProbeArgs(page, '5'); await expect(fieldErrors).toBeHidden(); - await expect(nInput).not.toHaveClass(/has-error/); + await expect(callShell).not.toHaveClass(/has-error/); await expect(probeBtn).toBeEnabled(); }); }); diff --git a/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue b/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue index bf01991..21ea9b6 100644 --- a/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue +++ b/purplex/client/src/components/activities/inputs/ProbeableCodeInput.vue @@ -19,7 +19,9 @@ :probe-count-display="probeCountDisplay" :probe-status-text="probeStatusHint" :probe-params="parameters" - :probe-inputs="probeInputs" + :args-line="argsLine" + :args-template="argsTemplate" + :line-error="lineError" :can-probe="canProbe" :is-executing="probing" :has-valid-inputs="hasValidInputs" @@ -31,7 +33,7 @@ :format-function-call="formatFunctionCall" :format-output="formatOutput" @execute-probe="executeProbe" - @update-input="updateProbeInput" + @update-args="setArgsLine" /> @@ -140,7 +142,7 @@ const { probing, probeError, probeHistory, - probeInputs, + argsLine, functionName, parameters, canProbe, @@ -149,12 +151,14 @@ const { probeStatusClass, hasValidInputs, inputErrors, + lineError, + argsTemplate, isDuplicate, cachedResult, executeProbe, formatFunctionCall, formatOutput, - updateProbeInput, + setArgsLine, } = useProbeState(() => props.problem, { isSubmitting: () => props.disabled }) const { t } = useI18n() diff --git a/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue b/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue index b2503e3..b59e111 100644 --- a/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue +++ b/purplex/client/src/components/activities/inputs/ProbeableSpecInput.vue @@ -19,7 +19,9 @@ :probe-count-display="probeCountDisplay" :probe-status-text="probeStatusHint" :probe-params="parameters" - :probe-inputs="probeInputs" + :args-line="argsLine" + :args-template="argsTemplate" + :line-error="lineError" :can-probe="canProbe" :is-executing="probing" :has-valid-inputs="hasValidInputs" @@ -31,7 +33,7 @@ :format-function-call="formatFunctionCall" :format-output="formatOutput" @execute-probe="executeProbe" - @update-input="updateProbeInput" + @update-args="setArgsLine" /> @@ -142,7 +144,7 @@ const { probing, probeError, probeHistory, - probeInputs, + argsLine, functionName, parameters, canProbe, @@ -151,12 +153,14 @@ const { probeStatusClass, hasValidInputs, inputErrors, + lineError, + argsTemplate, isDuplicate, cachedResult, executeProbe, formatFunctionCall, formatOutput, - updateProbeInput, + setArgsLine, } = useProbeState(() => props.problem, { isSubmitting: () => props.disabled }) const { t } = useI18n() diff --git a/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue b/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue index 2687eaf..e67a4ee 100644 --- a/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue +++ b/purplex/client/src/components/activities/inputs/shared/ProbePanel.vue @@ -3,36 +3,35 @@
- {{ functionName }} - ( - - ) + id="probe-args" + ref="argsRef" + class="call-args" + :contenteditable="editable ? EDITABLE_MODE : 'false'" + role="textbox" + aria-multiline="false" + spellcheck="false" + :data-placeholder="argsPlaceholder" + :aria-placeholder="argsPlaceholder" + :aria-label="$t('problems.probe.argsAriaLabel', { name: functionName })" + :aria-disabled="editable ? undefined : 'true'" + :aria-invalid="hasErrors ? 'true' : undefined" + :aria-describedby="hasErrors ? 'probe-field-errors probe-args-help' : 'probe-args-help'" + @input="onArgsInput" + @keydown="onArgsKeydown" + @paste="onArgsPaste" + /> + ) +
→
- + + +
    +
  • + {{ lineError }} +
  • + +
    + {{ $t('problems.probe.argsHelp') }} +
    +