+ {/* The bar is one line tall. On a narrow screen the badges are clipped rather than
+ wrapped, because wrapped text grows past the bar and over the header above it. */}
+
{step !== undefined && (
<>
diff --git a/src/engine/analysis/explore.ts b/src/engine/analysis/explore.ts
index b00ed13..e02841c 100644
--- a/src/engine/analysis/explore.ts
+++ b/src/engine/analysis/explore.ts
@@ -1,6 +1,7 @@
import type { Layer, RecipeStep } from '../types';
import { getOperation } from '../operations';
import { byteLength, previewText } from '../core/bytes';
+import { randomId } from '../core/uid';
import { detect } from '../detection/detect';
import { findEmbedded } from '../detection/embedded';
import type { AnalysisNode, AnalyseOptions } from './types';
@@ -98,7 +99,7 @@ function makeNode(
): AnalysisNode {
const path = extendPath(parentPath, format);
return {
- id: `${format}-${depth}-${Math.random().toString(36).slice(2, 8)}`,
+ id: randomId(`${format}-${depth}`),
format,
confidence,
depth,
diff --git a/src/engine/core/strip.ts b/src/engine/core/strip.ts
new file mode 100644
index 0000000..a89d09e
--- /dev/null
+++ b/src/engine/core/strip.ts
@@ -0,0 +1,16 @@
+/*
+ * Removes every match of a pattern, repeating until nothing changes.
+ *
+ * One pass is not enough for markup. Removing the inner tag of `ipt>` joins
+ * the outer one back together, and removing the inner comment of `- x -->` leaves a
+ * whole one behind. An operation that promises to strip tags or comments should leave none, so
+ * it repeats. Every pass that changes anything makes the text shorter, so the loop ends.
+ */
+export function removeAll(text: string, pattern: RegExp): string {
+ let previous: string;
+ do {
+ previous = text;
+ text = text.replace(pattern, '');
+ } while (text !== previous);
+ return text;
+}
diff --git a/src/engine/core/uid.ts b/src/engine/core/uid.ts
new file mode 100644
index 0000000..a7b3f4e
--- /dev/null
+++ b/src/engine/core/uid.ts
@@ -0,0 +1,11 @@
+/*
+ * Identifiers for recipe steps, analysis nodes and saved recipes. They only have to be unique
+ * within one browser, never secret, but drawing them from the platform's CSPRNG costs nothing and
+ * leaves no Math.random in anything that looks like an identifier. getRandomValues, unlike
+ * randomUUID, also works outside a secure context, which the single-file build opened from disk
+ * may be.
+ */
+export function randomId(prefix: string): string {
+ const [high = 0, low = 0] = crypto.getRandomValues(new Uint32Array(2));
+ return `${prefix}-${high.toString(36)}${low.toString(36)}`;
+}
diff --git a/src/engine/detection.redos.test.ts b/src/engine/detection.redos.test.ts
new file mode 100644
index 0000000..bebca68
--- /dev/null
+++ b/src/engine/detection.redos.test.ts
@@ -0,0 +1,44 @@
+import { describe, expect, it } from 'vitest';
+import { detect } from './index';
+import { getOperation } from './operations';
+
+/*
+ * Detection runs on whatever is pasted, before anyone has decided it is CSV, so every
+ * detection pattern has to finish in time proportional to the input. A pattern that
+ * backtracks exponentially lets a few dozen characters freeze the page.
+ */
+describe('detection patterns stay linear on hostile input', () => {
+ const csvPattern = getOperation('csv-to-json')?.detection?.pattern;
+
+ it('has a CSV detection pattern to test', () => {
+ expect(csvPattern).toBeInstanceOf(RegExp);
+ });
+
+ it('rejects many comma-only lines that fail at the end, quickly', () => {
+ // 16 of these lines took seconds with the old pattern, 18 took over thirty.
+ const hostile = 'a,b' + '\n,,,'.repeat(5000) + '\nx';
+ const started = performance.now();
+ expect(csvPattern?.test(hostile)).toBe(false);
+ expect(performance.now() - started).toBeLessThan(250);
+ });
+
+ it('still recognises ordinary CSV, including CRLF line endings', () => {
+ expect(csvPattern?.test('name,age\nalice,30\nbob,25')).toBe(true);
+ expect(csvPattern?.test('name,age\r\nalice,30\r\nbob,25\r\n'.trim())).toBe(true);
+ expect(csvPattern?.test('a,b,c\n1,2,3\n4,5,6')).toBe(true);
+ });
+
+ it('still rejects text that is not CSV', () => {
+ expect(csvPattern?.test('one line only, with a comma')).toBe(false);
+ expect(csvPattern?.test('a,b\nno comma here')).toBe(false);
+ expect(csvPattern?.test('a;b\nc;d')).toBe(false);
+ });
+
+ it('lets full detection finish on the hostile input', async () => {
+ const hostile = 'a,b' + '\n,,,'.repeat(200) + '\nx';
+ const started = performance.now();
+ const candidates = await detect(hostile);
+ expect(performance.now() - started).toBeLessThan(5000);
+ expect(candidates.some((c) => c.format === 'CSV')).toBe(false);
+ });
+});
diff --git a/src/engine/detection/bruteForce.ts b/src/engine/detection/bruteForce.ts
index 501896a..c24c0ab 100644
--- a/src/engine/detection/bruteForce.ts
+++ b/src/engine/detection/bruteForce.ts
@@ -1,5 +1,6 @@
import type { Candidate, Evidence, RecipeStep } from '../types';
import { asBytes, bytesToLatin1, previewText } from '../core/bytes';
+import { randomId } from '../core/uid';
import { containsMarker, readability } from './readability';
@@ -67,7 +68,7 @@ function evidenceFor(output: string, score: number, how: string): Evidence[] {
}
function uid(opId: string): string {
- return `${opId}-${Math.random().toString(36).slice(2, 9)}`;
+ return randomId(opId);
}
function xorStep(key: string): RecipeStep {
diff --git a/src/engine/detection/steps.ts b/src/engine/detection/steps.ts
index c5a3922..96c651b 100644
--- a/src/engine/detection/steps.ts
+++ b/src/engine/detection/steps.ts
@@ -1,5 +1,6 @@
import type { OperationArg, RecipeStep } from '../types';
import { getOperation } from '../operations';
+import { randomId } from '../core/uid';
import type { Operation } from '../operations/types';
/*
@@ -11,7 +12,7 @@ import type { Operation } from '../operations/types';
*/
function uid(opId: string): string {
- return `${opId}-${Math.random().toString(36).slice(2, 9)}`;
+ return randomId(opId);
}
type Overrides = Record;
diff --git a/src/engine/index.ts b/src/engine/index.ts
index 5d5cf8b..2a06eb4 100644
--- a/src/engine/index.ts
+++ b/src/engine/index.ts
@@ -67,6 +67,7 @@ export const INPUT_ENCODINGS: string[] = ['Raw bytes', ...ENCODABLE_CHARSET_NAME
export { asBytes } from './core/bytes';
export { hexdumpOf } from './core/views';
+export { randomId } from './core/uid';
export { rewrap, INVERSES, whyNotInvertible } from './core/rewrap';
export type { RewrapResult, RewrapBlocker } from './core/rewrap';
diff --git a/src/engine/operations/structured.ts b/src/engine/operations/structured.ts
index 80cda70..ef47ae5 100644
--- a/src/engine/operations/structured.ts
+++ b/src/engine/operations/structured.ts
@@ -1,4 +1,5 @@
import { OperationError } from '../types';
+import { removeAll } from '../core/strip';
import { arg, type Operation } from './types';
function parseCsv(text: string, delimiter: string): string[][] {
@@ -137,7 +138,11 @@ export const structuredOperations: Operation[] = [
},
detection: {
formatName: 'CSV',
- pattern: /^[^\n,]*,[^\n,]*(,[^\n,]*)*(\r?\n[^\n]*,[^\n]*)+$/,
+ // Every line holds a comma. Each line can match in exactly one way: up to its first comma,
+ // then the rest of the line. The earlier `[^\n]*,[^\n]*` could split a line at any of its
+ // commas, and on input that failed at the end those choices multiplied line by line: 16
+ // lines of ",,," froze the page. Detection runs on whatever is pasted, so it must be linear.
+ pattern: /^[^\r\n,]*,[^\r\n]*(?:\r?\n[^\r\n,]*,[^\r\n]*)+\r?$/,
minLength: 12,
},
},
@@ -282,7 +287,7 @@ export const structuredOperations: Operation[] = [
run: (input, args) => {
const text = arg(args, 'Preserve comments', true)
? input
- : input.replace(//g, '');
+ : removeAll(input, //g);
return text.replace(/>\s+<').trim();
},
},
@@ -307,7 +312,7 @@ export const structuredOperations: Operation[] = [
case 'SQL':
return input.replace(/--.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trimEnd();
case 'XML / HTML':
- return input.replace(//g, '').trimEnd();
+ return removeAll(input, //g).trimEnd();
default:
return input
.replace(/\/\*[\s\S]*?\*\//g, '')
diff --git a/src/engine/operations/text.ts b/src/engine/operations/text.ts
index 2935967..3e9e7e6 100644
--- a/src/engine/operations/text.ts
+++ b/src/engine/operations/text.ts
@@ -1,4 +1,5 @@
import { OperationError } from '../types';
+import { removeAll } from '../core/strip';
import { arg, type Operation } from './types';
function buildRegex(pattern: string, flags: string): RegExp {
@@ -239,10 +240,9 @@ export const textOperations: Operation[] = [
run: (input, args) => {
let text = input;
if (arg(args, 'Remove scripts and styles', true)) {
- text = text.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, '');
+ text = removeAll(text, /<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi);
}
- return text
- .replace(/<[^>]+>/g, '')
+ return removeAll(text, /<[^>]+>/g)
.replace(/ /g, ' ')
.replace(/\n{3,}/g, '\n\n')
.trim();
diff --git a/src/engine/strip.test.ts b/src/engine/strip.test.ts
new file mode 100644
index 0000000..771fc95
--- /dev/null
+++ b/src/engine/strip.test.ts
@@ -0,0 +1,33 @@
+import { describe, expect, it } from 'vitest';
+import { removeAll } from './core/strip';
+import { getOperation } from './operations';
+
+async function runOp(id: string, input: string, overrides: Record = {}) {
+ const op = getOperation(id);
+ if (!op) throw new Error(`no operation ${id}`);
+ const args = op.args.map((a) => (a.name in overrides ? { ...a, value: overrides[a.name] } : a));
+ return op.run(input, args as typeof op.args);
+}
+
+describe('removing markup leaves none behind', () => {
+ it('repeats until a removal no longer rebuilds a match', () => {
+ // One pass over "aabb" removes the middle "ab" and joins the rest into a new "ab".
+ expect('aabb'.replace(/ab/g, '')).toBe('ab');
+ expect(removeAll('aabb', /ab/g)).toBe('');
+ expect(removeAll('<b>bold', /<[^>]+>/g)).not.toMatch(/<[^>]+>/);
+ expect(removeAll('plain text', /<[^>]+>/g)).toBe('plain text');
+ });
+
+ it('Strip HTML tags leaves no tag that one pass would have rebuilt', async () => {
+ const out = String(await runOp('strip-html', 'Hi xipt>alert(1) there
'));
+ expect(out).not.toMatch(/<[^>]+>/);
+ expect(out).toContain('Hi');
+ expect(out).toContain('there');
+ });
+
+ it('Strip comments (XML / HTML) removes nested comment fragments', async () => {
+ const out = String(await runOp('strip-comments', '- hidden -->', { Style: 'XML / HTML' }));
+ expect(out).not.toContain('