Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/components/panes/OutputPane.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ export function OutputPane() {
<span className="rounded-chip border border-line px-1.5 py-px font-mono text-[10px] uppercase tracking-wider text-muted">
{isImage ? 'image' : json ? 'json' : 'text'}
</span>
<span className="font-mono text-micro tabular-nums text-faint">
<span className="font-mono text-micro tabular-nums text-faint max-sm:hidden">
{formatBytes(output.length)}
{!isImage && ` · ${t.status.lines(shown.split('\n').length)}`}
</span>
Expand Down
4 changes: 3 additions & 1 deletion src/components/panes/Pane.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@ export function Pane({ title, step, meta, actions, children, className }: Props)
className={cx('flex min-h-0 min-w-0 flex-1 flex-col bg-bg', className)}
>
<div className="flex h-[24px] shrink-0 items-center justify-between gap-2 px-3">
<div className="flex min-w-0 items-center gap-1.5">
{/* The bar is one line tall. On a narrow screen the badges are clipped rather than
wrapped, because wrapped text grows past the bar and over the header above it. */}
<div className="flex min-w-0 items-center gap-1.5 overflow-hidden whitespace-nowrap">
<h2 className="flex items-center gap-1.5 text-[11px] leading-none text-faint">
{step !== undefined && (
<>
Expand Down
3 changes: 2 additions & 1 deletion src/engine/analysis/explore.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import type { Layer, RecipeStep } from '../types';
import { getOperation } from '../operations';
import { byteLength, previewText } from '../core/bytes';
import { randomId } from '../core/uid';
import { detect } from '../detection/detect';
import { findEmbedded } from '../detection/embedded';
import type { AnalysisNode, AnalyseOptions } from './types';
Expand Down Expand Up @@ -98,7 +99,7 @@ function makeNode(
): AnalysisNode {
const path = extendPath(parentPath, format);
return {
id: `${format}-${depth}-${Math.random().toString(36).slice(2, 8)}`,
id: randomId(`${format}-${depth}`),
format,
confidence,
depth,
Expand Down
16 changes: 16 additions & 0 deletions src/engine/core/strip.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Removes every match of a pattern, repeating until nothing changes.
*
* One pass is not enough for markup. Removing the inner tag of `<scr<script></script>ipt>` joins
* the outer one back together, and removing the inner comment of `<!-<!---->- x -->` leaves a
* whole one behind. An operation that promises to strip tags or comments should leave none, so
* it repeats. Every pass that changes anything makes the text shorter, so the loop ends.
*/
export function removeAll(text: string, pattern: RegExp): string {
let previous: string;
do {
previous = text;
text = text.replace(pattern, '');
} while (text !== previous);
return text;
}
11 changes: 11 additions & 0 deletions src/engine/core/uid.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/*
* Identifiers for recipe steps, analysis nodes and saved recipes. They only have to be unique
* within one browser, never secret, but drawing them from the platform's CSPRNG costs nothing and
* leaves no Math.random in anything that looks like an identifier. getRandomValues, unlike
* randomUUID, also works outside a secure context, which the single-file build opened from disk
* may be.
*/
export function randomId(prefix: string): string {
const [high = 0, low = 0] = crypto.getRandomValues(new Uint32Array(2));
return `${prefix}-${high.toString(36)}${low.toString(36)}`;
}
44 changes: 44 additions & 0 deletions src/engine/detection.redos.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import { describe, expect, it } from 'vitest';
import { detect } from './index';
import { getOperation } from './operations';

/*
* Detection runs on whatever is pasted, before anyone has decided it is CSV, so every
* detection pattern has to finish in time proportional to the input. A pattern that
* backtracks exponentially lets a few dozen characters freeze the page.
*/
describe('detection patterns stay linear on hostile input', () => {
const csvPattern = getOperation('csv-to-json')?.detection?.pattern;

it('has a CSV detection pattern to test', () => {
expect(csvPattern).toBeInstanceOf(RegExp);
});

it('rejects many comma-only lines that fail at the end, quickly', () => {
// 16 of these lines took seconds with the old pattern, 18 took over thirty.
const hostile = 'a,b' + '\n,,,'.repeat(5000) + '\nx';
const started = performance.now();
expect(csvPattern?.test(hostile)).toBe(false);
expect(performance.now() - started).toBeLessThan(250);
});

it('still recognises ordinary CSV, including CRLF line endings', () => {
expect(csvPattern?.test('name,age\nalice,30\nbob,25')).toBe(true);
expect(csvPattern?.test('name,age\r\nalice,30\r\nbob,25\r\n'.trim())).toBe(true);
expect(csvPattern?.test('a,b,c\n1,2,3\n4,5,6')).toBe(true);
});

it('still rejects text that is not CSV', () => {
expect(csvPattern?.test('one line only, with a comma')).toBe(false);
expect(csvPattern?.test('a,b\nno comma here')).toBe(false);
expect(csvPattern?.test('a;b\nc;d')).toBe(false);
});

it('lets full detection finish on the hostile input', async () => {
const hostile = 'a,b' + '\n,,,'.repeat(200) + '\nx';
const started = performance.now();
const candidates = await detect(hostile);
expect(performance.now() - started).toBeLessThan(5000);
expect(candidates.some((c) => c.format === 'CSV')).toBe(false);
});
});
3 changes: 2 additions & 1 deletion src/engine/detection/bruteForce.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { Candidate, Evidence, RecipeStep } from '../types';
import { asBytes, bytesToLatin1, previewText } from '../core/bytes';
import { randomId } from '../core/uid';
import { containsMarker, readability } from './readability';


Expand Down Expand Up @@ -67,7 +68,7 @@ function evidenceFor(output: string, score: number, how: string): Evidence[] {
}

function uid(opId: string): string {
return `${opId}-${Math.random().toString(36).slice(2, 9)}`;
return randomId(opId);
}

function xorStep(key: string): RecipeStep {
Expand Down
3 changes: 2 additions & 1 deletion src/engine/detection/steps.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { OperationArg, RecipeStep } from '../types';
import { getOperation } from '../operations';
import { randomId } from '../core/uid';
import type { Operation } from '../operations/types';

/*
Expand All @@ -11,7 +12,7 @@ import type { Operation } from '../operations/types';
*/

function uid(opId: string): string {
return `${opId}-${Math.random().toString(36).slice(2, 9)}`;
return randomId(opId);
}

type Overrides = Record<string, string | number | boolean>;
Expand Down
1 change: 1 addition & 0 deletions src/engine/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ export const INPUT_ENCODINGS: string[] = ['Raw bytes', ...ENCODABLE_CHARSET_NAME

export { asBytes } from './core/bytes';
export { hexdumpOf } from './core/views';
export { randomId } from './core/uid';
export { rewrap, INVERSES, whyNotInvertible } from './core/rewrap';
export type { RewrapResult, RewrapBlocker } from './core/rewrap';

Expand Down
11 changes: 8 additions & 3 deletions src/engine/operations/structured.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { OperationError } from '../types';
import { removeAll } from '../core/strip';
import { arg, type Operation } from './types';

function parseCsv(text: string, delimiter: string): string[][] {
Expand Down Expand Up @@ -137,7 +138,11 @@ export const structuredOperations: Operation[] = [
},
detection: {
formatName: 'CSV',
pattern: /^[^\n,]*,[^\n,]*(,[^\n,]*)*(\r?\n[^\n]*,[^\n]*)+$/,
// Every line holds a comma. Each line can match in exactly one way: up to its first comma,
// then the rest of the line. The earlier `[^\n]*,[^\n]*` could split a line at any of its
// commas, and on input that failed at the end those choices multiplied line by line: 16
// lines of ",,," froze the page. Detection runs on whatever is pasted, so it must be linear.
pattern: /^[^\r\n,]*,[^\r\n]*(?:\r?\n[^\r\n,]*,[^\r\n]*)+\r?$/,
minLength: 12,
},
},
Expand Down Expand Up @@ -282,7 +287,7 @@ export const structuredOperations: Operation[] = [
run: (input, args) => {
const text = arg(args, 'Preserve comments', true)
? input
: input.replace(/<!--[\s\S]*?-->/g, '');
: removeAll(input, /<!--[\s\S]*?-->/g);
return text.replace(/>\s+</g, '><').trim();
},
},
Expand All @@ -307,7 +312,7 @@ export const structuredOperations: Operation[] = [
case 'SQL':
return input.replace(/--.*$/gm, '').replace(/\/\*[\s\S]*?\*\//g, '').trimEnd();
case 'XML / HTML':
return input.replace(/<!--[\s\S]*?-->/g, '').trimEnd();
return removeAll(input, /<!--[\s\S]*?-->/g).trimEnd();
default:
return input
.replace(/\/\*[\s\S]*?\*\//g, '')
Expand Down
6 changes: 3 additions & 3 deletions src/engine/operations/text.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { OperationError } from '../types';
import { removeAll } from '../core/strip';
import { arg, type Operation } from './types';

function buildRegex(pattern: string, flags: string): RegExp {
Expand Down Expand Up @@ -239,10 +240,9 @@ export const textOperations: Operation[] = [
run: (input, args) => {
let text = input;
if (arg(args, 'Remove scripts and styles', true)) {
text = text.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi, '');
text = removeAll(text, /<(script|style)\b[^>]*>[\s\S]*?<\/\1>/gi);
}
return text
.replace(/<[^>]+>/g, '')
return removeAll(text, /<[^>]+>/g)
.replace(/&nbsp;/g, ' ')
.replace(/\n{3,}/g, '\n\n')
.trim();
Expand Down
33 changes: 33 additions & 0 deletions src/engine/strip.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { describe, expect, it } from 'vitest';
import { removeAll } from './core/strip';
import { getOperation } from './operations';

async function runOp(id: string, input: string, overrides: Record<string, unknown> = {}) {
const op = getOperation(id);
if (!op) throw new Error(`no operation ${id}`);
const args = op.args.map((a) => (a.name in overrides ? { ...a, value: overrides[a.name] } : a));
return op.run(input, args as typeof op.args);
}

describe('removing markup leaves none behind', () => {
it('repeats until a removal no longer rebuilds a match', () => {
// One pass over "aabb" removes the middle "ab" and joins the rest into a new "ab".
expect('aabb'.replace(/ab/g, '')).toBe('ab');
expect(removeAll('aabb', /ab/g)).toBe('');
expect(removeAll('<<b>b>bold</b>', /<[^>]+>/g)).not.toMatch(/<[^>]+>/);
expect(removeAll('plain text', /<[^>]+>/g)).toBe('plain text');
});

it('Strip HTML tags leaves no tag that one pass would have rebuilt', async () => {
const out = String(await runOp('strip-html', '<p>Hi <scr<script>x</script>ipt>alert(1)</script> there</p>'));
expect(out).not.toMatch(/<[^>]+>/);
expect(out).toContain('Hi');
expect(out).toContain('there');
});

it('Strip comments (XML / HTML) removes nested comment fragments', async () => {
const out = String(await runOp('strip-comments', '<a/><!-<!---->- hidden --><b/>', { Style: 'XML / HTML' }));
expect(out).not.toContain('<!--');
expect(out).not.toContain('hidden');
});
});
3 changes: 2 additions & 1 deletion src/lib/recipeText.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { OperationDef, RecipeStep } from '../engine';
import { randomId } from '../engine';
import { safeArgValue } from './share';


Expand Down Expand Up @@ -75,7 +76,7 @@ export function fromText(text: string, operations: OperationDef[]): ParseResult
}

steps.push({
uid: `${entry.op}-${Math.random().toString(36).slice(2, 9)}`,
uid: randomId(entry.op),
opId: entry.op,
args: definition.args.map((arg) => ({
...arg,
Expand Down
5 changes: 3 additions & 2 deletions src/lib/recipes.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { OperationArg, RecipeStep } from '../engine';
import { randomId } from '../engine';
import { safeArgValue } from './share';

export interface SavedRecipe {
Expand Down Expand Up @@ -88,7 +89,7 @@ function persist(recipes: SavedRecipe[]): SavedRecipe[] {

export function saveRecipe(name: string, steps: RecipeStep[]): SavedRecipe[] {
const recipe: SavedRecipe = {
id: `r-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`,
id: randomId('r'),
name: name.trim() || 'Untitled recipe',
steps: steps.map((s) => ({ ...s, args: s.args.map((a) => ({ ...a })) })),
savedAt: Date.now(),
Expand All @@ -103,7 +104,7 @@ export function deleteRecipe(id: string): SavedRecipe[] {
export function instantiate(recipe: SavedRecipe): RecipeStep[] {
return recipe.steps.map((step) => ({
...step,
uid: `${step.opId}-${Math.random().toString(36).slice(2, 9)}`,
uid: randomId(step.opId),
args: step.args.map((a) => ({ ...a })),
}));
}
3 changes: 2 additions & 1 deletion src/lib/share.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { OperationDef, RecipeStep } from '../engine';
import { randomId } from '../engine';


const VERSION = 1;
Expand Down Expand Up @@ -152,7 +153,7 @@ export async function decodeShare(
if (!definition) continue;

steps.push({
uid: `${entry.op}-${Math.random().toString(36).slice(2, 9)}`,
uid: randomId(entry.op),
opId: entry.op,
args: definition.args.map((arg) => {
const toggle = entry.toggles?.[arg.name];
Expand Down
11 changes: 6 additions & 5 deletions src/store/useStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
inspect,
listOperations,
MAX_AUTO_DEPTH,
randomId,
rewrap,
toChain,
type BakeResult,
Expand Down Expand Up @@ -195,7 +196,7 @@ function encodingFor(payload: string, current: string): string {

function newStep(op: OperationDef): RecipeStep {
return {
uid: `${op.id}-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`,
uid: randomId(op.id),
opId: op.id,
args: op.args.map((a) => ({ ...a })),
disabled: false,
Expand Down Expand Up @@ -441,7 +442,7 @@ export const useStore = create<State>((set, get) => ({
if (steps.length === 0) return;

const result = rewrap(steps, operations, () =>
`rewrap-${Date.now().toString(36)}${Math.random().toString(36).slice(2, 6)}`,
randomId('rewrap'),
);
if (!result.ok) {
set({ rewrapBlockers: result.blockers });
Expand Down Expand Up @@ -581,7 +582,7 @@ export const useStore = create<State>((set, get) => ({
set({
steps: steps.map((s) => ({
...s,
uid: `${s.opId}-${Math.random().toString(36).slice(2, 8)}`,
uid: randomId(s.opId),
args: s.args.map((a) => ({ ...a })),
})),
breakpoints: [],
Expand Down Expand Up @@ -624,7 +625,7 @@ export const useStore = create<State>((set, get) => ({
set({
steps: hint.steps.map((s) => ({
...s,
uid: `${s.opId}-${Math.random().toString(36).slice(2, 8)}`,
uid: randomId(s.opId),
args: s.args.map((a) => ({ ...a })),
})),
breakpoints: [],
Expand All @@ -640,7 +641,7 @@ export const useStore = create<State>((set, get) => ({
set({
steps: steps.map((step) => ({
...step,
uid: `${step.opId}-${Math.random().toString(36).slice(2, 8)}`,
uid: randomId(step.opId),
args: step.args.map((a) => ({ ...a })),
})),
breakpoints: [],
Expand Down