From d657c592ebba89d6da4826ed5540ca8558d7c06e Mon Sep 17 00:00:00 2001 From: Jona Date: Sat, 17 Oct 2026 00:24:23 -0400 Subject: [PATCH 1/2] feat: add bounded read-only preview for large CSV files (#80) --- README.md | 14 +- media/large-preview.js | 30 ++++ package.json | 34 +++++ src/bounded-csv-reader.ts | 220 ++++++++++++++++++++++++++++ src/commands.ts | 2 + src/extension.ts | 5 +- src/large-file-preview.ts | 167 +++++++++++++++++++++ src/test/bounded-csv-reader.test.ts | 157 ++++++++++++++++++++ src/test/edit-integrity-harness.ts | 17 ++- src/test/large-file-preview.test.ts | 175 ++++++++++++++++++++++ 10 files changed, 817 insertions(+), 4 deletions(-) create mode 100644 media/large-preview.js create mode 100644 src/bounded-csv-reader.ts create mode 100644 src/large-file-preview.ts create mode 100644 src/test/bounded-csv-reader.test.ts create mode 100644 src/test/large-file-preview.test.ts diff --git a/README.md b/README.md index f9dee2e..a76b87c 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Working with CSV files shouldn’t be a chore. With CSV, you get: - **Clickable Links:** URLs in cells are automatically detected and displayed as clickable links. Ctrl/Cmd+click to open them in your browser. - **Preserved CSV Integrity:** All modifications respect CSV formatting—no unwanted extra characters or formatting issues. - **Optimized for Performance:** Designed for medium-sized datasets, ensuring a smooth editing experience without compromising on functionality. -- **Large File Support:** Loads big CSVs in chunks so even large datasets open quickly. +- **Large File Support:** The editable view chunks table rendering; an optional read-only preview bounds source reading and displays one page at a time. - **CSV/TSV/TAB/PSV Support:** `.csv`, `.tsv`, `.tab`, and `.psv` files are recognized automatically. Defaults: comma for CSV, tab for TSV/TAB, pipe for PSV. --- @@ -83,6 +83,7 @@ Cursor (built on VS Code 1.99) and the latest VS Code releases (1.102). Open the Command Palette and search for: - `CSV: Toggle Extension On/Off` (`csv.toggleExtension`) +- `CSV: Open Large File Preview (Read-only)` (`csv.openLargeFilePreview`) - `CSV: Toggle First Row as Header` (`csv.toggleHeader`) - `CSV: Toggle Serial Index Column` (`csv.toggleSerialIndex`) - `CSV: Change CSV Separator` (`csv.changeSeparator`) @@ -112,6 +113,7 @@ Global (Settings UI or `settings.json`): - `csv.defaultSeparator` (string, default `,`): Fallback separator. Use `\\t` in `settings.json` for tabs. - `csv.separatorByExtension` (object): Extension-to-separator mapping (defaults include `.csv`→`,`, `.tsv`/`.tab`→tab, `.psv`→`|`). - `csv.maxFileSizeMB` (number, default `10`): Soft limit for opening files in CSV view. If exceeded, CSV prompts: `Cancel`, `Continue This Time`, or `Ignore Forever` (sets this setting to `0`). +- `csv.largeFilePreviewRows` (integer, default `1000`, range `1`–`10000`): Maximum records per read-only preview page; byte and cell limits can make a page smaller. - Per-file encoding: `CSV: Change File Encoding` opens VS Code's encoding picker. Use **Reopen with Encoding** and select the source file's encoding to correct incorrectly displayed characters before editing or saving. Per-file (stored by the extension; set via commands): @@ -133,6 +135,16 @@ For example, the original attachment in [issue #76](https://github.com/jonarapha --- +## Large File Preview + +To inspect a very large saved file without creating a full VS Code text document, right-click a CSV/TSV/TAB/PSV file in Explorer and choose **CSV: Open Large File Preview (Read-only)**. The same command in the Command Palette opens a file picker. Use this direct path before opening the file in an editable tab; switching an already-open large text document to the preview does not undo memory already allocated by VS Code. + +The preview reads the saved bytes in bounded chunks and displays one page at a time. **Next**, **Previous**, and **Back to Start** navigate complete CSV records, including quoted multiline cells. The first record is displayed as data. It has no editing, global search, sorting, or filtering controls, and never writes to the source. Unsaved edits in another editor are not part of the saved-byte preview. + +`csv.largeFilePreviewRows` limits a page to 1–10,000 records (default 1,000). Independent limits cap a page at 512 KiB of source records and 20,000 cells, one record at 256 KiB, and a record at 1,000 columns. Pages may contain fewer records. Oversized or malformed records produce a visible error instead of being truncated. Only the current page and at most 32 page positions are retained; Previous reaches the retained history, and Back to Start always restarts the file. If the file changes between reads, Back to Start reloads it. + +This preview currently supports **valid UTF-8 and host-local `file:` URIs**. It decodes saved bytes directly in the extension host instead of requesting VS Code's decoded TextDocument. A UTF-8 BOM is accepted; other encodings must use the normal editor's encoding workflow. The extension host can be remote; arbitrary remote filesystem providers are unsupported. Separators follow the per-file override or extension/default settings; automatic content detection falls back to the extension mapping in preview mode. Normal editable CSV views retain their existing full-document behavior. + ## Editing Modes and Shortcuts - Quick edit: diff --git a/media/large-preview.js b/media/large-preview.js new file mode 100644 index 0000000..d758553 --- /dev/null +++ b/media/large-preview.js @@ -0,0 +1,30 @@ +const vscode = acquireVsCodeApi(); +const status = document.getElementById('status'); +const records = document.getElementById('records'); +const error = document.getElementById('error'); +const buttons = ['start', 'previous', 'next'].map(id => document.getElementById(id)); +const busy = () => { + buttons.forEach(button => { button.disabled = true; }); + status.textContent = 'Loading saved file…'; +}; +buttons.forEach(button => { + button.addEventListener('click', () => { + busy(); + vscode.postMessage({ type: button.id }); + }); +}); +window.addEventListener('message', event => { + if (event.data.type !== 'page') return; + const message = event.data; + // Markup is built by the extension from escaped cell values. Replacing the + // page bounds DOM growth regardless of how many pages have been visited. + records.innerHTML = message.html; + records.scrollTop = records.scrollLeft = 0; + status.textContent = message.status; + error.textContent = message.error || ''; + buttons[0].disabled = false; + buttons[1].disabled = !message.canBack; + buttons[2].disabled = !message.canNext; +}); +busy(); +vscode.postMessage({ type: 'ready' }); diff --git a/package.json b/package.json index a1745ea..c11e0ee 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,8 @@ "onLanguage:csv", "onLanguage:tsv", "onCommand:csv.toggleExtension", + "onCommand:csv.openLargeFilePreview", + "onCustomEditor:csv.largeFilePreview", "onCommand:csv.toggleHeader", "onCommand:csv.toggleSerialIndex", "onCommand:csv.changeSeparator", @@ -64,6 +66,10 @@ } ], "commands": [ + { + "command": "csv.openLargeFilePreview", + "title": "CSV: Open Large File Preview (Read-only)" + }, { "command": "csv.toggleExtension", "title": "CSV: Toggle Extension On/Off" @@ -207,10 +213,38 @@ "default": 10, "minimum": 0, "description": "Soft max size in MB for opening files in CSV view. 0 disables the limit. When exceeded, CSV prompts to cancel, continue this time, or disable the limit." + }, + "csv.largeFilePreviewRows": { + "type": "integer", + "default": 1000, + "minimum": 1, + "maximum": 10000, + "scope": "resource", + "description": "Maximum records per read-only Large File Preview page. Byte/cell safety limits can produce smaller pages." } } }, + "menus": { + "explorer/context": [ + { + "command": "csv.openLargeFilePreview", + "when": "resourceScheme == file && resourceExtname =~ /\\.(csv|tsv|tab|psv)$/i && config.csv.enabled", + "group": "navigation" + } + ] + }, "customEditors": [ + { + "viewType": "csv.largeFilePreview", + "displayName": "CSV Large File Preview (Read-only)", + "priority": "option", + "selector": [ + { "filenamePattern": "*.csv" }, + { "filenamePattern": "*.tsv" }, + { "filenamePattern": "*.tab" }, + { "filenamePattern": "*.psv" } + ] + }, { "viewType": "csv.editor", "displayName": "CSV", diff --git a/src/bounded-csv-reader.ts b/src/bounded-csv-reader.ts new file mode 100644 index 0000000..494c5a8 --- /dev/null +++ b/src/bounded-csv-reader.ts @@ -0,0 +1,220 @@ +import { promises as fs } from 'fs'; +import { TextDecoder } from 'util'; +import Papa from 'papaparse'; + +export const previewLimits = { + readBytes: 64 * 1024, + pageBytes: 512 * 1024, + recordBytes: 256 * 1024, + pageCells: 20000, + columns: 1000, + history: 32 +}; + +export type CsvPreviewPage = { + rows: string[][]; + offset: number; + nextOffset: number; + done: boolean; + version: string; + bytesRead: number; +}; + +export type CsvPreviewReadOptions = { + delimiter: string; + offset?: number; + maxRows?: number; + expectedVersion?: string; + cancelled?: () => boolean; + // Smaller read buffers make boundary handling deterministic in unit tests. + readBytes?: number; +}; + +function version(stat: { dev: number; ino: number; size: number; mtimeMs: number }): string { + return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`; +} + +export async function readCsvPreviewPage(filePath: string, options: CsvPreviewReadOptions): Promise { + if (options.delimiter.length !== 1 || options.delimiter.charCodeAt(0) > 127 || /[\r\n"\0]/.test(options.delimiter)) { + throw new Error('Large File Preview requires a single ASCII separator. Set a separator override or use the editable CSV view.'); + } + const checkCancelled = () => { + if (options.cancelled?.()) { + throw new Error('Large File Preview was cancelled.'); + } + }; + checkCancelled(); + const handle = await fs.open(filePath, 'r'); + try { + const stat = await handle.stat(); + if (!stat.isFile()) { + throw new Error('Large File Preview can only read regular files.'); + } + const fileVersion = version(stat); + if (options.expectedVersion && options.expectedVersion !== fileVersion) { + throw new Error('The file changed. Choose Back to Start to reload it.'); + } + const offset = options.offset ?? 0; + if (!Number.isSafeInteger(offset) || offset < 0 || offset > stat.size) { + throw new Error('Invalid preview position. Choose Back to Start.'); + } + const requestedRows = Math.trunc(Number(options.maxRows ?? 1000)); + const maxRows = Number.isFinite(requestedRows) ? Math.min(10000, Math.max(1, requestedRows)) : 1000; + const readSize = Math.min(previewLimits.readBytes, Math.max(1, Math.trunc(options.readBytes ?? previewLimits.readBytes))); + const readBuffer = Buffer.alloc(readSize); + const recordBuffer = Buffer.alloc(previewLimits.recordBytes); + const rows: string[][] = []; + const decoder = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }); + const delimiter = options.delimiter.charCodeAt(0); + let position = offset; + let recordStart = offset; + let recordLength = 0; + let recordFields = 1; + let usedBytes = 0; + let usedCells = 0; + let bytesRead = 0; + let state: 'start' | 'plain' | 'quoted' | 'closed' = 'start'; + let pendingCr = false; + let stopped = false; + let nextOffset = offset; + + if (offset === 0) { + const signature = Buffer.alloc(3); + const first = await handle.read(signature, 0, 3, 0); + bytesRead += first.bytesRead; + if ((signature[0] === 0xFF && signature[1] === 0xFE) || (signature[0] === 0xFE && signature[1] === 0xFF)) { + throw new Error('Large File Preview supports UTF-8 files only. Use the normal editor and Reopen with Encoding for this file.'); + } + if (first.bytesRead === 3 && signature.equals(Buffer.from([0xEF, 0xBB, 0xBF]))) { + position = recordStart = 3; + } + } + + const finishRecord = (end: number, terminatorBytes: number) => { + if (usedBytes + recordLength > previewLimits.pageBytes && rows.length > 0) { + stopped = true; + nextOffset = recordStart; + return; + } + let text: string; + try { + text = decoder.decode(recordBuffer.subarray(0, recordLength - terminatorBytes)); + } catch { + throw new Error('Large File Preview supports valid UTF-8 only. Reopen the original file with its source encoding in the normal editor.'); + } + if (text.includes('\0')) { + throw new Error('The file contains NUL bytes. Large File Preview supports UTF-8 text files only.'); + } + // Prefix an empty field so Papa does not strip an intentional U+FEFF at + // the start of a later record when parsing records independently. + const parsed = Papa.parse(`${options.delimiter}${text}`, { delimiter: options.delimiter, dynamicTyping: false }); + if (parsed.errors.some(error => error.type === 'Quotes') || parsed.data.length !== 1) { + throw new Error(`Malformed CSV record at byte ${recordStart}. No partial record was displayed.`); + } + const row = parsed.data[0].slice(1); + if (row.length > previewLimits.columns) { + throw new Error(`A record exceeds the ${previewLimits.columns}-column preview limit. No fields were truncated.`); + } + if (usedCells + row.length > previewLimits.pageCells && rows.length > 0) { + stopped = true; + nextOffset = recordStart; + return; + } + rows.push(row); + usedBytes += recordLength; + usedCells += row.length; + nextOffset = recordStart = end; + recordLength = 0; + recordFields = 1; + state = 'start'; + pendingCr = false; + stopped = rows.length >= maxRows || usedBytes >= previewLimits.pageBytes || usedCells >= previewLimits.pageCells; + }; + + while (!stopped && position < stat.size) { + checkCancelled(); + const read = await handle.read(readBuffer, 0, readBuffer.length, position); + bytesRead += read.bytesRead; + if (read.bytesRead === 0) { + break; + } + for (let index = 0; index < read.bytesRead && !stopped; index++) { + const byte = readBuffer[index]; + if (pendingCr && byte !== 10) { + finishRecord(position, 1); + if (stopped) { + break; + } + } + if (recordLength >= recordBuffer.length) { + throw new Error(`A record exceeds the ${previewLimits.recordBytes / 1024} KiB preview limit. No partial record was displayed.`); + } + recordBuffer[recordLength++] = byte; + position++; + if (pendingCr) { + finishRecord(position, 2); + } else if (state === 'quoted') { + if (byte === 34) { + state = 'closed'; + } + } else if (state === 'closed' && byte === 34) { + state = 'quoted'; + } else if (byte === 13) { + pendingCr = true; + } else if (byte === 10) { + finishRecord(position, 1); + } else if (byte === delimiter) { + recordFields++; + if (recordFields > previewLimits.columns) { + throw new Error(`A record exceeds the ${previewLimits.columns}-column preview limit. No fields were truncated.`); + } + state = 'start'; + } else if (state === 'start' && byte === 34) { + state = 'quoted'; + } else if (state === 'closed') { + if (byte !== 32 && byte !== 9) { + throw new Error(`Malformed quoted field at byte ${position - 1}. No partial record was displayed.`); + } + } else { + state = 'plain'; + } + } + } + if (!stopped && recordLength > 0) { + if (state === 'quoted') { + throw new Error(`Unterminated quoted record at byte ${recordStart}. No partial record was displayed.`); + } + finishRecord(position, pendingCr ? 1 : 0); + } + if (rows.length === 0) { + nextOffset = position; + } + checkCancelled(); + if (version(await fs.stat(filePath)) !== fileVersion || version(await handle.stat()) !== fileVersion) { + throw new Error('The file changed while reading. Choose Back to Start to reload it.'); + } + return { rows, offset, nextOffset, done: nextOffset >= stat.size, version: fileVersion, bytesRead }; + } finally { + await handle.close(); + } +} + +export type PreviewPosition = { offset: number; record: number }; + +export class PreviewHistory { + private positions: PreviewPosition[] = [{ offset: 0, record: 0 }]; + private index = 0; + get current(): PreviewPosition { return this.positions[this.index]; } + get canGoBack(): boolean { return this.index > 0; } + get size(): number { return this.positions.length; } + reset(): void { this.positions = [{ offset: 0, record: 0 }]; this.index = 0; } + previous(): void { if (this.canGoBack) { this.index--; } } + next(position: PreviewPosition): void { + this.positions.splice(this.index + 1); + this.positions.push(position); + if (this.positions.length > previewLimits.history) { + this.positions.shift(); + } + this.index = this.positions.length - 1; + } +} diff --git a/src/commands.ts b/src/commands.ts index f0feb8d..2a25700 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -1,6 +1,7 @@ import { getFonts } from 'font-list'; import * as vscode from 'vscode'; import { CsvEditorProvider } from './CsvEditorProvider'; +import { openLargeFilePreview } from './large-file-preview'; function displaySeparator(separator: string): string { return separator === '\t' ? '\\t' : separator; @@ -21,6 +22,7 @@ async function toggleBooleanConfig(key: string, defaultVal: boolean, messagePref export function registerCsvCommands(context: vscode.ExtensionContext) { context.subscriptions.push( + vscode.commands.registerCommand('csv.openLargeFilePreview', openLargeFilePreview), vscode.commands.registerCommand('csv.toggleExtension', () => toggleBooleanConfig('enabled', true, 'CSV extension') ), diff --git a/src/extension.ts b/src/extension.ts index 50ec42b..68a7213 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -1,6 +1,7 @@ import * as vscode from 'vscode'; import { CsvEditorProvider } from './CsvEditorProvider'; import { registerCsvCommands } from './commands'; +import { LargeFilePreviewProvider } from './large-file-preview'; export function activate(context: vscode.ExtensionContext) { console.log('CSV: Extension activated'); @@ -30,6 +31,7 @@ export function activate(context: vscode.ExtensionContext) { console.log(`[CSV(enable)]: group[${gi}] tab[${ti}] vt=${viewType ?? '(text?)'} uri=${path} active=${tab.isActive} preview=${tab.isPreview}`); if (!input) return; if (viewType === CsvEditorProvider.viewType) return; // already our editor + if (viewType === LargeFilePreviewProvider.viewType) { return; } if (!uri) return; const fsPath = uri.fsPath?.toLowerCase?.() || ''; const isCsvLike = fsPath.endsWith('.csv') || fsPath.endsWith('.tsv') || fsPath.endsWith('.tab') || fsPath.endsWith('.psv'); @@ -68,7 +70,7 @@ export function activate(context: vscode.ExtensionContext) { const vt = inp?.viewType; const u: vscode.Uri | undefined = inp?.uri instanceof vscode.Uri ? (inp.uri as vscode.Uri) : undefined; const key = u?.toString(); - if (key && processed.has(key) && vt !== CsvEditorProvider.viewType) { + if (key && processed.has(key) && vt !== CsvEditorProvider.viewType && vt !== LargeFilePreviewProvider.viewType) { console.log(`[CSV(enable)]: final-sweep closing group[${gi}] tab[${ti}] uri=${u?.fsPath} vt=${vt}`); stale.push(t); } @@ -107,6 +109,7 @@ export function activate(context: vscode.ExtensionContext) { // Register the custom editor provider for CSV/TSV context.subscriptions.push( + vscode.window.registerCustomEditorProvider(LargeFilePreviewProvider.viewType, new LargeFilePreviewProvider(context)), vscode.window.registerCustomEditorProvider( CsvEditorProvider.viewType, new CsvEditorProvider(context), diff --git a/src/large-file-preview.ts b/src/large-file-preview.ts new file mode 100644 index 0000000..a54fc0b --- /dev/null +++ b/src/large-file-preview.ts @@ -0,0 +1,167 @@ +import * as vscode from 'vscode'; +import * as path from 'path'; +import { randomBytes } from 'crypto'; +import { CsvEditorProvider } from './CsvEditorProvider'; +import { CsvPreviewPage, PreviewHistory, readCsvPreviewPage } from './bounded-csv-reader'; + +const escapeHtml = (value: string): string => value.replace(/[&<>"']/g, char => ({ + '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' +}[char] as string)); + +export function renderPreviewRows(rows: string[][], firstRecord: number): string { + if (rows.length === 0) { + return '

No records in this file.

'; + } + const width = Math.max(...rows.map(row => row.length)); + const headings = Array.from({ length: width }, (_, index) => `Column ${index + 1}`).join(''); + const body = rows.map((row, index) => `${firstRecord + index + 1}${row.map(value => `${escapeHtml(value)}`).join('')}`).join(''); + return `${headings}${body}
Record
`; +} + +class PreviewDocument implements vscode.CustomDocument { + disposed = false; + constructor(readonly uri: vscode.Uri) {} + dispose(): void { this.disposed = true; } +} + +export class LargeFilePreviewProvider implements vscode.CustomReadonlyEditorProvider { + static readonly viewType = 'csv.largeFilePreview'; + constructor(private readonly context: vscode.ExtensionContext) {} + + async openCustomDocument(uri: vscode.Uri): Promise { + if (uri.scheme !== 'file') { + throw new Error('Large File Preview currently supports host-local file URIs only. Use the normal CSV editor for this location.'); + } + return new PreviewDocument(uri); + } + + async resolveCustomEditor(document: PreviewDocument, panel: vscode.WebviewPanel, token: vscode.CancellationToken): Promise { + panel.webview.options = { + enableScripts: true, + localResourceRoots: [vscode.Uri.file(path.join(this.context.extensionPath, 'media'))] + }; + panel.webview.html = this.html(panel.webview); + const history = new PreviewHistory(); + let page: CsvPreviewPage | undefined; + let fileVersion: string | undefined; + let selectedDelimiter: string | undefined; + let lastError = ''; + let disposed = false; + let pending = Promise.resolve(); + const cancelled = () => disposed || document.disposed || token.isCancellationRequested; + const send = (error?: string) => { + if (cancelled()) { + return; + } + const first = history.current.record; + void panel.webview.postMessage({ + type: 'page', + html: page ? renderPreviewRows(page.rows, first) : '', + status: page?.rows.length ? `Saved file · Records ${first + 1}–${first + page.rows.length} · Read-only` : 'Saved file · Read-only', + canBack: history.canGoBack, + canNext: !!page && !page.done && !error, + error: error ?? '' + }); + }; + const read = async (reset: boolean) => { + const config = vscode.workspace.getConfiguration('csv', document.uri); + if (!config.get('enabled', true)) { + throw new Error('CSV is disabled. Enable it before opening Large File Preview.'); + } + const settings = CsvEditorProvider.getSeparatorSettings(document.uri); + const delimiter = CsvEditorProvider.getSeparatorForUri(this.context, document.uri) + ?? CsvEditorProvider.resolveInheritedSeparator(document.uri.fsPath, '', settings); + if (!reset && selectedDelimiter !== delimiter) { + throw new Error('The separator changed. Choose Back to Start to reload the preview.'); + } + selectedDelimiter = delimiter; + page = await readCsvPreviewPage(document.uri.fsPath, { + delimiter, + offset: history.current.offset, + maxRows: config.get('largeFilePreviewRows', 1000), + expectedVersion: reset ? undefined : fileVersion, + cancelled + }); + fileVersion = page.version; + lastError = ''; + send(); + }; + panel.webview.onDidReceiveMessage(message => { + pending = pending.then(async () => { + if (cancelled()) { + return; + } + if (message.type === 'ready') { + if (page) { + send(); + } else if (lastError) { + send(lastError); + } else { + await read(true); + } + } else if (message.type === 'start') { + history.reset(); + page = undefined; + fileVersion = undefined; + lastError = ''; + await read(true); + } else if (message.type === 'next' && page && !page.done) { + history.next({ offset: page.nextOffset, record: history.current.record + page.rows.length }); + await read(false); + } else if (message.type === 'previous' && history.canGoBack) { + history.previous(); + await read(false); + } + }).catch(error => { + // Discard stale rows if navigation or source validation fails. The error + // remains visible; Back to Start can always retry a changed file. + page = undefined; + lastError = error instanceof Error ? error.message : 'Unable to read this preview page.'; + send(lastError); + }); + return pending; + }); + panel.onDidDispose(() => { disposed = true; }); + } + + private html(webview: vscode.Webview): string { + const nonce = randomBytes(16).toString('hex'); + const script = webview.asWebviewUri(vscode.Uri.file(path.join(this.context.extensionPath, 'media', 'large-preview.js'))); + return ` + + + + +

Large File Preview · UTF-8 · Read-only. Each page replaces the previous page. The first record is shown as data.

+
+`; + } +} + +export async function openLargeFilePreview(uri?: vscode.Uri): Promise { + if (!vscode.workspace.getConfiguration('csv').get('enabled', true)) { + void vscode.window.showInformationMessage('Enable CSV before opening Large File Preview.'); + return; + } + const target = uri instanceof vscode.Uri ? uri : (await vscode.window.showOpenDialog({ + canSelectMany: false, + openLabel: 'Open Read-only Preview', + filters: { 'CSV/TSV/PSV': ['csv', 'tsv', 'tab', 'psv'] } + }))?.[0]; + if (!target) { + return; + } + if (target.scheme !== 'file') { + void vscode.window.showErrorMessage('Large File Preview currently supports host-local files only. Use the normal CSV editor for this location.'); + return; + } + await vscode.commands.executeCommand('vscode.openWith', target, LargeFilePreviewProvider.viewType, { preview: false }); +} diff --git a/src/test/bounded-csv-reader.test.ts b/src/test/bounded-csv-reader.test.ts new file mode 100644 index 0000000..56bfe9e --- /dev/null +++ b/src/test/bounded-csv-reader.test.ts @@ -0,0 +1,157 @@ +import assert from 'assert'; +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { describe, it } from 'node:test'; +import Papa from 'papaparse'; +import { previewLimits, PreviewHistory, readCsvPreviewPage } from '../bounded-csv-reader'; + +async function fixture(value: string | Buffer, run: (file: string) => Promise) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'csv-bounded-')); + const file = path.join(directory, 'source.csv'); + try { + await fs.writeFile(file, value); + await run(file); + } finally { + await fs.rm(directory, { recursive: true, force: true }); + } +} + +describe('Bounded CSV reader', () => { + for (const delimiter of [',', '\t', '|']) { + for (const readBytes of [1, 2, 3, 7, 64]) { + it(`preserves UTF-8, CRLF and quoted boundaries with ${JSON.stringify(delimiter)} / ${readBytes}-byte reads`, async () => { + const data = [['name', 'value'], ['éä😀', `inside${delimiter}field`], ['two\r\nlines', 'a"b'], ['\uFEFFintentional', 'end']]; + const source = '\uFEFF' + Papa.unparse(data, { delimiter, newline: '\r\n' }) + '\r\n'; + await fixture(source, async file => { + const actual: string[][] = []; + let offset = 0; + let fileVersion: string | undefined; + let done = false; + while (!done) { + const page = await readCsvPreviewPage(file, { delimiter, offset, readBytes, maxRows: 1, expectedVersion: fileVersion }); + actual.push(...page.rows); + assert.ok(page.nextOffset > offset); + offset = page.nextOffset; + fileVersion = page.version; + done = page.done; + } + assert.deepStrictEqual(actual, data); + assert.strictEqual(offset, Buffer.byteLength(source)); + }); + }); + } + } + + for (const source of ['', '\uFEFF', '\n', '\r\n', 'a,b\r', 'a,b\nc,d', 'a,b\n\n']) { + it(`handles EOF and empty records in ${JSON.stringify(source)}`, async () => { + await fixture(source, async file => { + const page = await readCsvPreviewPage(file, { delimiter: ',', readBytes: 1 }); + const expected = source === '' || source === '\uFEFF' ? [] + : source === '\n' || source === '\r\n' ? [['']] + : source === 'a,b\nc,d' ? [['a', 'b'], ['c', 'd']] + : source === 'a,b\n\n' ? [['a', 'b'], ['']] + : [['a', 'b']]; + assert.deepStrictEqual(page.rows, expected); + assert.ok(page.done); + }); + }); + } + + it('reads only a bounded prefix and never a whole large file', async () => { + const source = 'alpha,beta\n'.repeat(200000); + await fixture(source, async file => { + const page = await readCsvPreviewPage(file, { delimiter: ',', maxRows: 1000 }); + assert.strictEqual(page.rows.length, 1000); + assert.ok(!page.done); + assert.ok(page.bytesRead <= previewLimits.readBytes + 3); + assert.ok(page.bytesRead < Buffer.byteLength(source) / 10); + }); + }); + + it('caps source bytes and cells independently of the requested row count', async () => { + const source = `${'x'.repeat(200000)}\n`.repeat(4); + await fixture(source, async file => { + const page = await readCsvPreviewPage(file, { delimiter: ',', maxRows: 10000 }); + assert.strictEqual(page.rows.length, 2); + assert.strictEqual(page.nextOffset, 400002); + assert.ok(page.bytesRead <= previewLimits.pageBytes + previewLimits.recordBytes + previewLimits.readBytes + 3); + }); + await fixture(`${Array(1000).fill('x').join(',')}\n`.repeat(30), async file => { + const page = await readCsvPreviewPage(file, { delimiter: ',', maxRows: 10000 }); + assert.strictEqual(page.rows.length, 20); + assert.strictEqual(page.rows.flat().length, previewLimits.pageCells); + assert.ok(!page.done); + }); + }); + + for (const [source, error] of [ + ['a,"unfinished\n', /Unterminated quoted record/], + ['a,"closed"junk\n', /Malformed quoted field/], + ['x'.repeat(previewLimits.recordBytes + 1), /No partial record was displayed/], + [Array(1001).fill('x').join(','), /No fields were truncated/] + ] as const) { + it(`rejects malformed/oversized input visibly (${error.source})`, async () => { + await fixture(source, file => assert.rejects(readCsvPreviewPage(file, { delimiter: ',' }), error)); + }); + } + + it('rejects non-UTF8, UTF16 BOMs, NULs and unsupported separators without guessing', async () => { + for (const bytes of [Buffer.from([0x84, 10]), Buffer.from([0xFF, 0xFE, 97, 0]), Buffer.from('a\0b')]) { + await fixture(bytes, file => assert.rejects(readCsvPreviewPage(file, { delimiter: ',' }), /UTF-8/)); + } + await fixture('a,b', file => assert.rejects(readCsvPreviewPage(file, { delimiter: '||' }), /single ASCII separator/)); + }); + + it('detects source changes between pages', async () => { + await fixture('a,b\n1,2\n', async file => { + const first = await readCsvPreviewPage(file, { delimiter: ',', maxRows: 1 }); + await fs.appendFile(file, '3,4\n'); + await assert.rejects(readCsvPreviewPage(file, { delimiter: ',', offset: first.nextOffset, expectedVersion: first.version }), /file changed/); + }); + }); + + it('closes the file descriptor after success, cancellation and malformed records', async () => { + const originalOpen = fs.open; + const handles: Array<{ closed: number }> = []; + (fs as any).open = async (...args: Parameters) => { + const handle = await originalOpen(...args); + const state = { closed: 0 }; + handles.push(state); + return new Proxy(handle, { + get(target, key) { + if (key === 'close') { + return async () => { state.closed++; await target.close(); }; + } + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + } + }); + }; + try { + await fixture('a,b\n1,2\n', async file => { + await readCsvPreviewPage(file, { delimiter: ',' }); + let checks = 0; + await assert.rejects(readCsvPreviewPage(file, { delimiter: ',', readBytes: 1, cancelled: () => ++checks > 2 }), /cancelled/); + }); + await fixture('"unfinished', file => assert.rejects(readCsvPreviewPage(file, { delimiter: ',' }), /Unterminated/)); + assert.deepStrictEqual(handles.map(handle => handle.closed), [1, 1, 1]); + } finally { + fs.open = originalOpen; + } + }); + + it('retains only a bounded navigation history', () => { + const history = new PreviewHistory(); + for (let index = 1; index < 1000; index++) { + history.next({ offset: index * 100, record: index }); + } + assert.strictEqual(history.size, previewLimits.history); + let previous = 0; + while (history.canGoBack) { history.previous(); previous++; } + assert.strictEqual(previous, previewLimits.history - 1); + history.reset(); + assert.deepStrictEqual(history.current, { offset: 0, record: 0 }); + assert.strictEqual(history.size, 1); + }); +}); diff --git a/src/test/edit-integrity-harness.ts b/src/test/edit-integrity-harness.ts index 6b1e5b9..c05184c 100644 --- a/src/test/edit-integrity-harness.ts +++ b/src/test/edit-integrity-harness.ts @@ -27,8 +27,12 @@ const vscodeStub = { }, showInformationMessage() {}, tabGroups: { - get all() { return [{ tabs: [{ input: { uri: activeHarness.document.uri, viewType: 'csv.editor' } }] }]; }, + get all() { return activeHarness.tabGroups ?? [{ tabs: [{ input: { uri: activeHarness.document.uri, viewType: 'csv.editor' } }] }]; }, close: async () => activeHarness.closeResult + }, + registerCustomEditorProvider: (viewType: string) => { + activeHarness.providers.push(viewType); + return { dispose() {} }; } }, commands: { @@ -47,12 +51,17 @@ const vscodeStub = { } }, workspace: { - getConfiguration: () => ({ get: (key: string, fallback: unknown) => key === 'maxFileSizeMB' ? 0 : fallback }), + getConfiguration: () => ({ get: (key: string, fallback: unknown) => + key in activeHarness.configuration ? activeHarness.configuration[key] : key === 'maxFileSizeMB' ? 0 : fallback }), applyEdit: (edit: { text: string }) => activeHarness.applyEdit(edit.text), onDidChangeTextDocument: (listener: (event: unknown) => void) => { activeHarness.changeListeners.push(listener); return { dispose() {} }; }, + onDidChangeConfiguration: (listener: (event: any) => void) => { + activeHarness.configurationListeners.push(listener); + return { dispose() {} }; + }, onWillSaveTextDocument: (listener: (event: unknown) => void) => { activeHarness.saveListeners.push(listener); return { dispose() {} }; @@ -131,6 +140,10 @@ export async function createEditorHarness(options: { }; const harness = { document, panel, messages, changeListeners, saveListeners, + configuration: {} as Record, + configurationListeners: [] as Array<(event: any) => void>, + providers: [] as string[], + tabGroups: undefined as any[] | undefined, warnings: [] as string[], warningChoice: options.warningChoice, commandError: undefined as Error | undefined, diff --git a/src/test/large-file-preview.test.ts b/src/test/large-file-preview.test.ts new file mode 100644 index 0000000..16a8dfe --- /dev/null +++ b/src/test/large-file-preview.test.ts @@ -0,0 +1,175 @@ +import assert from 'assert'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { describe, it } from 'node:test'; +import { createEditorHarness } from './edit-integrity-harness'; +import { LargeFilePreviewProvider, renderPreviewRows } from '../large-file-preview'; +import { activate } from '../extension'; +const { JSDOM } = require('jsdom'); + +async function previewFixture(run: (fixture: any) => Promise) { + const host = await createEditorHarness(); + host.configuration.largeFilePreviewRows = 1; + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'csv-preview-')); + const file = path.join(directory, 'preview.csv'); + fs.writeFileSync(file, 'name,value\none,two\nthree,four\n'); + const Uri = host.document.uri.constructor as any; + const provider = new LargeFilePreviewProvider({ extensionPath: process.cwd(), workspaceState: { get: (_key: string, fallback: unknown) => fallback } } as any); + const document = await provider.openCustomDocument(new Uri(file)); + const messages: any[] = []; + let receive!: (message: unknown) => Promise; + let dispose!: () => void; + const panel = { + webview: { + html: '', options: {}, + asWebviewUri: (uri: unknown) => String(uri), + postMessage: (message: unknown) => { messages.push(message); return Promise.resolve(true); }, + onDidReceiveMessage: (listener: typeof receive) => { receive = listener; } + }, + onDidDispose: (listener: () => void) => { dispose = listener; } + }; + try { + await provider.resolveCustomEditor(document, panel as any, { isCancellationRequested: false } as any); + await run({ host, file, provider, document, panel, messages, receive: (message: unknown) => receive(message), dispose: () => dispose() }); + } finally { + document.dispose(); + fs.rmSync(directory, { recursive: true, force: true }); + } +} + +describe('Read-only large file provider', () => { + it('pages complete records, reports source changes and reloads on explicit Start', async () => { + await previewFixture(async fixture => { + await fixture.receive({ type: 'ready' }); + assert.ok(fixture.messages.at(-1).html.includes('name')); + await fixture.receive({ type: 'next' }); + assert.ok(fixture.messages.at(-1).html.includes('one')); + assert.ok(!fixture.messages.at(-1).html.includes('name')); + fs.appendFileSync(fixture.file, 'changed,row\n'); + await fixture.receive({ type: 'next' }); + assert.match(fixture.messages.at(-1).error, /file changed/); + assert.strictEqual(fixture.messages.at(-1).html, ''); + await fixture.receive({ type: 'previous' }); + assert.match(fixture.messages.at(-1).error, /file changed/); + await fixture.receive({ type: 'start' }); + assert.strictEqual(fixture.messages.at(-1).error, ''); + assert.ok(fixture.messages.at(-1).html.includes('name')); + }); + }); + + it('does not read or post after panel/document disposal and rejects remote file providers', async () => { + await previewFixture(async fixture => { + fixture.dispose(); + await fixture.receive({ type: 'ready' }); + assert.strictEqual(fixture.messages.length, 0); + const Uri = fixture.host.document.uri.constructor; + await assert.rejects(fixture.provider.openCustomDocument(new Uri('/remote/file.csv', 'vscode-remote')), /host-local/); + }); + }); + + it('respects csv.enabled before any preview data read', async () => { + await previewFixture(async fixture => { + fixture.host.configuration.enabled = false; + await fixture.receive({ type: 'ready' }); + assert.match(fixture.messages.at(-1).error, /CSV is disabled/); + assert.strictEqual(fixture.messages.at(-1).html, ''); + }); + }); + + it('opens directly as a readonly custom document and never writes saved bytes', async () => { + await previewFixture(async fixture => { + fixture.host.registerCommands(); + const before = fs.readFileSync(fixture.file); + await fixture.host.commands.get('csv.openLargeFilePreview')(fixture.document.uri); + assert.strictEqual(fixture.host.executedCommands.at(-1).name, 'vscode.openWith'); + assert.strictEqual(fixture.host.executedCommands.at(-1).args[1], LargeFilePreviewProvider.viewType); + await fixture.receive({ type: 'ready' }); + await fixture.receive({ type: 'editCell', row: 0, col: 0, value: 'attempted mutation' }); + await fixture.receive({ type: 'next' }); + assert.deepStrictEqual(fs.readFileSync(fixture.file), before); + }); + }); + + it('keeps a readonly preview from upgrading to a full text document when CSV is enabled', async () => { + await previewFixture(async fixture => { + fixture.host.tabGroups = [{ tabs: [{ input: { uri: fixture.document.uri, viewType: LargeFilePreviewProvider.viewType } }] }]; + activate({ subscriptions: [], extensionPath: process.cwd() } as any); + fixture.host.configurationListeners[0]({ affectsConfiguration: (key: string) => key === 'csv.enabled' }); + await new Promise(resolve => setImmediate(resolve)); + assert.strictEqual(fixture.host.executedCommands.length, 0); + assert.ok(fixture.host.providers.includes(LargeFilePreviewProvider.viewType)); + }); + }); + + it('shows malformed records as errors without displaying partial data', async () => { + await previewFixture(async fixture => { + fs.writeFileSync(fixture.file, 'name,value\n"unfinished'); + await fixture.receive({ type: 'ready' }); + await fixture.receive({ type: 'next' }); + assert.match(fixture.messages.at(-1).error, /Unterminated quoted record/); + assert.strictEqual(fixture.messages.at(-1).html, ''); + assert.strictEqual(fixture.messages.at(-1).canNext, false); + }); + }); + + it('closes an in-flight read when its document is disposed and posts no late data', async () => { + await previewFixture(async fixture => { + const originalOpen = fs.promises.open; + let release!: () => void; + let reading!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { reading = resolve; }); + let closed = 0; + (fs.promises as any).open = async (...args: Parameters) => { + const handle = await originalOpen(...args); + return new Proxy(handle, { + get(target, key) { + if (key === 'read') { + return async (...readArgs: any[]) => { reading(); await gate; return (target.read as any)(...readArgs); }; + } + if (key === 'close') { + return async () => { closed++; await target.close(); }; + } + const value = Reflect.get(target, key); + return typeof value === 'function' ? value.bind(target) : value; + } + }); + }; + try { + const pending = fixture.receive({ type: 'ready' }); + await started; + fixture.document.dispose(); + release(); + await pending; + assert.strictEqual(closed, 1); + assert.strictEqual(fixture.messages.length, 0); + } finally { + release(); + fs.promises.open = originalOpen; + } + }); + }); + + it('renders escaped values and replaces rather than accumulating DOM pages', async () => { + await previewFixture(async fixture => { + const dom = new JSDOM(fixture.panel.webview.html, { runScripts: 'outside-only' }); + const messages: unknown[] = []; + dom.window.acquireVsCodeApi = () => ({ postMessage: (message: unknown) => messages.push(message) }); + dom.window.eval(fs.readFileSync(path.join(process.cwd(), 'media', 'large-preview.js'), 'utf8')); + const payload = ' & "quoted"'; + for (let page = 0; page < 100; page++) { + dom.window.dispatchEvent(new dom.window.MessageEvent('message', { data: { + type: 'page', html: renderPreviewRows([[payload, String(page)]], page), status: 'Read-only', canBack: true, canNext: true + } })); + } + assert.strictEqual(dom.window.document.querySelectorAll('#records tbody tr').length, 1); + assert.strictEqual(dom.window.document.querySelectorAll('#records img, #records script').length, 0); + assert.ok(dom.window.document.getElementById('records').textContent.includes(payload)); + assert.strictEqual(dom.window.document.querySelectorAll('[contenteditable]').length, 0); + dom.window.document.getElementById('next').click(); + assert.deepStrictEqual(messages.map((message: any) => message.type), ['ready', 'next']); + dom.window.close(); + }); + }); +}); From 98bda23d54b8514b4270ba8e7ad8afaf21119708 Mon Sep 17 00:00:00 2001 From: Jona Date: Sat, 17 Oct 2026 00:25:16 -0400 Subject: [PATCH 2/2] fix: reject unsupported automatic preview separator detection --- README.md | 2 +- src/large-file-preview.ts | 7 +++++-- src/test/large-file-preview.test.ts | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a76b87c..ee0683c 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ The preview reads the saved bytes in bounded chunks and displays one page at a t `csv.largeFilePreviewRows` limits a page to 1–10,000 records (default 1,000). Independent limits cap a page at 512 KiB of source records and 20,000 cells, one record at 256 KiB, and a record at 1,000 columns. Pages may contain fewer records. Oversized or malformed records produce a visible error instead of being truncated. Only the current page and at most 32 page positions are retained; Previous reaches the retained history, and Back to Start always restarts the file. If the file changes between reads, Back to Start reloads it. -This preview currently supports **valid UTF-8 and host-local `file:` URIs**. It decodes saved bytes directly in the extension host instead of requesting VS Code's decoded TextDocument. A UTF-8 BOM is accepted; other encodings must use the normal editor's encoding workflow. The extension host can be remote; arbitrary remote filesystem providers are unsupported. Separators follow the per-file override or extension/default settings; automatic content detection falls back to the extension mapping in preview mode. Normal editable CSV views retain their existing full-document behavior. +This preview currently supports **valid UTF-8 and host-local `file:` URIs**. It decodes saved bytes directly in the extension host instead of requesting VS Code's decoded TextDocument. A UTF-8 BOM is accepted; other encodings must use the normal editor's encoding workflow. The extension host can be remote; arbitrary remote filesystem providers are unsupported. Separators follow the per-file override or extension/default settings. Automatic separator detection requires an explicit override or a change to `csv.separatorMode`; otherwise the preview shows an actionable error. Normal editable CSV views retain their existing full-document behavior. ## Editing Modes and Shortcuts diff --git a/src/large-file-preview.ts b/src/large-file-preview.ts index a54fc0b..8d63c12 100644 --- a/src/large-file-preview.ts +++ b/src/large-file-preview.ts @@ -69,8 +69,11 @@ export class LargeFilePreviewProvider implements vscode.CustomReadonlyEditorProv throw new Error('CSV is disabled. Enable it before opening Large File Preview.'); } const settings = CsvEditorProvider.getSeparatorSettings(document.uri); - const delimiter = CsvEditorProvider.getSeparatorForUri(this.context, document.uri) - ?? CsvEditorProvider.resolveInheritedSeparator(document.uri.fsPath, '', settings); + const override = CsvEditorProvider.getSeparatorForUri(this.context, document.uri); + if (!override && settings.mode === 'auto') { + throw new Error('Large File Preview does not support automatic separator detection. Set a per-file separator override in the normal editor, or set csv.separatorMode to extension/default, then choose Back to Start.'); + } + const delimiter = override ?? CsvEditorProvider.resolveInheritedSeparator(document.uri.fsPath, '', settings); if (!reset && selectedDelimiter !== delimiter) { throw new Error('The separator changed. Choose Back to Start to reload the preview.'); } diff --git a/src/test/large-file-preview.test.ts b/src/test/large-file-preview.test.ts index 16a8dfe..f9b6d1b 100644 --- a/src/test/large-file-preview.test.ts +++ b/src/test/large-file-preview.test.ts @@ -77,6 +77,20 @@ describe('Read-only large file provider', () => { }); }); + it('rejects unsupported automatic separator mode explicitly rather than displaying wrong columns', async () => { + await previewFixture(async fixture => { + fixture.host.configuration.separatorMode = 'auto'; + await fixture.receive({ type: 'ready' }); + assert.match(fixture.messages.at(-1).error, /does not support automatic separator detection/); + assert.match(fixture.messages.at(-1).error, /csv.separatorMode/); + assert.strictEqual(fixture.messages.at(-1).html, ''); + fixture.host.configuration.separatorMode = 'extension'; + await fixture.receive({ type: 'start' }); + assert.strictEqual(fixture.messages.at(-1).error, ''); + assert.ok(fixture.messages.at(-1).html.includes('name')); + }); + }); + it('opens directly as a readonly custom document and never writes saved bytes', async () => { await previewFixture(async fixture => { fixture.host.registerCommands();