Diagnose possible encoding mismatch and preserve cancelled reopen edits

README.md

@@ -112,7 +112,7 @@ Global (Settings UI or `settings.json`):
 - `csv.defaultSeparator` (string, default `,`): Fallback separator. Use `\\t` in `settings.json` for tabs.
 - `csv.separatorByExtension` (object): Extension-to-separator mapping (defaults include `.csv`→`,`, `.tsv`/`.tab`→tab, `.psv`→`|`).
 - `csv.maxFileSizeMB` (number, default `10`): Soft limit for opening files in CSV view. If exceeded, CSV prompts: `Cancel`, `Continue This Time`, or `Ignore Forever` (sets this setting to `0`).
-- Per-file encoding: use `CSV: Change File Encoding` to set a file's encoding (e.g., `utf8`, `utf16le`, `windows1250`, `gbk`). The extension will reopen the file using the chosen encoding.
+- Per-file encoding: `CSV: Change File Encoding` opens VS Code's encoding picker. Use **Reopen with Encoding** and select the source file's encoding to correct incorrectly displayed characters before editing or saving.
 
 Per-file (stored by the extension; set via commands):
 
@@ -121,6 +121,16 @@ Per-file (stored by the extension; set via commands):
 - CSV separator override — `CSV: Change CSV Separator` (or clear it with `CSV: Reset CSV Separator (Inherit)`)
 - Hide first N rows (default `0`) — `CSV: Hide First N Rows`
 
+### File encodings and accented characters
+
+CSV displays the Unicode text supplied by VS Code; VS Code controls decoding the file's bytes and encoding saved files. If accents look wrong, run `CSV: Change File Encoding`, choose **Reopen with Encoding**, and select the encoding used by the source. Check the names before saving. Cancelling the unsaved-changes prompt stops the command and preserves your open edits.
+
+The editor shows a nonmodal warning once per opened document when it contains the replacement character `�`, offering the same encoding command. This character can be legitimate text, so the warning indicates a possible mismatch and does not guess an encoding or change any data. Some incorrect decodings produce other characters instead and will not trigger this warning.
+
+For example, the original attachment in [issue #76](https://github.com/jonaraphael/csv/issues/76) is not valid UTF-8. Explicitly decoding it as **DOS (CP 437)** or **Western European DOS (CP 850)** restores names such as `Knäckebröd` and `Thüringer`; both encodings produce the same text for that particular file, so its bytes do not uniquely identify which one created it. Use the source's documented encoding for other files. Available picker labels depend on your VS Code version.
+
+**Save with Encoding** changes how the current text is written; it cannot restore characters already decoded incorrectly. If a corrupted version was saved, reopen the original source or a backup using the correct encoding first.
+
 ---
 
 ## Editing Modes and Shortcuts

package.json

@@ -250,6 +250,7 @@
     "@typescript-eslint/parser": "^7.9.0",
     "@vscode/vsce": "^3.7.1",
     "eslint": "^8.57.0",
+    "iconv-lite": "^0.6.3",
     "jsdom": "^26.1.0",
     "typescript": "^4.4.3",
     "vscode": "^1.1.37"

src/CsvEditorProvider.ts

@@ -61,9 +61,11 @@
   private static readonly DEFAULT_MAX_FILE_SIZE_MB = 10;
   private static readonly LARGE_FILE_CONTINUE_THIS_TIME = 'Continue This Time';
   private static readonly LARGE_FILE_IGNORE_FOREVER = 'Ignore Forever';
+  private static readonly encodingWarnings = new WeakSet<vscode.TextDocument>();
 
   private isUpdatingDocument = false;
   private isSaving = false;
+  private disposed = false;
   private pendingOperation: Promise<void> = Promise.resolve();
   private currentWebviewPanel: vscode.WebviewPanel | undefined;
   private document!: vscode.TextDocument;
@@ -109,6 +111,9 @@
     if (webviewPanel.active) {
       CsvEditorProvider.currentActive = this;
     }
+    void this.warnAboutPossibleDecoding().catch(error => {
+      console.warn('CSV: Encoding warning action failed', error);
+    });
 
     webviewPanel.webview.postMessage({ type: 'focus' });
     let wasVisible = webviewPanel.visible;
@@ -209,6 +214,7 @@
     });
 
     webviewPanel.onDidDispose(() => {
+      this.disposed = true;
       changeDocumentSubscription.dispose();
       saveDocumentSubscription.dispose();
       CsvEditorProvider.editors = CsvEditorProvider.editors.filter(ed => ed !== this);
@@ -225,6 +231,22 @@
     return this.pendingOperation;
   }
 
+  private async warnAboutPossibleDecoding(): Promise<void> {
+    if (!this.document.getText().includes('\uFFFD') || CsvEditorController.encodingWarnings.has(this.document)) {
+      return;
+    }
+    CsvEditorController.encodingWarnings.add(this.document);
+    const action = 'Change File Encoding';
+    const selected = await vscode.window.showWarningMessage(
+      'CSV: This file contains replacement characters (�), which can indicate a decoding mismatch. Check the source encoding before saving.',
+      action
+    );
+    if (selected === action && !this.disposed) {
+      // Bind the action to this document even if another tab becomes active.
+      await vscode.commands.executeCommand('csv.changeEncoding', this.document.uri);
+    }
+  }
+
   private getMaxFileSizeLimitMb(config: vscode.WorkspaceConfiguration): number {
     const raw = Number(config.get<number>('maxFileSizeMB', CsvEditorController.DEFAULT_MAX_FILE_SIZE_MB));
     if (!Number.isFinite(raw) || raw <= 0) {
@@ -329,7 +351,7 @@
   }
 
   private forceReload() {
     if (!this.currentWebviewPanel) return;
     const panel = this.currentWebviewPanel;
     // First, blank the DOM to ensure a full script/style reinit on next set
     panel.webview.html = '<!DOCTYPE html><html><head><meta charset="UTF-8"></head><body></body></html>';
@@ -849,7 +871,7 @@
   }
 
   private buildFindRegex(query: string, options: { regex: boolean; wholeWord: boolean; matchCase: boolean }): RegExp | undefined {
     if (!query) return undefined;
     const useRegex = !!options.regex;
     const wholeWord = !!options.wholeWord;
     const matchCase = !!options.matchCase;
@@ -941,17 +963,17 @@
       }
       data[row][col] = '';
     } else {
       while (data.length <= row) data.push([]);
       while (data[row].length <= col) data[row].push('');
       data[row][col] = value;
     }
 
     let trimmed = false;
     if (wasEditingLastRow) {
       const isRowEmpty = (arr: string[] | undefined) => {
         if (!arr || arr.length === 0) return true;
         for (let i = 0; i < arr.length; i++) {
           if ((arr[i] ?? '') !== '') return false;
         }
         return true;
       };
@@ -989,7 +1011,7 @@
     const data = result.data as string[][];
     for (const row of data) {
       if (index > row.length) {
         while (row.length < index) row.push('');
       }
       row.splice(index, 0, '');
     }
@@ -1007,7 +1029,7 @@
   }
 
   private async insertColumns(index: number, count: number) {
     if (count <= 0) return;
     this.isUpdatingDocument = true;
     const separator = this.getSeparator();
     const text = this.document.getText();
@@ -1016,7 +1038,7 @@
     for (let k = 0; k < count; k++) {
       for (const row of data) {
         if (index > row.length) {
           while (row.length < index) row.push('');
         }
         row.splice(index, 0, '');
       }
@@ -1059,7 +1081,7 @@
   }
 
   private async deleteColumns(indices: number[]) {
     if (!indices || !indices.length) return;
     this.isUpdatingDocument = true;
     const separator = this.getSeparator();
     const text = this.document.getText();

src/commands.ts

@@ -127,10 +127,10 @@ export function registerCsvCommands(context: vscode.ExtensionContext) {
         .forEach(ed => ed.refresh());
       vscode.window.showInformationMessage(`CSV: Hiding first ${n} row(s) for this file.`);
     }),
-    vscode.commands.registerCommand('csv.changeEncoding', async () => {
+    vscode.commands.registerCommand('csv.changeEncoding', async (requestedUri?: vscode.Uri) => {
       const active = CsvEditorProvider.getActiveProvider();
-      if (!active) { vscode.window.showInformationMessage('Open a CSV/TSV/PSV file in the CSV editor.'); return; }
-      const uri = active.getDocumentUri();
+      const uri = requestedUri instanceof vscode.Uri ? requestedUri : active?.getDocumentUri();
+      if (!uri) { vscode.window.showInformationMessage('Open a CSV/TSV/PSV file in the CSV editor.'); return; }
 
       // Close any existing tabs for this URI so we can reuse the slot cleanly
       try {
@@ -144,10 +144,14 @@ export function registerCsvCommands(context: vscode.ExtensionContext) {
         });
         if (toClose.length) {
           console.log(`[CSV(encoding)]: closing ${toClose.length} tab(s) for ${uri.fsPath}`);
-          await vscode.window.tabGroups.close(toClose);
+          const closed = await vscode.window.tabGroups.close(toClose);
+          if (!closed) {
+            return;
+          }
         }
       } catch (e) {
         console.warn('[CSV(encoding)]: failed to close existing tabs', e);
+        return;
       }
 
       // Open the default text editor in-place and invoke the built-in encoding picker

src/test/edit-integrity-harness.ts

@@ -19,7 +19,33 @@ const vscodeStub = {
     replace(_uri: unknown, _range: unknown, text: string) { this.text = text; }
   },
   ColorThemeKind: { Dark: 1 },
-  window: { activeColorTheme: { kind: 1 } },
+  window: {
+    activeColorTheme: { kind: 1 },
+    showWarningMessage: async (message: string) => {
+      activeHarness.warnings.push(message);
+      return activeHarness.warningChoice;
+    },
+    showInformationMessage() {},
+    tabGroups: {
+      get all() { return [{ tabs: [{ input: { uri: activeHarness.document.uri, viewType: 'csv.editor' } }] }]; },
+      close: async () => activeHarness.closeResult
+    }
+  },
+  commands: {
+    registerCommand: (name: string, command: (...args: any[]) => unknown) => {
+      activeHarness.commands.set(name, command);
+      return { dispose() {} };
+    },
+    executeCommand: async (name: string, ...args: unknown[]) => {
+      activeHarness.executedCommands.push({ name, args });
+      if (activeHarness.commandError) {
+        throw activeHarness.commandError;
+      }
+      if (name === 'workbench.action.editor.changeEncoding') {
+        await activeHarness.commandWait;
+      }
+    }
+  },
   workspace: {
     getConfiguration: () => ({ get: (key: string, fallback: unknown) => key === 'maxFileSizeMB' ? 0 : fallback }),
     applyEdit: (edit: { text: string }) => activeHarness.applyEdit(edit.text),
@@ -45,18 +71,21 @@ const { CsvEditorProvider } = require('../CsvEditorProvider') as typeof import('
 const { JSDOM } = require('jsdom');
 
 export async function createEditorHarness(options: {
+  text?: string;
+  warningChoice?: string | Promise<string | undefined>;
   rows?: string[][];
   separator?: string;
   header?: boolean;
   remote?: boolean;
 } = {}) {
   const separator = options.separator ?? ',';
-  let text = Papa.unparse(options.rows ?? [['name', 'value'], ['one', 'two']], { delimiter: separator });
+  let text = options.text ?? Papa.unparse(options.rows ?? [['name', 'value'], ['one', 'two']], { delimiter: separator });
   let diskText = text;
   let messageListener: (message: any) => Promise<void>;
   const changeListeners: Array<(event: unknown) => void> = [];
   const viewListeners: Array<(event: unknown) => void> = [];
   const saveListeners: Array<(event: unknown) => void> = [];
+  const disposeListeners: Array<() => void> = [];
   const messages: any[] = [];
   const delayedEdits: Array<() => void> = [];
   let deferEdits = false;
@@ -76,6 +105,7 @@ export async function createEditorHarness(options: {
     }
   };
   const context = {
+    subscriptions: [] as unknown[],
     extensionPath: process.cwd(),
     workspaceState: {
       values: {} as Record<string, unknown>,
@@ -97,10 +127,22 @@ export async function createEditorHarness(options: {
       onDidReceiveMessage: (listener: typeof messageListener) => { messageListener = listener; }
     },
     onDidChangeViewState: (listener: (event: unknown) => void) => { viewListeners.push(listener); },
-    onDidDispose() {}
+    onDidDispose(listener: () => void) { disposeListeners.push(listener); }
   };
   const harness = {
     document, panel, messages, changeListeners, saveListeners,
+    warnings: [] as string[],
+    warningChoice: options.warningChoice,
+    commandError: undefined as Error | undefined,
+    commandWait: undefined as Promise<void> | undefined,
+    dispose() { disposeListeners.forEach(listener => listener()); },
+    closeResult: true,
+    commands: new Map<string, (...args: any[]) => unknown>(),
+    executedCommands: [] as Array<{ name: string; args: unknown[] }>,
+    registerCommands() {
+      const { registerCsvCommands } = require('../commands') as typeof import('../commands');
+      registerCsvCommands(context as any);
+    },
     get diskText() { return diskText; },
     get pendingEdits() { return delayedEdits.length; },
     deferEdits() { deferEdits = true; },

src/test/encoding-integrity.test.ts

@@ -0,0 +1,128 @@
+import assert from 'assert';
+import fs from 'fs';
+import path from 'path';
+import { describe, it } from 'node:test';
+import { createEditorHarness } from './edit-integrity-harness';
+import iconv from 'iconv-lite';
+
+describe('Explicit source decoding preserves original non-UTF8 characters', () => {
+  const original = fs.readFileSync(path.join(process.cwd(), 'src', 'test', 'fixtures', 'issue-76-products.csv'));
+  const decoded = iconv.decode(original, 'cp437');
+
+  it('keeps the reporter fixture byte-for-byte and establishes the supported decoding ambiguity', () => {
+    assert.strictEqual(original.length, 4052);
+    assert.ok(original.toString('utf8').includes('\uFFFD'));
+    assert.ok(!decoded.includes('\uFFFD'));
+    assert.strictEqual(iconv.decode(original, 'cp850'), decoded);
+    assert.deepStrictEqual(iconv.encode(decoded, 'cp437'), original);
+    for (const name of ['Knäckebröd', 'Thüringer', 'Guaraná', 'Nuß']) {
+      assert.ok(decoded.includes(name));
+    }
+  });
+
+  it('renders and edits the correctly decoded document without changing accented names', async () => {
+    const host = await createEditorHarness({ text: decoded });
+    assert.strictEqual(host.warnings.length, 0);
+    assert.ok(host.panel.webview.html.includes('Knäckebröd'));
+    const row = host.rows().findIndex(record => record[1]?.includes('Knäckebröd'));
+    await host.receive({ type: 'editCell', row, col: 5, value: '22.00' });
+    await host.receive({ type: 'save' });
+    const saved = iconv.decode(iconv.encode(host.diskText, 'cp437'), 'cp437');
+    for (const name of ['Knäckebröd', 'Thüringer', 'Guaraná', 'Nuß']) {
+      assert.ok(saved.includes(name));
+    }
+  });
+});
+
+describe('Possible decoding mismatches use an explicit encoding workflow', () => {
+  it('warns once without changing already decoded data when dismissed', async () => {
+    const host = await createEditorHarness({ rows: [['text'], ['replacement � may be intentional']] });
+    const originalText = host.document.getText();
+    assert.strictEqual(host.warnings.length, 1);
+    assert.match(host.warnings[0], /can indicate a decoding mismatch/);
+    await host.view(false, false);
+    await host.view(true, true);
+    assert.strictEqual(host.warnings.length, 1);
+    assert.strictEqual(host.executedCommands.length, 0);
+    assert.strictEqual(host.document.getText(), originalText);
+  });
+
+  it('binds the warning action to the originating document URI', async () => {
+    const host = await createEditorHarness({ rows: [['text'], ['�']], warningChoice: 'Change File Encoding' });
+    await new Promise(resolve => setImmediate(resolve));
+    assert.deepStrictEqual(host.executedCommands, [{ name: 'csv.changeEncoding', args: [host.document.uri] }]);
+  });
+
+  it('does not act on a warning from a disposed editor', async () => {
+    let choose!: (value: string) => void;
+    const choice = new Promise<string>(resolve => { choose = resolve; });
+    const host = await createEditorHarness({ rows: [['text'], ['�']], warningChoice: choice });
+    host.dispose();
+    choose('Change File Encoding');
+    await new Promise(resolve => setImmediate(resolve));
+    assert.strictEqual(host.executedCommands.length, 0);
+  });
+
+  it('handles a rejected warning action without an unhandled promise rejection', async () => {
+    let choose!: (value: string) => void;
+    const choice = new Promise<string>(resolve => { choose = resolve; });
+    const host = await createEditorHarness({ rows: [['text'], ['�']], warningChoice: choice });
+    host.commandError = new Error('test command rejection');
+    const warnings: unknown[][] = [];
+    const originalWarn = console.warn;
+    console.warn = (...args: unknown[]) => { warnings.push(args); };
+    try {
+      choose('Change File Encoding');
+      await new Promise(resolve => setImmediate(resolve));
+      assert.strictEqual(warnings[0]?.[0], 'CSV: Encoding warning action failed');
+    } finally {
+      console.warn = originalWarn;
+    }
+  });
+
+  it('stops the encoding command when the user cancels closing a dirty document', async () => {
+    const host = await createEditorHarness();
+    await host.receive({ type: 'editCell', row: 1, col: 0, value: 'unsaved' });
+    host.closeResult = false;
+    host.registerCommands();
+    await host.commands.get('csv.changeEncoding')?.();
+    assert.strictEqual(host.executedCommands.length, 0);
+    assert.strictEqual(host.rows()[1][0], 'unsaved');
+    assert.ok(!host.diskText.includes('unsaved'));
+  });
+
+  it('opens the native encoding picker and returns to the CSV view after a successful close', async () => {
+    const host = await createEditorHarness();
+    host.registerCommands();
+    await host.commands.get('csv.changeEncoding')?.(host.document.uri);
+    assert.deepStrictEqual(host.executedCommands.map(command => command.name), [
+      'vscode.openWith', 'workbench.action.editor.changeEncoding', 'vscode.openWith'
+    ]);
+    assert.strictEqual(host.executedCommands[0].args[0], host.document.uri);
+    assert.strictEqual(host.executedCommands[2].args[1], 'csv.editor');
+  });
+
+  it('waits for the native picker command to finish before reopening the CSV view', async () => {
+    const host = await createEditorHarness();
+    let complete!: () => void;
+    host.commandWait = new Promise<void>(resolve => { complete = resolve; });
+    host.registerCommands();
+    const changing = host.commands.get('csv.changeEncoding')?.();
+    await new Promise(resolve => setImmediate(resolve));
+    assert.deepStrictEqual(host.executedCommands.map(command => command.name), ['vscode.openWith', 'workbench.action.editor.changeEncoding']);
+    complete();
+    await changing;
+    assert.strictEqual(host.executedCommands[2].name, 'vscode.openWith');
+  });
+
+  it('round-trips UTF-8 accented CSV and TSV without triggering a warning', async () => {
+    for (const separator of [',', '\t']) {
+      const host = await createEditorHarness({ separator, rows: [['name', 'value'], ['éäöüßøå', 'before']] });
+      await host.receive({ type: 'editCell', row: 1, col: 1, value: 'after' });
+      await host.receive({ type: 'save' });
+      assert.strictEqual(host.warnings.length, 0);
+      assert.strictEqual(host.rows()[1][0], 'éäöüßøå');
+      assert.ok(Buffer.from(host.diskText, 'utf8').toString('utf8').includes('éäöüßøå'));
+    }
+  });
+});

src/test/fixtures/README.md

@@ -0,0 +1,7 @@
+`issue-76-products.csv` is the original 4,052-byte attachment from
+https://github.com/jonaraphael/csv/issues/76, downloaded without conversion from
+https://github.com/user-attachments/files/25353528/w3schools_dbo_Products.csv.
+
+It is not valid UTF-8. CP437 and CP850 produce identical Unicode text for all
+bytes present in this particular fixture; the fixture cannot distinguish which
+encoding created it. Tests decode explicitly and do not infer or repair bytes.