tui: images are drawn in the terminal, through tmux too

apps/cli/src/commands/tui.ts

@@ -5,7 +5,7 @@ import { failure, type Output } from '../output.js'
 import { type ParsedArgv } from '../parse-argv.js'
 import { resolveEndpoint, sshConfigHosts } from '../resolve.js'
 import { blankPane, buildEndpointChoices, defaultLocalPath, Tui } from '../tui/app.js'
-import { runInherited } from '../tui/launch.js'
+import { enableTmuxPassthrough, runInherited } from '../tui/launch.js'
 
 /**
  * `diskpush tui` — the two-pane browser, in a terminal.
@@ -42,6 +42,9 @@ export async function runTui(parsed: ParsedArgv, store: DiskPushStore, output: O
   const choices = buildEndpointChoices(await store.listConnections(), sshConfigHosts(), defaultLocalPath())
   const tui = new Tui(panes[0]!, panes[1]!, choices)
 
+  // Images ride on escape sequences tmux drops by default.
+  enableTmuxPassthrough()
+
   try {
     void tui.loadBoth()
     // The browser runs until it quits, or until it wants the terminal handed

apps/cli/src/tui/app.ts

@@ -11,7 +11,8 @@
  * screen can be rendered and asserted on in a test with no pty.
  */
 import { join, posix } from 'node:path'
-import type { App, Container, KeyEvent, MouseEvent, Theme } from '@profullstack/hqtui'
+import type { App, Container, KeyEvent, MouseEvent, Rect, Theme } from '@profullstack/hqtui'
+import { detectCapabilities } from '@profullstack/hqtui'
 import { knownHostsPath } from '@diskpush/database'
 import { SftpBrowser, SshSession } from '@diskpush/ssh-core'
 import { defaultRsyncOptions, type Change, type Connection } from '@diskpush/schemas'
@@ -28,11 +29,13 @@ import {
   SORT_KEYS,
   type Transfer,
   DOCUMENT_LIMIT,
+  IMAGE_LIMIT,
   blankDocument,
   blankPane,
   clampIndex,
   endpointString,
   fillDocument,
+  isImageName,
   listLocal,
   looksBinary,
   nothingToDo,
@@ -48,6 +51,7 @@ import {
 import { type Action, type Tone, type ViewState, draw, filterChoices } from './view.js'
 import { maxScroll, readsAsMarkdown } from './document.js'
 import { type Launch, type Launcher, type Target, editLaunch, openLaunch, systemLauncher } from './launch.js'
+import { type Protocol, chooseProtocol, deleteImage, encodeImage, placeAt, tmuxPassthrough } from './graphics.js'
 
 export {
   blankPane,
@@ -74,6 +78,12 @@ export class Tui {
   private documentLayout = { total: 0, rows: 1 }
   /** A program waiting for the terminal, once the app has given it up. See `start`. */
   private handoff: Launch | null = null
+  /** Where the frame left room for an image, if it did. Set by the view, read after the frame. */
+  private imageRect: Rect | null = null
+  /** The image on screen and the cells it was drawn into, so the next frame knows whether to draw again. */
+  private shown: { doc: Document; cells: string } | null = null
+  /** How this terminal is handed an image. Decided once: the terminal does not change. */
+  private readonly protocol: Protocol
   private filtering: Side | null = null
   private status: { text: string; tone: Tone } | null = null
   private busy = false
@@ -91,13 +101,21 @@ export class Tui {
     right: Pane,
     private readonly choices: readonly EndpointChoice[] = [],
     private readonly launcher: Launcher = systemLauncher(),
+    protocol?: Protocol,
   ) {
     this.panes = { left, right }
+    this.protocol = protocol ?? chooseProtocol(detectCapabilities({}, launcher.env).program)
   }
 
-  /** Binds the app so background work (a load, a transfer tick) can redraw. */
+  /**
+   * Binds the app so background work (a load, a transfer tick) can redraw,
+   * and so an image can be drawn once each frame is on the terminal.
+   */
   attach(app: App): void {
     this.app = app
+    this.shown = null
+    app.on('frame', () => this.afterFrame())
+    app.on('exit', () => this.clearImage())
   }
 
   private invalidate(): void {
@@ -122,7 +140,11 @@ export class Tui {
 
   /** The render callback handed to `app.render`. */
   view(ui: Container, theme: Theme, width: number, height: number): void {
+    this.imageRect = null
     draw(ui, theme, width, height, this.snapshot(), {
+      onImageRect: (rect) => {
+        this.imageRect = rect
+      },
       onPaneFocus: (side) => {
         this.active = side
         this.invalidate()
@@ -637,10 +659,12 @@ export class Tui {
     // A finished transfer's panel and the document want the same rows.
     if (this.transfer && !this.transfer.running) this.transfer = null
     this.invalidate()
+    // An image is handed to the terminal whole, so it is read whole.
+    const limit = isImageName(row.entry.name) ? IMAGE_LIMIT : DOCUMENT_LIMIT
     try {
       const head = pane.connection
-        ? await this.readRemoteHead(pane, posix.join(pane.path, row.rel))
-        : readLocalHead(join(pane.path, row.rel), DOCUMENT_LIMIT)
+        ? await this.readRemoteHead(pane, posix.join(pane.path, row.rel), limit)
+        : readLocalHead(join(pane.path, row.rel), limit)
       if (this.document !== doc) return
       fillDocument(doc, head, readsAsMarkdown)
     } catch (error) {
@@ -652,6 +676,48 @@ export class Tui {
     }
   }
 
+  // ------------------------------------------------------------------ images
+
+  private writeRaw(data: string): void {
+    if (data) this.app?.terminal.write(data)
+  }
+
+  /**
+   * After each frame: draw the image the frame left room for, or take down
+   * the one that is no longer wanted.
+   *
+   * The image is not part of the frame. hqtui diffs cells, and an image is
+   * not a cell, so it is handed to the terminal separately, once, into cells
+   * the frame left blank — and drawn again only when the file or the box
+   * changes, never on every mouse move. Taking it down is a full repaint:
+   * an iTerm2 image is erased by drawing over it, and hqtui's diff would
+   * otherwise leave the untouched cells exactly as they were, image and all.
+   */
+  private afterFrame(): void {
+    const doc = this.document
+    const rect = this.imageRect
+    const want =
+      doc?.kind === 'image' && doc.image && rect && rect.width > 0 && rect.height > 0
+        ? { doc, cells: `${rect.x},${rect.y},${rect.width},${rect.height}` }
+        : null
+    if (this.shown && (!want || want.doc !== this.shown.doc || want.cells !== this.shown.cells)) {
+      this.clearImage()
+      this.app?.redraw()
+      return
+    }
+    if (!want || this.shown || !rect || !doc?.image) return
+    const sequence = encodeImage(this.protocol, doc.bytes, doc.image, { cols: rect.width, rows: rect.height })
+    if (sequence) this.writeRaw(placeAt(sequence, rect.x, rect.y, this.launcher.inTmux))
+    this.shown = want
+  }
+
+  private clearImage(): void {
+    if (!this.shown) return
+    const sequence = deleteImage(this.protocol)
+    if (sequence) this.writeRaw(this.launcher.inTmux ? tmuxPassthrough(sequence) : sequence)
+    this.shown = null
+  }
+
   // --------------------------------------------------------------- launching
 
   /** The row under the cursor, as something to hand to a program. */
@@ -752,10 +818,10 @@ export class Tui {
     this.invalidate()
   }
 
-  private async readRemoteHead(pane: Pane, path: string): Promise<{ bytes: Uint8Array; size: number }> {
+  private async readRemoteHead(pane: Pane, path: string, limit: number): Promise<{ bytes: Uint8Array; size: number }> {
     const browser = await SftpBrowser.open(await this.session(pane.connection!))
     try {
-      return await browser.readHead(path, DOCUMENT_LIMIT)
+      return await browser.readHead(path, limit)
     } finally {
       browser.close()
     }

apps/cli/src/tui/document.ts

@@ -41,6 +41,8 @@ function fromMarkdown(span: MarkdownSpan, theme: Theme): SpanLine[number] {
 }
 
 function render(doc: Document, width: number, theme: Theme): SpanLine[] {
+  // An image is drawn by the terminal, over cells this leaves blank.
+  if (doc.kind === 'image') return []
   if (doc.kind === 'markdown') {
     return renderMarkdown(doc.text, Math.max(20, width)).map((line) => line.spans.map((span) => fromMarkdown(span, theme)))
   }

apps/cli/src/tui/graphics.test.ts

@@ -0,0 +1,121 @@
+import { describe, expect, it } from 'vitest'
+import { KITTY_IMAGE_ID, chooseProtocol, deleteImage, encodeImage, fitCells, fitsNaturally, placeAt, tmuxPassthrough } from './graphics.js'
+import { blankDocument, fillDocument, imageInfo, isImageName } from './model.js'
+
+/** A 1×1 transparent PNG. */
+export const PNG_1x1 = Buffer.from(
+  'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
+  'base64',
+)
+
+describe('choosing a protocol', () => {
+  it('speaks kitty to kitty and ghostty, and iTerm2 to everyone else, tmux included', () => {
+    expect(chooseProtocol('kitty')).toBe('kitty')
+    expect(chooseProtocol('ghostty')).toBe('kitty')
+    expect(chooseProtocol('wezterm')).toBe('iterm2')
+    expect(chooseProtocol('iterm')).toBe('iterm2')
+    expect(chooseProtocol('tmux')).toBe('iterm2')
+    expect(chooseProtocol('unknown')).toBe('iterm2')
+  })
+})
+
+describe('fitting an image to a box', () => {
+  it('keeps the shape, counting a cell as twice as tall as wide', () => {
+    // A 200×100 image is 200 columns by 50 rows in cell units: twice the box's width matters.
+    expect(fitCells({ width: 200, height: 100 }, { cols: 100, rows: 50 })).toEqual({ cols: 100, rows: 25 })
+    expect(fitCells({ width: 100, height: 400 }, { cols: 100, rows: 50 })).toEqual({ cols: 25, rows: 50 })
+    expect(fitCells({ width: 0, height: 0 }, { cols: 100, rows: 50 })).toEqual({ cols: 0, rows: 0 })
+  })
+
+  it('draws an icon at its own size and shrinks a photo', () => {
+    expect(fitsNaturally({ width: 64, height: 64 }, { cols: 80, rows: 20 })).toBe(true)
+    expect(fitsNaturally({ width: 4000, height: 3000 }, { cols: 80, rows: 20 })).toBe(false)
+  })
+})
+
+describe('encoding', () => {
+  const info = imageInfo(PNG_1x1)!
+
+  const photo = { ...info, width: 4000, height: 3000 }
+
+  it('iTerm2: one OSC 1337 carrying the file, sized to the box, aspect kept', () => {
+    const seq = encodeImage('iterm2', PNG_1x1, photo, { cols: 80, rows: 20 })!
+    expect(seq.startsWith(`\x1b]1337;File=inline=1;size=${PNG_1x1.length};width=80;height=20;preserveAspectRatio=1:`)).toBe(true)
+    expect(encodeImage('iterm2', PNG_1x1, info, { cols: 80, rows: 20 })).toContain(';width=auto;height=auto;')
+    expect(seq.endsWith('\x07')).toBe(true)
+    const payload = seq.slice(seq.indexOf(':') + 1, -1)
+    expect(Buffer.from(payload, 'base64').equals(PNG_1x1)).toBe(true)
+  })
+
+  it('kitty: chunked APC with the id, the fitted cell box, and PNG only', () => {
+    const seq = encodeImage('kitty', PNG_1x1, photo, { cols: 80, rows: 20 })!
+    expect(seq.startsWith(`\x1b_Ga=T,f=100,i=${KITTY_IMAGE_ID},c=53,r=20,q=2,m=0;`)).toBe(true)
+    expect(encodeImage('kitty', PNG_1x1, info, { cols: 80, rows: 20 })).toContain(`i=${KITTY_IMAGE_ID},q=2,m=0;`)
+    expect(seq.endsWith('\x1b\\')).toBe(true)
+    expect(encodeImage('kitty', PNG_1x1, { ...info, format: 'jpeg' }, { cols: 80, rows: 20 })).toBeNull()
+    // A big payload is cut into 4096-character chunks, m=1 on all but the last.
+    const big = Buffer.concat([PNG_1x1, Buffer.alloc(9000)])
+    const chunked = encodeImage('kitty', big, photo, { cols: 80, rows: 20 })!
+    const parts = chunked.split('\x1b\\').filter(Boolean)
+    expect(parts.length).toBe(3)
+    expect(parts[0]).toContain(',m=1;')
+    expect(parts[1]!.startsWith('\x1b_Gm=1;')).toBe(true)
+    expect(parts[2]!.startsWith('\x1b_Gm=0;')).toBe(true)
+    expect(Buffer.from(parts.map((part) => part.slice(part.indexOf(';') + 1)).join(''), 'base64').equals(big)).toBe(true)
+  })
+
+  it('deletes by id under kitty, and has nothing to delete under iTerm2', () => {
+    expect(deleteImage('kitty')).toBe(`\x1b_Ga=d,d=I,i=${KITTY_IMAGE_ID},q=2\x1b\\`)
+    expect(deleteImage('iterm2')).toBe('')
+  })
+})
+
+describe('getting it through tmux and onto a cell', () => {
+  it('wraps in a DCS envelope with every ESC doubled', () => {
+    expect(tmuxPassthrough('\x1b]1337;x\x07')).toBe('\x1bPtmux;\x1b\x1b]1337;x\x07\x1b\\')
+  })
+
+  it('moves to the cell, draws, and puts the cursor back, wrapping only the image', () => {
+    expect(placeAt('IMG', 4, 9, false)).toBe('\x1b7\x1b[10;5HIMG\x1b8')
+    expect(placeAt('\x1bX', 0, 0, true)).toBe('\x1b7\x1b[1;1H\x1bPtmux;\x1b\x1bX\x1b\\\x1b8')
+  })
+})
+
+describe('reading image headers', () => {
+  it('knows PNG, GIF, BMP, WebP and JPEG by their headers, and nothing else', () => {
+    expect(imageInfo(PNG_1x1)).toEqual({ format: 'png', width: 1, height: 1 })
+    expect(imageInfo(Buffer.from('GIF89a\x40\x01\xf0\x00', 'latin1'))).toEqual({ format: 'gif', width: 320, height: 240 })
+    const bmp = Buffer.alloc(26)
+    bmp.write('BM', 0, 'latin1')
+    bmp.writeInt32LE(640, 18)
+    bmp.writeInt32LE(-480, 22)
+    expect(imageInfo(bmp)).toEqual({ format: 'bmp', width: 640, height: 480 })
+    const webp = Buffer.alloc(30)
+    webp.write('RIFF', 0, 'latin1')
+    webp.write('WEBP', 8, 'latin1')
+    webp.write('VP8X', 12, 'latin1')
+    webp.writeUIntLE(1023, 24, 3)
+    webp.writeUIntLE(767, 27, 3)
+    expect(imageInfo(webp)).toEqual({ format: 'webp', width: 1024, height: 768 })
+    // SOI, an APP0 segment to step over, then SOF0 with height 600 and width 800.
+    const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x04, 0x00, 0x00, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x02, 0x58, 0x03, 0x20, 0x03])
+    expect(imageInfo(jpeg)).toEqual({ format: 'jpeg', width: 800, height: 600 })
+    expect(imageInfo(Buffer.from('not an image at all, really'))).toBeNull()
+    expect(imageInfo(new Uint8Array())).toBeNull()
+  })
+})
+
+describe('an image as a document', () => {
+  it('is an image when read whole, and a hex dump when the read was cut short', () => {
+    const whole = blankDocument('left', 'dot.png', '/x/dot.png')
+    fillDocument(whole, { bytes: PNG_1x1, size: PNG_1x1.length })
+    expect(whole.kind).toBe('image')
+    expect(whole.image).toEqual({ format: 'png', width: 1, height: 1 })
+    const cut = blankDocument('left', 'huge.png', '/x/huge.png')
+    fillDocument(cut, { bytes: PNG_1x1, size: 50_000_000 })
+    expect(cut.kind).toBe('binary')
+    expect(cut.image).toBeNull()
+    expect(isImageName('a.JPG')).toBe(true)
+    expect(isImageName('a.svg')).toBe(false)
+  })
+})