Plugin architecture, with MediaAnalyzer as the first plugin

README.md

@@ -101,6 +101,10 @@ diskpush fleet upgrade --on tag:production --sudo
 
 # Or run anything, anywhere
 diskpush fleet run "systemctl reload nginx" --on 'web-*' --sudo
+
+# Describe a folder of photos, and sort them into folders by what they show
+diskpush mediaanalyzer login
+diskpush mediaanalyzer analyze ~/Pictures/2024 --sort
 ```
 
 ## Safety
@@ -120,8 +124,12 @@ to make that flag hard to trigger by accident.
   through verbatim.
 - **Host keys are verified.** A changed host key blocks the connection. There
   is no global setting to turn that off.
-- **No credentials on disk.** The local database holds no passwords or
-  passphrases.
+- **No SSH credentials on disk.** The local database holds no passwords or
+  passphrases. A plugin's sign-in (such as MediaAnalyzer's token) is kept in
+  it, which is why DiskPush keeps that file owner-only (`0600`).
+- **Plugins stay out of the renderer.** Plugin code runs in the CLI or the
+  desktop's main process, and the renderer can only name a plugin action and
+  entries inside a local directory. See [docs/plugins.md](docs/plugins.md).
 
 ## Documentation
 
@@ -136,6 +144,7 @@ to make that flag hard to trigger by accident.
 | [docs/file-browser.md](docs/file-browser.md) | Why browsing is SFTP and transfers are rsync |
 | [docs/profiles.md](docs/profiles.md) | Saved, repeatable directory pairs |
 | [docs/fleet.md](docs/fleet.md) | Running one command, or an upgrade, across many servers |
+| [docs/plugins.md](docs/plugins.md) | Plugins, MediaAnalyzer, writing one, and the security model |
 | [docs/security.md](docs/security.md) | Threat model and the decisions that follow from it |
 | [docs/architecture.md](docs/architecture.md) | Packages, processes and boundaries |
 | [docs/troubleshooting.md](docs/troubleshooting.md) | What the errors mean |
@@ -152,6 +161,8 @@ packages/rsync-core the transfer engine, with no Electron in it
 packages/ssh-core   SSH sessions, SFTP browsing, host keys, preflight
 packages/fleet-core one command across many servers, with no Electron in it
 packages/database   the local store shared by desktop and CLI
+packages/plugin-api the plugin contract, registry and external loader
+packages/plugin-mediaanalyzer  the built-in MediaAnalyzer plugin
 ```
 
 `rsync-core` deliberately has no dependency on Electron or on the CLI, so the

apps/cli/package.json

@@ -18,6 +18,8 @@
   "dependencies": {
     "@diskpush/database": "workspace:*",
     "@diskpush/fleet-core": "workspace:*",
+    "@diskpush/plugin-api": "workspace:*",
+    "@diskpush/plugin-mediaanalyzer": "workspace:*",
     "@diskpush/rsync-core": "workspace:*",
     "@diskpush/schemas": "workspace:*",
     "@diskpush/ssh-core": "workspace:*",

apps/cli/src/bin.ts

@@ -14,11 +14,51 @@ import { runTransfer, TRANSFER_ALIASES } from './commands/transfer.js'
 import { EXIT } from './exit-codes.js'
 import { HELP, VERSION } from './help.js'
 import { Output } from './output.js'
-import { ArgvError, hasFlag, looksLikeEndpoint, parseArgv } from './parse-argv.js'
+import { ArgvError, hasFlag, isKnownCommand, looksLikeEndpoint, parseArgv } from './parse-argv.js'
 import { autoUpdate, reexec } from './self-update.js'
+import { findPluginCall, runPluginCommand, runPlugins } from './commands/plugins.js'
+import { pluginHelp } from './help.js'
+import { loadPlugins } from './plugins.js'
+import { PluginError } from '@diskpush/plugin-api'
 import { existsSync } from 'node:fs'
 
+/** A first word that is neither a command nor a path might be a plugin: `diskpush mediaanalyzer login`. */
+function mightBePlugin(argv: readonly string[]): boolean {
+  const head = argv.find((token) => !token.startsWith('-'))
+  return head !== undefined && /^[a-z][a-z0-9-]*$/.test(head) && !isKnownCommand(head) && !looksLikeEndpoint(head, existsSync)
+}
+
+async function runPlugin(argv: readonly string[]): Promise<number | null> {
+  const store = await DiskPushStore.open()
+  try {
+    const { registry, failures } = await loadPlugins(store)
+    const call = findPluginCall(argv, registry)
+    if (!call) return null
+    const output = new Output({
+      json: argv.includes('--json'),
+      quiet: argv.includes('--quiet') || argv.includes('-q'),
+      progress: !argv.includes('--no-progress'),
+    })
+    for (const { name, error } of failures) output.warn(`plugin ${name} did not load: ${error}`)
+    if (await autoUpdate(call.pluginId, output) === 'updated') reexec()
+    return await runPluginCommand(registry, call.pluginId, call.args, output)
+  } catch (error) {
+    if (error instanceof PluginError) {
+      process.stderr.write(`${error.message}\n`)
+      return EXIT.configuration
+    }
+    throw error
+  } finally {
+    await store.close()
+  }
+}
+
 async function main(argv: readonly string[]): Promise<number> {
+  if (mightBePlugin(argv)) {
+    const code = await runPlugin(argv)
+    if (code !== null) return code
+  }
+
   let parsed
   try {
     parsed = parseArgv(argv)
@@ -34,7 +74,7 @@ async function main(argv: readonly string[]): Promise<number> {
   })
 
   if (hasFlag(parsed, '--help') || parsed.command === 'help') {
-    process.stdout.write(HELP)
+    process.stdout.write(HELP + pluginHelp((await loadPlugins(null)).registry.all()))
     return EXIT.ok
   }
   if (hasFlag(parsed, '--version') || parsed.command === 'version') {
@@ -99,6 +139,9 @@ async function main(argv: readonly string[]): Promise<number> {
         return await runFleetCommand(parsed, store, output)
       case 'ls':
         return await runLs(parsed, store, output)
+      case 'plugins':
+      case 'plugin':
+        return await runPlugins(parsed, store, output)
       default:
         output.error(`Unknown command ${JSON.stringify(command)}. Run \`diskpush --help\`.`)
         return EXIT.usage
@@ -111,6 +154,7 @@ async function main(argv: readonly string[]): Promise<number> {
 function describeError(error: unknown): { message: string; code: number } {
   if (error instanceof ArgvError) return { message: error.message, code: EXIT.usage }
   if (error instanceof EndpointParseError) return { message: error.message, code: EXIT.usage }
+  if (error instanceof PluginError) return { message: error.message, code: EXIT.configuration }
   if (error instanceof ZodError) {
     const first = error.issues[0]
     return {

apps/cli/src/commands/plugins.test.ts

@@ -0,0 +1,147 @@
+import { mkdtempSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { DiskPushStore } from '@diskpush/database'
+import { PluginRegistry, definePlugin, memoryBackend, type CommandContext } from '@diskpush/plugin-api'
+import { Output } from '../output.js'
+import { parseArgv } from '../parse-argv.js'
+import { pluginHelp } from '../help.js'
+import { assignKeys, loadPlugins } from '../plugins.js'
+import { findPluginCall, runPluginCommand, runPlugins, stripGlobalFlags } from './plugins.js'
+
+const seen: { args: string[]; ctx: CommandContext }[] = []
+
+const echo = definePlugin({
+  id: 'echo',
+  name: 'Echo',
+  version: '1.0.0',
+  description: 'repeats itself',
+  commands: [
+    {
+      name: 'say',
+      summary: 'print the arguments',
+      usage: 'say WORDS... [--loud]',
+      async run(args, ctx) {
+        seen.push({ args, ctx })
+        await ctx.settings.set('last', args)
+        if (ctx.json) ctx.printJson({ said: args })
+        else ctx.print(args.join(' '))
+        return args.includes('--fail') ? 3 : 0
+      },
+    },
+  ],
+})
+
+let stdout: string[]
+let stderr: string[]
+
+beforeEach(() => {
+  seen.length = 0
+  stdout = []
+  stderr = []
+  vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => {
+    stdout.push(String(chunk))
+    return true
+  })
+  vi.spyOn(process.stderr, 'write').mockImplementation((chunk) => {
+    stderr.push(String(chunk))
+    return true
+  })
+})
+
+afterEach(() => {
+  vi.restoreAllMocks()
+  vi.unstubAllEnvs()
+})
+
+const output = (json = false) => new Output({ json, quiet: false, progress: false })
+
+function registry() {
+  const r = new PluginRegistry(memoryBackend())
+  r.register(echo)
+  return r
+}
+
+describe('dispatch to a plugin', () => {
+  it('finds `diskpush <plugin-id> ...` on raw argv, flags before the id included', () => {
+    const r = registry()
+    expect(findPluginCall(['echo', 'say', 'hi'], r)).toEqual({ pluginId: 'echo', args: ['say', 'hi'] })
+    expect(findPluginCall(['--json', 'echo', 'say', '--timeout'], r)).toEqual({
+      pluginId: 'echo',
+      args: ['--json', 'say', '--timeout'],
+    })
+    expect(findPluginCall(['sync', './a', './b'], r)).toBeNull()
+    expect(findPluginCall(['--help'], r)).toBeNull()
+    expect(stripGlobalFlags(['--json', 'say', '-q', 'x'])).toEqual(['say', 'x'])
+  })
+
+  it('runs the command with its own arguments and a context bound to the plugin', async () => {
+    const r = registry()
+    const code = await runPluginCommand(r, 'echo', ['say', 'hello', 'world', '--loud'], output())
+    expect(code).toBe(0)
+    expect(seen[0]!.args).toEqual(['hello', 'world', '--loud'])
+    expect(seen[0]!.ctx.surface).toBe('cli')
+    expect(stdout.join('')).toBe('hello world --loud\n')
+    expect(await r.backend.getSetting('plugin:echo:last', null)).toEqual(['hello', 'world', '--loud'])
+  })
+
+  it('passes the exit code through, and --json reaches the plugin as ctx.json', async () => {
+    const r = registry()
+    expect(await runPluginCommand(r, 'echo', ['--json', 'say', 'x', '--fail'], output(true))).toBe(3)
+    expect(seen[0]!.ctx.json).toBe(true)
+    expect(JSON.parse(stdout.join(''))).toEqual({ said: ['x', '--fail'] })
+  })
+
+  it('prints the plugin help for no command, and refuses an unknown one', async () => {
+    const r = registry()
+    expect(await runPluginCommand(r, 'echo', [], output())).toBe(64)
+    expect(stdout.join('')).toContain('diskpush echo say WORDS... [--loud]')
+    expect(await runPluginCommand(r, 'echo', ['shout'], output())).toBe(64)
+    expect(stderr.join('')).toContain('Echo has no command "shout"')
+  })
+
+  it('will not run a disabled plugin', async () => {
+    const r = registry()
+    await r.disable('echo')
+    await expect(runPluginCommand(r, 'echo', ['say'], output())).rejects.toThrow(/disabled.*diskpush plugins enable echo/)
+  })
+
+  it('lists plugin commands in the help text', () => {
+    expect(pluginHelp([echo])).toContain('PLUGIN COMMANDS\n  echo say WORDS... [--loud]  print the arguments')
+    expect(pluginHelp([])).toBe('')
+  })
+})
+
+describe('diskpush plugins', () => {
+  it('lists the built-in plugins, and enable/disable persist in the store', async () => {
+    vi.stubEnv('DISKPUSH_HOME', mkdtempSync(join(tmpdir(), 'dp-home-')))
+    const store = await DiskPushStore.open({ path: ':memory:' })
+    try {
+      expect(await runPlugins(parseArgv(['plugins']), store, output())).toBe(0)
+      expect(stdout.join('')).toMatch(/mediaanalyzer\s+enabled\s+builtin/)
+      expect(stdout.join('')).toContain('diskpush mediaanalyzer analyze DIR [--sort]')
+
+      expect(await runPlugins(parseArgv(['plugins', 'disable', 'mediaanalyzer']), store, output())).toBe(0)
+      expect(await store.getSetting('plugins.disabled', [])).toEqual(['mediaanalyzer'])
+      const { registry: again } = await loadPlugins(store)
+      expect(await again.isEnabled('mediaanalyzer')).toBe(false)
+
+      expect(await runPlugins(parseArgv(['plugins', 'enable', 'mediaanalyzer']), store, output())).toBe(0)
+      expect(await store.getSetting('plugins.disabled', [])).toEqual([])
+
+      expect(await runPlugins(parseArgv(['plugins', 'enable', 'nope']), store, output())).toBe(65)
+      expect(await runPlugins(parseArgv(['plugins', 'remove', 'mediaanalyzer']), store, output())).toBe(66)
+    } finally {
+      await store.close()
+    }
+  })
+})
+
+describe('action keys', () => {
+  it('keeps each hinted letter unless the menu or an earlier action has it', () => {
+    const choice = (label: string) => ({ pluginId: 'p', pluginName: 'P', actionId: label, label, description: '' })
+    const keys = assignKeys([choice('a'), choice('b'), choice('c'), choice('d')], ['m', 'm', 'j', undefined]).map((c) => c.key)
+    expect(keys).toEqual(['m', null, null, null])
+  })
+})