From 4acb3e574abef41eca3d6cd25443f1c1a6b33bf0 Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:08:49 -0400 Subject: [PATCH 1/3] plugins: library sources build a drive from what a plugin reads, copying only audio files in the folders the user grants A plugin may now contribute library sources ([[contributes.sources]], feature `sources`): `library/read {source, options, roots, cursor?}` answers pages of tracks (an absolute audio path, tags, key, BPM, cues, a beat grid, a waveform, My Tags) and playlists, a big page in a blob. It needs the new `fs:read` capability, and is given only the folders the user grants for that build. The host trusts none of the plugin's paths. A track is copied only when its audio file, links resolved, is a regular file players play inside a granted folder and not on the drive written to; the host reads its format and length itself, hashes it before copying and checks the copy against that hash. What isn't is left out with a warning, and so is a playlist entry for it, a cue in a taken slot, a grid that isn't one. Cues, beat grids and waveforms go into new analysis files made with the anlz writers and read back before they're kept. collection_export takes a source-neutral library (`Collected`) that rekordbox's collection and plugins both fill, so a source builds a drive through the same library_write and convert::build_databases path, with the drive remembering which source it was built from. `collection export DRIVE --from PLUGIN:SOURCE --grant FOLDER [--option K=V]`, the player's "build from ..." form and the sidecar's `plugins.source.plan` / `.apply` run it. The test plugin's sources read the files in their folders. --- apps/locallibrary/CHANGELOG.md | 8 + apps/locallibrary/docs/commands.md | 1 + apps/locallibrary/docs/player.md | 5 + apps/locallibrary/docs/plugins.md | 23 +- apps/locallibrary/docs/rpc/locallibrary.d.ts | 60 +- apps/locallibrary/docs/rpc/schema.json | 166 +++- apps/locallibrary/docs/sidecar.md | 9 + apps/locallibrary/src/collection_cmd.rs | 88 +- apps/locallibrary/src/export_cmd.rs | 2 +- .../src/plugin_cmd/contributed.rs | 10 +- .../src/plugin_cmd/contributed/tests.rs | 5 +- apps/locallibrary/src/plugin_cmd/listing.rs | 4 + apps/locallibrary/src/plugins.rs | 7 +- apps/locallibrary/src/rpc.rs | 6 +- apps/locallibrary/src/rpc/methods/plugins.rs | 2 + .../src/rpc/methods/plugins/sources.rs | 23 + apps/locallibrary/src/rpc/tests.rs | 18 +- apps/locallibrary/src/tui/app.rs | 1 + apps/locallibrary/src/tui/drives.rs | 1 + apps/locallibrary/src/tui/edits.rs | 10 + apps/locallibrary/src/tui/form.rs | 8 +- apps/locallibrary/src/tui/keymap.rs | 2 + apps/locallibrary/src/tui/plugins.rs | 14 +- apps/locallibrary/src/tui/plugins/sources.rs | 84 ++ apps/locallibrary/src/tui/plugins/tests.rs | 1 + .../src/tui/plugins/tests/sources.rs | 193 ++++ apps/locallibrary/src/tui/ui.rs | 1 + apps/locallibrary/tests/it/plugins.rs | 1 + apps/locallibrary/tests/it/plugins/props.rs | 19 +- apps/locallibrary/tests/it/plugins/sources.rs | 203 +++++ crates/locallibrary/src/analysis_edit.rs | 33 +- crates/locallibrary/src/api/collection.rs | 35 +- .../locallibrary/src/bin/test-plugin/main.rs | 3 +- .../src/bin/test-plugin/plugin.rs | 68 +- crates/locallibrary/src/collection_export.rs | 325 ++++++- crates/locallibrary/src/library_write.rs | 10 +- crates/locallibrary/src/plugin.rs | 8 +- crates/locallibrary/src/plugin/sources.rs | 418 +++++++++ crates/locallibrary/tests/it/plugins.rs | 1 + .../locallibrary/tests/it/plugins/sources.rs | 827 ++++++++++++++++++ crates/loclib-plugin/Cargo.toml | 2 +- crates/loclib-plugin/README.md | 25 +- crates/loclib-plugin/src/features.rs | 1 + crates/loclib-plugin/src/hooks.rs | 1 + crates/loclib-plugin/src/hooks/analysis.rs | 3 +- crates/loclib-plugin/src/hooks/sources.rs | 179 ++++ crates/loclib-plugin/src/lib.rs | 1 + crates/loclib-plugin/src/manifest.rs | 9 +- crates/loclib-plugin/src/manifest/tests.rs | 56 +- crates/loclib-plugin/src/protocol.rs | 1 + crates/loclib-plugin/src/sdk/tests.rs | 36 +- 51 files changed, 2887 insertions(+), 130 deletions(-) create mode 100644 apps/locallibrary/src/rpc/methods/plugins/sources.rs create mode 100644 apps/locallibrary/src/tui/plugins/sources.rs create mode 100644 apps/locallibrary/src/tui/plugins/tests/sources.rs create mode 100644 apps/locallibrary/tests/it/plugins/sources.rs create mode 100644 crates/locallibrary/src/plugin/sources.rs create mode 100644 crates/locallibrary/tests/it/plugins/sources.rs create mode 100644 crates/loclib-plugin/src/hooks/sources.rs diff --git a/apps/locallibrary/CHANGELOG.md b/apps/locallibrary/CHANGELOG.md index 462f9eb5..51b3976f 100644 --- a/apps/locallibrary/CHANGELOG.md +++ b/apps/locallibrary/CHANGELOG.md @@ -269,6 +269,14 @@ Windows builds are experimental and untested on hardware (see Platforms in the R player where the drive has none, and written with `analyze --apply grid,phrases,waveform` (the sidecar's `plugins.apply_analysis.plan/apply`), a grid setting the track's BPM; operations plan the same edit with `analysis:plan`. Checked against rekordbox's files, not yet on players. +- **Drives built from a plugin's library source.** `collection export DRIVE --from PLUGIN:SOURCE + --grant FOLDER` builds a drive from a library a plugin reads (feature `sources`, capability + `fs:read`), as rekordbox's collection is written: the plugin is given only the folders granted, + and only audio files players play in them (links resolved) are copied, each read and hashed by + locallibrary itself and checked as it's copied; the rest is left out with a warning. Cues, beat + grids and waveforms a track brings go into analysis files made for it, read back first. The + player has "build from …" in the palette and a drive's menu, the sidecar + `plugins.source.plan/apply`. Not yet tried on players. - **The library is the unit.** Each drive holding a library is a clone of it, and its name doesn't matter much: a library's name works where a drive does (but in commands about one particular drive) and means its lead clone, the newest of its drives plugged in (the primary when they're diff --git a/apps/locallibrary/docs/commands.md b/apps/locallibrary/docs/commands.md index df6fe5b1..a48b6b3f 100644 --- a/apps/locallibrary/docs/commands.md +++ b/apps/locallibrary/docs/commands.md @@ -135,6 +135,7 @@ its drives. | `locallibrary diff --summary` | Which drive is newer, the evidence, and the differences (in both databases) | | `locallibrary collection [info\|tracks\|playlists\|diff ]` | rekordbox's own collection on this computer (`master.db`), read only, and what a drive lacks against it, see [rekordbox's collection](other-software.md#rekordboxs-collection) | | `locallibrary collection export [--playlist P]...` | Write the collection to a drive or folder as rekordbox's Export does, Cloud Library Sync tracks included, see [Exporting it without rekordbox](other-software.md#exporting-it-without-rekordbox) | +| `locallibrary collection export --from PLUGIN:SOURCE --grant FOLDER...` | Build it from a plugin's library source instead, copying only audio files in the folders granted, see [Library sources](plugins.md#library-sources) | | `locallibrary export [PLAYLIST] -o FILE` | A playlist, folder, history list or the whole library as M3U8, CSV or text | | `locallibrary export [PLAYLIST] --format traktor\|serato\|engine -o DIR [--option KEY=VALUE]... [--dry-run]` | The library for other DJ software (the interop plugin's), written on this computer, see [Other DJ software](other-software.md#other-dj-software) | | `locallibrary export [PLAYLIST] --format FORMAT -o PATH [--option KEY=VALUE]... [--dry-run] [--replace]` | A format a plugin adds; overwriting a file is asked about first unless `--replace`, see [Plugins](plugins.md) | diff --git a/apps/locallibrary/docs/player.md b/apps/locallibrary/docs/player.md index c566dbd5..02682590 100644 --- a/apps/locallibrary/docs/player.md +++ b/apps/locallibrary/docs/player.md @@ -55,6 +55,11 @@ listening to your library the way the players see it: alone is chosen for you, and for each other track you pick one from a list (or none); then the changes are confirmed like any edit. "provider settings…" in the palette sets how high (as `plugin providers --auto-pick` does). +- **Library sources**: "build from …" in the palette (or a drive's `e` menu) builds the drive + plugged in from a plugin's library source: a form asks which folders it may read files in + (`~/Music` to begin with, several split by commas), whether to replace a library the drive + holds, and for the source's options; the plan says what it read and left out, and is confirmed + like any change. - **Plugins' settings**: "Cover art plugin settings…" in the palette opens a plugin's settings in the same form, as they're set now; `enter` saves what changed, and a field left empty goes back to its default. A secret is never filled in: typed, it's kept in the system's keychain; left diff --git a/apps/locallibrary/docs/plugins.md b/apps/locallibrary/docs/plugins.md index b976263c..d82d846b 100644 --- a/apps/locallibrary/docs/plugins.md +++ b/apps/locallibrary/docs/plugins.md @@ -27,8 +27,8 @@ are: `locallibrary plugin` lists them, what each may do (its capabilities, in plain words) and what it adds. `locallibrary plugin info ID` has one in full: its commands, how to run them and which of -the player's menus offer them, its analyzers' fields, its export formats, its providers and its -settings. +the player's menus offer them, its analyzers' fields, its export formats, its providers, its +library sources and its settings. A plugin whose id, commands or export formats clash with locallibrary's, or with another plugin's, is left out: every command warns which and why, and the other plugins work. Two plugins clashing @@ -97,6 +97,21 @@ format can't hold), and overwriting a file is asked about first unless `--replac they're in the palette and a drive's menu: an export shows what it would write and asks before writing it. +## Library sources + +A library source is a library a plugin reads from somewhere on this computer (another DJ +program's collection, a folder of tagged files), which locallibrary builds a drive from as it +writes rekordbox's collection: `locallibrary collection export DRIVE --from PLUGIN:SOURCE --grant +FOLDER` (`--grant` again for more folders, the source's options as `--option KEY=VALUE`, +`--dry-run` to see the plan). The plugin is given only the folders you grant it, for that build, +and locallibrary copies only audio files players play that are in them (their links resolved), +not on the drive itself: it reads each file's length and format itself, hashes it before copying +it, and checks the copy. What it leaves out (a file elsewhere, one that isn't audio, a cue in a +slot taken) is listed as a warning. Cues, beat grids and waveforms a track comes with go into +analysis files made for it, each read back first. Building again copies only what changed; a +library on the drive that wasn't built from that source is only replaced with `--replace`. In the +player, "build from …" in the palette and a drive's menu asks for the folders and options first. + ## Providers A provider finds cover art or details (tags) for tracks, online usually: `locallibrary art fetch @@ -174,7 +189,9 @@ plugin's log and of the errors it's told (what a server answers is passed on as Images and other large answers come back as files in a folder locallibrary makes for each request (`plugins/ID/scratch/`), which it reads itself, never past the folder (nor a link, or a file with another name) or more than 512 MiB of them, and removes; `store gc` removes those a locallibrary -that crashed left. +that crashed left. A library source names audio files by path, and locallibrary copies a file +only when it's in a folder you granted for that build, its links resolved (see +[Library sources](#library-sources)). A plugin runs as you and isn't sandboxed, so this is what locallibrary does, not a wall your system keeps: a plugin could still read any file you can. diff --git a/apps/locallibrary/docs/rpc/locallibrary.d.ts b/apps/locallibrary/docs/rpc/locallibrary.d.ts index 17805dd5..48f0183b 100644 --- a/apps/locallibrary/docs/rpc/locallibrary.d.ts +++ b/apps/locallibrary/docs/rpc/locallibrary.d.ts @@ -2445,6 +2445,36 @@ export type ChooseRole = { analyzer?: string | null; }; +/** Building a drive from a plugin's library source. */ +export type SourceExport = { + plugin: string; + /** One of its library sources, by name. */ + source: string; + /** As the source's `options` schema says. */ + options?: unknown; + /** + * The folders the plugin may read files in, absolute: granted for this read alone, and only + * audio files in them are copied. + */ + grants: string[]; + /** The drive or folder to write to. */ + drive: string; + /** + * Playlists or folders (paths) to export, with the tracks they use. Empty: every track and + * playlist, or the selection the target was last exported with. + */ + playlists?: string[]; + /** Export every track and playlist, whatever the target was exported with last. */ + all?: boolean; + /** + * Replace a library the target holds that wasn't exported from the collection, or the + * files on a target with no library. + */ + replace?: boolean; + /** Write to an archive or retired drive (see [`crate::fleet::check_target`]). */ + overwrite_archive?: boolean; +}; + /** What `initialize` answers. */ export type Initialized = { /** Changes only when a change would break clients written for the previous one. */ @@ -7442,8 +7472,10 @@ export type DriveDiff = { }; export type CollectionExportPlan = { - /** The `master.db` exported. */ + /** The `master.db` exported (empty for a plugin's library source). */ collection: string; + /** The plugin's library source written instead, `:`. */ + source?: string | null; target: string; target_root: string; /** The playlists chosen; empty for the whole collection. */ @@ -7472,7 +7504,10 @@ export type CollectionExportPlan = { * computer. */ syncing: string[]; - /** Tracks with an analysis file that can't be read: exported without it. */ + /** + * Tracks with an analysis file that can't be read, or (from a plugin's source) whose cues, + * beat grid or waveform can't be written: exported without it. + */ unreadable: string[]; /** * The target holds a library that wasn't exported from the collection, or files the export @@ -7527,14 +7562,15 @@ export type Listed = { version: string; capabilities: Granted[]; /** - * What it adds: its commands, operations, analyzers and providers by name, its export formats - * and its settings' keys. + * What it adds: its commands, operations, analyzers, providers and library sources by name, + * its export formats and its settings' keys. */ commands: string[]; operations: string[]; analyzers: string[]; exporters: string[]; providers: string[]; + sources: string[]; settings: string[]; /** It's running now: plugins start when they're first used, and stop once they're idle. */ running: boolean; @@ -7598,6 +7634,7 @@ export type Contributions = { analyzers: Analyzer[]; exporters: Exporter[]; providers: Provider[]; + sources: LibrarySource[]; settings: Setting[]; }; @@ -7699,6 +7736,17 @@ export type Provider = { kind: ProviderKind; }; +/** + * Read with `library/read`, a page at a time, for the host to build a drive from. It's given the + * folders the user grants it, which takes `fs:read`. + */ +export type LibrarySource = { + name: string; + title: string; + /** A JSON Schema for its options (an object). */ + options?: unknown; +}; + /** A setting the host stores and shows in its settings. */ export type Setting = { key: string; @@ -8881,6 +8929,10 @@ export interface Methods { "plugins.roles": { params: NoParams; result: RoleChoice[] }; /** immediate method */ "plugins.roles.set": { params: ChooseRole & WriteOptions; result: RoleChoice[] }; + /** plan method; paired with `plugins.source.apply`; long running: sends `progress`, can be cancelled; can delete or overwrite something */ + "plugins.source.plan": { params: SourceExport & WriteOptions; result: CollectionExportPlan & PlanFields }; + /** apply method; paired with `plugins.source.plan`; long running: sends `progress`, can be cancelled; can delete or overwrite something */ + "plugins.source.apply": { params: ApplyPlan | (SourceExport & WriteOptions); result: CollectionExportReport }; } export type MethodName = keyof Methods; diff --git a/apps/locallibrary/docs/rpc/schema.json b/apps/locallibrary/docs/rpc/schema.json index 1b622d23..dbc4297a 100644 --- a/apps/locallibrary/docs/rpc/schema.json +++ b/apps/locallibrary/docs/rpc/schema.json @@ -5353,6 +5353,64 @@ "$ref": "#/$defs/RoleChoice" } } + }, + { + "name": "plugins.source.plan", + "aliases": [], + "kind": "plan", + "pair": "plugins.source.apply", + "operation": "plugins.source", + "long_running": true, + "destructive": true, + "params": { + "allOf": [ + { + "$ref": "#/$defs/SourceExport" + }, + { + "$ref": "#/$defs/WriteOptions" + } + ] + }, + "result": { + "allOf": [ + { + "$ref": "#/$defs/CollectionExportPlan" + }, + { + "$ref": "#/$defs/PlanFields" + } + ] + } + }, + { + "name": "plugins.source.apply", + "aliases": [], + "kind": "apply", + "pair": "plugins.source.plan", + "operation": "plugins.source", + "long_running": true, + "destructive": true, + "params": { + "anyOf": [ + { + "$ref": "#/$defs/ApplyPlan" + }, + { + "allOf": [ + { + "$ref": "#/$defs/SourceExport" + }, + { + "$ref": "#/$defs/WriteOptions" + } + ] + } + ] + }, + "result": { + "$ref": "#/$defs/CollectionExportReport" + } } ], "notifications": { @@ -11977,6 +12035,63 @@ "role" ] }, + "SourceExport": { + "type": "object", + "properties": { + "plugin": { + "type": "string" + }, + "source": { + "type": "string", + "description": "One of its library sources, by name." + }, + "options": { + "description": "As the source's `options` schema says.", + "default": null + }, + "grants": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The folders the plugin may read files in, absolute: granted for this read alone, and only\naudio files in them are copied." + }, + "drive": { + "type": "string", + "description": "The drive or folder to write to." + }, + "playlists": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Playlists or folders (paths) to export, with the tracks they use. Empty: every track and\nplaylist, or the selection the target was last exported with.", + "default": [] + }, + "all": { + "type": "boolean", + "description": "Export every track and playlist, whatever the target was exported with last.", + "default": false + }, + "replace": { + "type": "boolean", + "description": "Replace a library the target holds that wasn't exported from the collection, or the\nfiles on a target with no library.", + "default": false + }, + "overwrite_archive": { + "type": "boolean", + "description": "Write to an archive or retired drive (see [`crate::fleet::check_target`]).", + "default": false + } + }, + "required": [ + "plugin", + "source", + "grants", + "drive" + ], + "description": "Building a drive from a plugin's library source." + }, "Initialized": { "type": "object", "properties": { @@ -29077,7 +29192,14 @@ "properties": { "collection": { "type": "string", - "description": "The `master.db` exported." + "description": "The `master.db` exported (empty for a plugin's library source)." + }, + "source": { + "type": [ + "string", + "null" + ], + "description": "The plugin's library source written instead, `:`." }, "target": { "type": "string" @@ -29167,7 +29289,7 @@ "items": { "type": "string" }, - "description": "Tracks with an analysis file that can't be read: exported without it." + "description": "Tracks with an analysis file that can't be read, or (from a plugin's source) whose cues,\nbeat grid or waveform can't be written: exported without it." }, "replaces_library": { "type": "boolean", @@ -29350,7 +29472,7 @@ "items": { "type": "string" }, - "description": "What it adds: its commands, operations, analyzers and providers by name, its export formats\nand its settings' keys." + "description": "What it adds: its commands, operations, analyzers, providers and library sources by name,\nits export formats and its settings' keys." }, "operations": { "type": "array", @@ -29376,6 +29498,12 @@ "type": "string" } }, + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, "settings": { "type": "array", "items": { @@ -29404,6 +29532,7 @@ "analyzers", "exporters", "providers", + "sources", "settings", "running", "left_out" @@ -29448,7 +29577,8 @@ "tags:plan", "cues:plan", "artwork:plan", - "analysis:plan" + "analysis:plan", + "fs:read" ] }, { @@ -29536,6 +29666,7 @@ "analyzers": [], "exporters": [], "providers": [], + "sources": [], "settings": [] } }, @@ -29595,6 +29726,13 @@ }, "default": [] }, + "sources": { + "type": "array", + "items": { + "$ref": "#/$defs/LibrarySource" + }, + "default": [] + }, "settings": { "type": "array", "items": { @@ -29609,6 +29747,7 @@ "analyzers", "exporters", "providers", + "sources", "settings" ] }, @@ -29835,6 +29974,25 @@ ], "description": "Asked with `provide/candidates` for what it has for tracks, of its kind: the host offers the\nuser every provider's, and makes the edits chosen itself. It's given the tracks' tags, which\ntakes `library:read`, and needs no capability to plan an edit." }, + "LibrarySource": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "title": { + "type": "string" + }, + "options": { + "description": "A JSON Schema for its options (an object)." + } + }, + "required": [ + "name", + "title" + ], + "description": "Read with `library/read`, a page at a time, for the host to build a drive from. It's given the\nfolders the user grants it, which takes `fs:read`." + }, "Setting": { "type": "object", "properties": { diff --git a/apps/locallibrary/docs/sidecar.md b/apps/locallibrary/docs/sidecar.md index d5623549..47f0938e 100644 --- a/apps/locallibrary/docs/sidecar.md +++ b/apps/locallibrary/docs/sidecar.md @@ -434,6 +434,15 @@ candidate as it was offered, with the images picked, or all of `images`) answers [{plugin, name, title, kind}], auto_pick}`, and `plugins.providers.set {auto_pick}` (0 to 1, more than 1: never; `null`: back to 0.9) sets it. +### Library sources + +`plugins.source.plan {plugin, source, options, grants, drive, playlists?, all?, replace?}` +(`grants`: absolute folders the plugin may read files in, for this build) answers a +`collection.export.plan`-shaped plan (`.source.`, with `source`: +`:`) building the drive or folder `drive` from what the plugin reads: only audio +files in the folders granted are copied, and what's left out is in its `warnings`. Applied by +`plugins.source.apply {plan_id}`, answering what `collection.export.apply` does. + ### Roles `plugins.roles` answers which analyzer each role (`key`, `bpm`, `loudness`...) is read from, diff --git a/apps/locallibrary/src/collection_cmd.rs b/apps/locallibrary/src/collection_cmd.rs index 42591c30..67505fe5 100644 --- a/apps/locallibrary/src/collection_cmd.rs +++ b/apps/locallibrary/src/collection_cmd.rs @@ -1,16 +1,19 @@ //! `locallibrary collection`: rekordbox's own collection (`master.db`) on this computer: what's in -//! it, its tracks and playlists, what a drive lacks against it, and exporting it to a drive or -//! folder. +//! it, its tracks and playlists, what a drive lacks against it, and exporting it (or a plugin's +//! library source, `--from`) to a drive or folder. use std::path::PathBuf; use std::process::ExitCode; -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use clap::{Args, Subcommand}; use console::style; use locallibrary::api; use locallibrary::collection_export::CollectionExportOptions; +use locallibrary::collection_export::CollectionExportPlan; use locallibrary::masterdb::CollectionInfo; +use locallibrary::op::Planned; +use locallibrary::plugin::SourceExport; use locallibrary::progress::Ctx; use locallibrary::query::{self, Filter}; @@ -58,11 +61,18 @@ pub enum CollectionCommand { }, /// The collection's playlist tree (intelligent playlists show empty) Playlists, - /// Write the collection to a drive or folder, as rekordbox's Export does + /// Write the collection (or a plugin's library source) to a drive or folder, as rekordbox's + /// Export does #[command( after_help = "Tracks Cloud Library Sync keeps in Dropbox or Google Drive are read from there (online-only \ files are downloaded). Exporting again copies only what changed. A folder exported to is a drive like any other: \ -`locallibrary pull FOLDER` saves it in the store, and `library push` writes it to a USB drive." +`locallibrary pull FOLDER` saves it in the store, and `library push` writes it to a USB drive. + +With --from, a plugin reads the library instead (`locallibrary plugin info` lists their sources): only audio files in \ +the folders you --grant it are copied, and cues and beat grids it brings are written to new analysis files.", + after_long_help = "Examples: + locallibrary collection export /Volumes/USB + locallibrary collection export /Volumes/USB --from PLUGIN:SOURCE --grant ~/Music" )] Export { /// The drive or folder to write to (a folder: its path) @@ -80,6 +90,16 @@ files are downloaded). Exporting again copies only what changed. A folder export /// Show the plan without changing anything #[arg(long)] dry_run: bool, + /// Build it from a plugin's library source instead, as PLUGIN:SOURCE + #[arg(long, value_name = "PLUGIN:SOURCE")] + from: Option, + /// With --from: a folder the plugin may read files in (repeatable) + #[arg(long = "grant", value_name = "FOLDER", requires = "from")] + grants: Vec, + /// With --from: an option for the source, like key=value (repeatable; `locallibrary plugin + /// info` shows them) + #[arg(long = "option", value_name = "KEY=VALUE", requires = "from")] + options: Vec, }, /// What a drive lacks against the collection: tracks, playlists and playlist entries Diff { @@ -130,9 +150,23 @@ pub fn run(args: CollectionArgs, store: &Option, out: Output, ctx: &Ctx let c = api::open_collection(&req)?; out.emit_paged(&c.library.playlists, |_| crate::render::playlist_tree(&c.library)); } - CollectionCommand::Export { drive, playlists, all, replace, dry_run } => { - let options = CollectionExportOptions { playlists, all, replace, overwrite_archive: false }; - return export(api::CollectionExportRequest { collection: req, drive, options }, dry_run, store, out, ctx); + CollectionCommand::Export { drive, playlists, all, replace, dry_run, from, grants, options } => { + let export = CollectionExportOptions { playlists, all, replace, overwrite_archive: false }; + let plan = match from { + Some(_) if req.db.is_some() || req.key.is_some() => { + bail!("--db and --key are for rekordbox's collection, not --from") + } + Some(from) => { + let existing = crate::diff_cmd::existing_store(store)?; + let req = from_source(&from, &grants, &options, drive.clone(), export)?; + crate::plugins::host()?.plan_source_export(&req, existing.as_ref(), ctx)? + } + None => { + let req = api::CollectionExportRequest { collection: req, drive: drive.clone(), options: export }; + api::collection_export_plan(&req, ctx)? + } + }; + return write(plan, &drive, dry_run, store, out, ctx); } CollectionCommand::Diff { drive, playlist } => { let d = api::collection_diff(&api::CollectionDiffRequest { collection: req, drive, playlist })?; @@ -159,17 +193,35 @@ pub fn run(args: CollectionArgs, store: &Option, out: Output, ctx: &Ctx Ok(ExitCode::SUCCESS) } -fn export( - req: api::CollectionExportRequest, +/// Building `drive` from `from`, a plugin's library source (`PLUGIN:SOURCE`), reading the folders +/// `grants` (made absolute) with the source's `--option`s. +fn from_source( + from: &str, + grants: &[PathBuf], + options: &[String], + drive: String, + export: CollectionExportOptions, +) -> Result { + let (plugin, source) = from.split_once(':').with_context(|| format!("--from takes PLUGIN:SOURCE, not {from:?}"))?; + let declared = crate::plugins::host()?.manifest(plugin)?.contributes.sources.iter().find(|s| s.name == source); + let declared = declared.with_context(|| format!("the {plugin} plugin has no library source {source:?}"))?; + let options = crate::plugin_cmd::options("source", declared.options.as_ref(), options)?; + let grants = grants.iter().map(std::path::absolute).collect::>()?; + Ok(SourceExport { plugin: plugin.into(), source: source.into(), options, grants, drive, export }) +} + +/// Shows what writing `plan` to its drive (`drive`, as it was given) does, asks first, and writes it. +fn write( + plan: Planned, + drive: &str, dry_run: bool, store: &Option, out: Output, ctx: &Ctx<'_>, ) -> Result { - let plan = api::collection_export_plan(&req, ctx)?; if !out.json { let what = match plan.playlists.as_slice() { - [] => "rekordbox's collection".to_string(), + [] => plan.what(), p => p.join(", "), }; say!( @@ -198,13 +250,17 @@ fn export( )) ); } + for s in plan.summary.iter().filter(|s| s.starts_with("read the files")) { + say!(" {}", dim(s)); + } for w in &plan.warnings { warn(w); } } - let question = match plan.replaces_library { - true => format!("Replace the library on {} with the collection?", plan.target), - false => format!("Export to {}?", plan.target), + let question = match (plan.replaces_library, &plan.source) { + (true, None) => format!("Replace the library on {} with the collection?", plan.target), + (true, Some(source)) => format!("Replace the library on {} with {source}?", plan.target), + (false, _) => format!("Export to {}?", plan.target), }; if let Some(o) = out.decide(dry_run, &question) { // A folder made for the export goes again, as nothing was written to it. @@ -228,7 +284,7 @@ fn export( )); output::hint(format!("Save it in the store with: locallibrary pull {:?}", plan.target_root)); if let Some(b) = &report.backup { - output::hint(format!("undo with: locallibrary undo {:?} (backup {b})", req.drive)); + output::hint(format!("undo with: locallibrary undo {drive:?} (backup {b})")); } })) } diff --git a/apps/locallibrary/src/export_cmd.rs b/apps/locallibrary/src/export_cmd.rs index 146d27ec..72e552a2 100644 --- a/apps/locallibrary/src/export_cmd.rs +++ b/apps/locallibrary/src/export_cmd.rs @@ -158,7 +158,7 @@ fn run_plugin( .manifests() .find_map(|m| Some((m.id.as_str(), m.contributes.exporters.iter().find(|e| e.format == format)?))) .expect("--format takes the formats there are"); - let options = crate::plugin_cmd::options(exporter.options.as_ref(), &args.options)?; + let options = crate::plugin_cmd::options("format", exporter.options.as_ref(), &args.options)?; let source = ExportSource { drive: args.drive, playlists: args.playlist.into_iter().collect(), diff --git a/apps/locallibrary/src/plugin_cmd/contributed.rs b/apps/locallibrary/src/plugin_cmd/contributed.rs index 3eee926f..47d03f3d 100644 --- a/apps/locallibrary/src/plugin_cmd/contributed.rs +++ b/apps/locallibrary/src/plugin_cmd/contributed.rs @@ -252,9 +252,9 @@ fn params_of(schema: Option<&Value>, m: &ArgMatches) -> Result { Ok(Value::Object(params)) } -/// An exporter's options from `--option KEY=VALUE`s (one given again adds to a list or a map), -/// each read as its schema says. A key it doesn't take is refused. -pub fn options(schema: Option<&Value>, given: &[String]) -> Result { +/// An exporter's or a library source's (`what`) options from `--option KEY=VALUE`s (one given +/// again adds to a list or a map), each read as its schema says. A key it doesn't take is refused. +pub fn options(what: &str, schema: Option<&Value>, given: &[String]) -> Result { let mut raws: Vec<(&str, Vec<&str>)> = Vec::new(); for pair in given { let (key, raw) = @@ -271,8 +271,8 @@ pub fn options(schema: Option<&Value>, given: &[String]) -> Result { let property = known.ok_or_else(|| { let all: Vec<&str> = properties(schema).map(|(name, _)| name.as_str()).collect(); usage(match all.is_empty() { - true => format!("this format has no options, not even {key:?}"), - false => format!("this format has no option {key:?}: its options are {}", all.join(", ")), + true => format!("this {what} has no options, not even {key:?}"), + false => format!("this {what} has no option {key:?}: its options are {}", all.join(", ")), }) })?; options.insert(key.to_string(), params::value(property, &raws, &format!("the {key} option")).map_err(usage)?); diff --git a/apps/locallibrary/src/plugin_cmd/contributed/tests.rs b/apps/locallibrary/src/plugin_cmd/contributed/tests.rs index 22a36d5d..f15e093d 100644 --- a/apps/locallibrary/src/plugin_cmd/contributed/tests.rs +++ b/apps/locallibrary/src/plugin_cmd/contributed/tests.rs @@ -141,7 +141,8 @@ fn options_take_their_schemas_types() { "root": { "type": "string", "x-path": true }, }, }); - let given = |pairs: &[&str]| options(Some(&schema), &pairs.iter().map(|p| p.to_string()).collect::>()); + let given = + |pairs: &[&str]| options("format", Some(&schema), &pairs.iter().map(|p| p.to_string()).collect::>()); let o = given(&["level=2", "loud=yes", "lists=a", "lists=b"]).unwrap(); assert_eq!(o, json!({ "level": 2, "loud": true, "lists": ["a", "b"] })); assert!(Path::new(given(&["root=out"]).unwrap()["root"].as_str().unwrap()).is_absolute()); @@ -541,7 +542,7 @@ fn options_drawn_from_a_schema_are_typed_or_refused(tc: hegel::TestCase) { } _ => None, }; - let got = options(Some(&schema), &typed); + let got = options("format", Some(&schema), &typed); match refused { Some(message) => assert_eq!(got.unwrap_err().to_string(), message), None => assert_eq!(got.unwrap(), Value::Object(expected), "{typed:?}"), diff --git a/apps/locallibrary/src/plugin_cmd/listing.rs b/apps/locallibrary/src/plugin_cmd/listing.rs index 04535712..bce976bd 100644 --- a/apps/locallibrary/src/plugin_cmd/listing.rs +++ b/apps/locallibrary/src/plugin_cmd/listing.rs @@ -43,6 +43,7 @@ pub(super) fn list(all: &[Listed]) { named("analyzer", &p.analyzers), named("export format", &p.exporters), named("provider", &p.providers), + named("library source", &p.sources), named("setting", &p.settings), ] .into_iter() @@ -111,6 +112,9 @@ pub(super) fn info(d: &Described) { (p.name.clone(), format!("{}: {}, for locallibrary {verb}", p.title, p.kind.found())) }); section("Providers:", providers.collect()); + let sources = c.sources.iter().map(|x| (format!("{}:{}", m.id, x.name), x.title.clone())); + let build = "locallibrary collection export DRIVE --from PLUGIN:SOURCE --grant FOLDER"; + section(&format!("Library sources ({build}):"), sources.collect()); let settings = c.settings.iter().map(|s| (s.key.clone(), format!("{}: {}", s.title, shown(s, &d.settings, &d.secrets)))); section(&format!("Settings (locallibrary plugin settings {}):", m.id), settings.collect()); diff --git a/apps/locallibrary/src/plugins.rs b/apps/locallibrary/src/plugins.rs index 2aa2edac..497df354 100644 --- a/apps/locallibrary/src/plugins.rs +++ b/apps/locallibrary/src/plugins.rs @@ -373,6 +373,7 @@ fn means(c: &Capability) -> String { Capability::CuesPlan => "plan cue changes, which you confirm", Capability::ArtworkPlan => "plan cover art changes, which you confirm", Capability::AnalysisPlan => "plan beat grid, phrase and waveform changes, which you confirm", + Capability::FsRead => "read audio files in folders you grant it, to build a drive from", Capability::Net(host) => { return match host.strip_prefix("*.") { Some(domain) => format!("talk to any server under {domain}"), @@ -390,13 +391,14 @@ pub struct Listed { pub name: String, pub version: String, pub capabilities: Vec, - /// What it adds: its commands, operations, analyzers and providers by name, its export formats - /// and its settings' keys. + /// What it adds: its commands, operations, analyzers, providers and library sources by name, + /// its export formats and its settings' keys. pub commands: Vec, pub operations: Vec, pub analyzers: Vec, pub exporters: Vec, pub providers: Vec, + pub sources: Vec, pub settings: Vec, /// It's running now: plugins start when they're first used, and stop once they're idle. pub running: bool, @@ -418,6 +420,7 @@ impl Listed { analyzers: c.analyzers.iter().map(|x| x.name.clone()).collect(), exporters: c.exporters.iter().map(|x| x.format.clone()).collect(), providers: c.providers.iter().map(|x| x.name.clone()).collect(), + sources: c.sources.iter().map(|x| x.name.clone()).collect(), settings: c.settings.iter().map(|x| x.key.clone()).collect(), running: plugins.is_running(&m.id), left_out: None, diff --git a/apps/locallibrary/src/rpc.rs b/apps/locallibrary/src/rpc.rs index 186a96e2..16689466 100644 --- a/apps/locallibrary/src/rpc.rs +++ b/apps/locallibrary/src/rpc.rs @@ -203,8 +203,9 @@ impl Plans { /// The operation (see [`methods::Method::operation`]) of the `.apply` that applies a plan of /// `operation`, for `method`: the plan's own (the host's `plugins.*`, too), but a plugin's -/// operation (`.`) is `plugins.op`'s and its export (`.export.`) -/// `plugins.export`'s. +/// operation (`.`) is `plugins.op`'s, its export (`.export.`) +/// `plugins.export`'s and a drive built from its library source (`.source.`) +/// `plugins.source`'s. fn applied_as<'a>(method: &str, operation: &'a str) -> &'a str { if !method.starts_with(PLUGINS) { return operation; @@ -212,6 +213,7 @@ fn applied_as<'a>(method: &str, operation: &'a str) -> &'a str { match operation.split('.').collect::>()[..] { ["plugins", _] => operation, [_, "export", _] => "plugins.export", + [_, "source", _] => "plugins.source", _ => "plugins.op", } } diff --git a/apps/locallibrary/src/rpc/methods/plugins.rs b/apps/locallibrary/src/rpc/methods/plugins.rs index 30ef097e..38563da4 100644 --- a/apps/locallibrary/src/rpc/methods/plugins.rs +++ b/apps/locallibrary/src/rpc/methods/plugins.rs @@ -9,6 +9,7 @@ mod exporters; mod operations; mod providers; mod roles; +mod sources; use loclib_plugin::Context; use schemars::JsonSchema; @@ -87,4 +88,5 @@ pub(super) fn register(r: &mut Registry) { analyzers::register(r); providers::register(r); roles::register(r); + sources::register(r); } diff --git a/apps/locallibrary/src/rpc/methods/plugins/sources.rs b/apps/locallibrary/src/rpc/methods/plugins/sources.rs new file mode 100644 index 00000000..755eb8d7 --- /dev/null +++ b/apps/locallibrary/src/rpc/methods/plugins/sources.rs @@ -0,0 +1,23 @@ +//! `plugins.source.*`: a drive built from a plugin's library source, planned (what it reads from +//! the folders granted, and what's copied and removed) and written. + +use locallibrary::collection_export::{CollectionExportOptions, CollectionExportPlan}; +use locallibrary::op::Planned; +use locallibrary::plugin::SourceExport; + +use crate::plugins::host; +use crate::rpc::RpcError; +use crate::rpc::methods::{Call, PlanStore, Registry}; + +pub(super) fn register(r: &mut Registry) { + r.plan("plugins.source.plan", plan).long().destructive(); + r.apply("plugins.source.apply", PlanStore::Optional, |c, req: SourceExport| { + Ok(plan(c, req)?.apply(c.store_opt().as_ref(), c.ctx)?) + }) + .long() + .destructive(); +} + +fn plan(c: &Call<'_>, req: SourceExport) -> Result, RpcError> { + Ok(host()?.plan_source_export(&req, c.existing()?.as_ref(), c.ctx)?) +} diff --git a/apps/locallibrary/src/rpc/tests.rs b/apps/locallibrary/src/rpc/tests.rs index fb775ffa..3dedc89b 100644 --- a/apps/locallibrary/src/rpc/tests.rs +++ b/apps/locallibrary/src/rpc/tests.rs @@ -250,15 +250,17 @@ impl AnyPlanned for Pretend { /// The apply a plan of `operation` kept by `method` is for, as the protocol has it: a plan of /// the sidecar's own by the apply of its operation, a plugin's export (`.export.`) -/// by `plugins.export` and any other plugin operation by `plugins.op`. +/// by `plugins.export`, a drive built from its library source (`.source.`) by +/// `plugins.source` and any other plugin operation by `plugins.op`. fn applied_by(method: &str, operation: &str) -> String { if !method.starts_with("plugins.") { return operation.to_string(); } let parts: Vec<&str> = operation.split('.').collect(); - match parts.len() == 3 && parts[1] == "export" { - true => "plugins.export".into(), - false => "plugins.op".into(), + match (parts.len(), parts.get(1)) { + (3, Some(&"export")) => "plugins.export".into(), + (3, Some(&"source")) => "plugins.source".into(), + _ => "plugins.op".into(), } } @@ -275,10 +277,16 @@ fn plans_are_kept_and_taken_as_modelled(tc: TestCase) { ("plugins.op.plan", "echo.playlist"), ("plugins.op.plan", "echo.export.m3u"), ("plugins.export.plan", "tidy.export.csv"), + ("plugins.source.plan", "tidy.source.folder"), ]; let makers = [&[("store.gc.plan", "store.gc"), ("edit.plan", "edit")][..], &theirs[..]]; let applies = [("store.gc.apply", "store.gc"), ("edit.apply", "edit")]; - let applies = [&applies[..], &[("plugins.op.apply", "plugins.op"), ("plugins.export.apply", "plugins.export")]]; + let their_applies = [ + ("plugins.op.apply", "plugins.op"), + ("plugins.export.apply", "plugins.export"), + ("plugins.source.apply", "plugins.source"), + ]; + let applies = [&applies[..], &their_applies[..]]; let mut model: [VecDeque<(String, String)>; 2] = Default::default(); let mut made = 0; for _ in 0..tc.draw(gs::integers::().min_value(1).max_value(12)) { diff --git a/apps/locallibrary/src/tui/app.rs b/apps/locallibrary/src/tui/app.rs index 39687331..41bed0e9 100644 --- a/apps/locallibrary/src/tui/app.rs +++ b/apps/locallibrary/src/tui/app.rs @@ -3339,6 +3339,7 @@ impl App { Act::PluginSettings(plugin) => self.plugin_settings(plugin), Act::ProviderSettings => self.provider_settings(), Act::Export(plugin, format) => self.plugin_export(plugin, format, self.action_drive()), + Act::Source(plugin, source) => self.plugin_source(plugin, source, self.action_drive()), Act::Fetch(kind) => { let target = (self.view == View::Library).then(|| self.target()).flatten(); self.fetch_form(kind, self.action_drive(), target.as_ref()); diff --git a/apps/locallibrary/src/tui/drives.rs b/apps/locallibrary/src/tui/drives.rs index 8ae9fcb7..955238ab 100644 --- a/apps/locallibrary/src/tui/drives.rs +++ b/apps/locallibrary/src/tui/drives.rs @@ -327,6 +327,7 @@ impl App { Act::Analyze(plugin, analyzer) => self.analyze(plugin, analyzer, Some(root)), Act::ApplyFields(plugin, analyzer) => self.apply_fields(plugin, analyzer, Some(root)), Act::Export(plugin, format) => self.plugin_export(plugin, format, Some(root)), + Act::Source(plugin, source) => self.plugin_source(plugin, source, Some(root)), act => self.perform(act, None), } } diff --git a/apps/locallibrary/src/tui/edits.rs b/apps/locallibrary/src/tui/edits.rs index a4110b1d..6400a3bf 100644 --- a/apps/locallibrary/src/tui/edits.rs +++ b/apps/locallibrary/src/tui/edits.rs @@ -14,6 +14,7 @@ use std::sync::mpsc::Sender; use std::time::{Duration, Instant}; use locallibrary::api::{self, EditRequest}; +use locallibrary::collection_export::{CollectionExportOptions, CollectionExportPlan}; use locallibrary::edit::{Edit, EditOptions, EditPlan, TrackFields}; use locallibrary::model::TrackColor; use locallibrary::op::{AnyPlanned, Planned}; @@ -517,6 +518,15 @@ pub fn wording(plan: &dyn AnyPlanned, drive: &str) -> Wording { } let written: Vec<&str> = (plan.drives().iter()).filter(|t| t.writes && t.drive).map(|t| t.name.as_str()).collect(); let who = if written.is_empty() { drive.to_string() } else { written.join(", ") }; + // Building a drive from a plugin's library source: the only collection export the player plans. + if let Some(b) = plan.as_any().downcast_ref::>() { + let ask = match b.plan.replaces_library { + true => format!("Replace the library on {drive} with {}?", b.plan.what()), + false => format!("Build {drive} from {}?", b.plan.what()), + }; + let what = "build it from a library source".into(); + return Wording { ask, yes: "build", who, what, doing: format!("Building {drive}") }; + } let w = |ask: String, yes, what: &str, doing: String| Wording { ask, yes, who: who.clone(), what: what.into(), doing }; match plan.operation() { diff --git a/apps/locallibrary/src/tui/form.rs b/apps/locallibrary/src/tui/form.rs index d89ec528..0d716ee0 100644 --- a/apps/locallibrary/src/tui/form.rs +++ b/apps/locallibrary/src/tui/form.rs @@ -1,5 +1,5 @@ -//! A form for a plugin's params, its settings (or providers'), an export's, or fetching from -//! providers: a field for each property of a JSON Schema, each typed as the command line takes it +//! A form for a plugin's params, its settings (or providers'), an export's, a build from a library +//! source, or fetching from providers: a field for each property of a JSON Schema, each typed as the command line takes it //! (see [`crate::plugins::params`]), and checked against the schema before anything runs or is //! saved. A switch and a choice of a few values are picked rather than typed. @@ -36,6 +36,9 @@ pub enum For { /// Where to export the drive at `root` (its `playlists` only, when some are named) in a /// plugin's format, and the format's options. Export { plugin: &'static str, format: &'static str, root: std::path::PathBuf, playlists: Vec }, + /// The folders a plugin's library source (`source`) may read for building the drive at `root`, + /// whether to replace a library there, and the source's options. + Source { plugin: &'static str, source: &'static str, root: std::path::PathBuf }, /// What to write to the tracks of the drive at `root` from what a plugin's analyzer worked /// out. ApplyFields { plugin: &'static str, analyzer: &'static str, root: std::path::PathBuf }, @@ -190,6 +193,7 @@ impl App { For::Export { plugin, format, root, playlists } => { self.plan_export(plugin, format, root, playlists, params) } + For::Source { plugin, source, root } => self.plan_source(plugin, source, root, params), For::ApplyFields { plugin, analyzer, root } => { self.plan_apply_fields(plugin, analyzer, root, params) } diff --git a/apps/locallibrary/src/tui/keymap.rs b/apps/locallibrary/src/tui/keymap.rs index 436407bc..11a48761 100644 --- a/apps/locallibrary/src/tui/keymap.rs +++ b/apps/locallibrary/src/tui/keymap.rs @@ -196,6 +196,8 @@ pub enum Act { ProviderSettings, /// Export the drive in a plugin's format (the plugin's id, the format). Export(&'static str, &'static str), + /// Build the drive from a plugin's library source (the plugin's id, the source). + Source(&'static str, &'static str), /// Fetch what plugins' providers of a kind find for the tracks selected. Fetch(loclib_plugin::ProviderKind), } diff --git a/apps/locallibrary/src/tui/plugins.rs b/apps/locallibrary/src/tui/plugins.rs index cc059648..deb76a46 100644 --- a/apps/locallibrary/src/tui/plugins.rs +++ b/apps/locallibrary/src/tui/plugins.rs @@ -12,6 +12,7 @@ mod analyzers; mod exporters; pub(super) mod providers; +mod sources; use std::path::{Path, PathBuf}; @@ -139,7 +140,8 @@ fn named(title: &str) -> String { } /// Who planned `operation` (as a plan is called), if plugins did: the name of the plugin whose -/// operation (`.`) or, with `export`, export (`.export.`) it is; or +/// operation (`.`), library source (`.source.`) or, with `export`, +/// export (`.export.`) it is; or /// `Plugins` for an edit the host plans from what plugins found or worked out (`plugins.provide`, /// `plugins.apply_fields`). pub fn planned_by(operation: &str, export: bool) -> Option<&'static str> { @@ -153,7 +155,10 @@ pub fn planned_by(operation: &str, export: bool) -> Option<&'static str> { (Some(rest), true) => { rest.strip_prefix("export.").is_some_and(|f| c.exporters.iter().any(|e| e.format == f)) } - (Some(name), false) => c.operations.iter().any(|o| o.name == name), + (Some(name), false) => { + let source = name.strip_prefix("source.").is_some_and(|s| c.sources.iter().any(|x| x.name == s)); + source || c.operations.iter().any(|o| o.name == name) + } (None, _) => false, } }); @@ -259,8 +264,8 @@ impl App { ); } let Ok(plugins) = crate::plugins::host() else { return items }; - // An analyzer reads the tracks' audio files and an exporter the library: on a drive plugged - // in, as a command. + // An analyzer reads the tracks' audio files, an exporter the library, and a library source + // writes one: on a drive plugged in, as a command. let why = match drive { None => Some("open a drive first"), Some(false) => Some("only on a plugged-in drive"), @@ -271,6 +276,7 @@ impl App { for m in plugins.manifests() { items.extend(analyzers::items(m, why)); items.extend(exporters::items(m, why)); + items.extend(sources::items(m, why)); } items } diff --git a/apps/locallibrary/src/tui/plugins/sources.rs b/apps/locallibrary/src/tui/plugins/sources.rs new file mode 100644 index 00000000..c7bd3133 --- /dev/null +++ b/apps/locallibrary/src/tui/plugins/sources.rs @@ -0,0 +1,84 @@ +//! Library sources in the player: a drive built from a plugin's library source, planned (what it +//! reads in the folders granted, what's copied and removed) and confirmed before it's written. + +use std::path::PathBuf; + +use locallibrary::Store; +use locallibrary::collection_export::CollectionExportOptions; +use locallibrary::plugin::SourceExport; +use loclib_plugin::Manifest; +use serde_json::{Map, Value, json}; + +use crate::plugins::params::properties; +use crate::tui::app::App; +use crate::tui::edits::Modal; +use crate::tui::form::{For, Form}; +use crate::tui::keymap::Act; +use crate::tui::palette::Item; + +/// The form's own fields, beside the source's options: the folders granted, and whether to +/// replace a library the drive holds. +const READ_FROM: &str = "read_from"; +const REPLACE: &str = "replace"; + +/// `m`'s library sources as the palette and a drive's menu offer them, each unavailable for `why`. +pub(super) fn items(m: &'static Manifest, why: Option<&str>) -> Vec<(Act, Item)> { + let sources = m.contributes.sources.iter().map(|s| { + let label = format!("build from {}…", super::named(&s.title)); + (Act::Source(&m.id, &s.name), Item { unavailable: why.map(String::from), ..Item::new(label, &m.name) }) + }); + sources.collect() +} + +impl App { + /// Build the drive at `root` from a plugin's library source: a form asks for the folders it + /// may read files in (`~/Music` to begin with) and for the source's options first. + pub(in crate::tui) fn plugin_source(&mut self, plugin: &'static str, source: &'static str, root: Option) { + let Ok(plugins) = crate::plugins::host() else { return }; + let Some(root) = self.plugged(root) else { return }; + let declared = + plugins.manifest(plugin).ok().and_then(|m| m.contributes.sources.iter().find(|s| s.name == source)); + let Some(s) = declared else { return }; + let mut fields = Map::new(); + let folders = "Folders it may read files in, on this computer"; + let read_from = + json!({ "type": "array", "items": { "type": "string" }, "x-path": true, "description": folders }); + fields.insert(READ_FROM.into(), read_from); + let replace = "Replace a library the drive holds that wasn't built from it"; + fields.insert(REPLACE.into(), json!({ "type": "boolean", "description": replace })); + fields.extend(properties(s.options.as_ref()).map(|(k, v)| (k.clone(), v.clone()))); + let mut schema = s.options.clone().unwrap_or_else(|| json!({ "type": "object" })); + schema["properties"] = Value::Object(fields); + let mut required = vec![Value::from(READ_FROM)]; + required.extend(schema["required"].as_array().cloned().unwrap_or_default()); + schema["required"] = Value::Array(required); + let filled = Map::from_iter([(READ_FROM.to_string(), json!(["~/Music"]))]); + let form = Form::with(s.title.clone(), Some(schema), &filled, For::Source { plugin, source, root }); + self.modal = Some(Modal::Form(form)); + } + + /// Plan the build the form has (`given`: the folders, whether to replace, and the source's + /// options) in the background: the plugin reads its library, and the plan comes back to + /// confirm, as any change does. + pub(in crate::tui) fn plan_source( + &mut self, + plugin: &'static str, + source: &'static str, + root: PathBuf, + given: Value, + ) { + let Ok(plugins) = crate::plugins::host() else { return }; + let mut options = given; + let mut take = |key: &str| options.as_object_mut().and_then(|o| o.remove(key)); + let grants = take(READ_FROM).and_then(|v| serde_json::from_value::>(v).ok()).unwrap_or_default(); + let replace = take(REPLACE) == Some(Value::Bool(true)); + let export = CollectionExportOptions { replace, ..CollectionExportOptions::default() }; + let drive = root.display().to_string(); + let req = SourceExport { plugin: plugin.into(), source: source.into(), options, grants, drive, export }; + let store_root = self.store_root.clone(); + self.plan(root, move |ctx| { + let store = Store::open(&store_root).ok(); + Ok(Box::new(plugins.plan_source_export(&req, store.as_ref(), ctx)?)) + }); + } +} diff --git a/apps/locallibrary/src/tui/plugins/tests.rs b/apps/locallibrary/src/tui/plugins/tests.rs index c5ceb006..1a21eb7e 100644 --- a/apps/locallibrary/src/tui/plugins/tests.rs +++ b/apps/locallibrary/src/tui/plugins/tests.rs @@ -1936,3 +1936,4 @@ fn a_plugins_plan_is_confirmed_and_applied_as_it_says(tc: TestCase) { } mod asking; +mod sources; diff --git a/apps/locallibrary/src/tui/plugins/tests/sources.rs b/apps/locallibrary/src/tui/plugins/tests/sources.rs new file mode 100644 index 00000000..4dfc9501 --- /dev/null +++ b/apps/locallibrary/src/tui/plugins/tests/sources.rs @@ -0,0 +1,193 @@ +//! Library sources in the player: "build from …" asks for the folders the plugin may read and the +//! source's options, then the plan is confirmed and written as any change is. + +use std::path::{Path, PathBuf}; + +use hegel::TestCase; +use hegel::generators as gs; +use ratatui::crossterm::event::KeyCode; + +use super::super::super::app::App; +use super::super::super::app::testing::{ + Plugged, app_on, confirming, demo_drive, demo_drive_in, echo_plugin, key, last_message, mounted, press, say_yes, + settle, wait, +}; +use super::super::super::edits::Modal; +use super::super::super::ui::screen; +use super::{Screen, entry, move_to, picker}; + +/// The echo plugin (in `dir`) granted `fs:read`, with the library source `folder`; and a folder +/// of music beside the drives, with one WAV file. +fn sourcing(dir: &Path) -> PathBuf { + echo_plugin(dir); + let file = dir.join("echo.toml"); + let manifest = std::fs::read_to_string(&file).unwrap(); + let granted = manifest.replace(r#""library:plan"]"#, r#""library:plan", "fs:read"]"#); + let source = "\n[[contributes.sources]]\nname = \"folder\"\ntitle = \"Audio files\"\n\ + options = { type = \"object\", properties = { per_page = { type = \"integer\" } } }\n"; + std::fs::write(&file, granted + source).unwrap(); + let music = dir.join("music"); + std::fs::create_dir_all(&music).unwrap(); + let pcm = loclib_mixid::test_util::synth(1.0, 1).into_iter().map(|s| (s * 12_000.0) as i32); + std::fs::write(music.join("a.wav"), loclib_mixid::test_util::wav_of(44_100, 16, pcm)).unwrap(); + music +} + +/// Fills the form open with `music` as the folder granted and `replace`, and submits it. +fn build(app: &mut App, music: &Path, replace: bool) { + press(app, ":build from audio files\n"); + let Some(Modal::Form(f)) = &mut app.modal else { panic!("no form: {}", last_message(app)) }; + assert_eq!(f.title, "Audio files"); + let names: Vec<&str> = f.fields.iter().map(|x| x.name.as_str()).collect(); + assert_eq!(names, ["read_from", "replace", "per_page"]); + assert_eq!(f.fields[0].text, "~/Music", "it starts at the music folder"); + f.fields[0].text = music.display().to_string(); + f.fields[1].text = replace.to_string(); + press(app, "\n"); +} + +/// On a drive plugged in, a library source builds it: a library already there is replaced only +/// when the form says so, and the plan says which folders the plugin read before it's written. +#[test] +fn a_drive_is_built_from_a_library_source() { + let (dir, root) = demo_drive(); + let music = sourcing(dir.path()); + let (mut blank, _store) = super::super::super::app::testing::blank(super::Screen::Library); + press(&mut blank, ":"); + assert_eq!(entry(&blank, "build from audio files…").1, Some("open a drive first")); + + mounted(dir.path()); + let mut app = app_on(&root); + build(&mut app, &music, false); + wait(&mut app, |a| a.job.is_none()); + let said = last_message(&app).to_string(); + assert!(said.contains("wasn't built from echo:folder; replace it"), "{said}"); + + build(&mut app, &music, true); + wait(&mut app, |a| a.job.is_none()); + let Some(Modal::Confirm(c)) = &app.modal else { panic!("no confirmation: {}", last_message(&app)) }; + assert_eq!(c.plan.operation(), "echo.source.folder"); + let s = screen(&mut app, 120, 40); + for needle in + ["Replace the library on", "with the library source echo:folder?", "read the files Audio files", "y build"] + { + assert!(s.contains(needle), "{needle:?} isn't in\n{s}"); + } + say_yes(&mut app); + let library = locallibrary::Device::open(&root).unwrap().library().unwrap(); + let titles: Vec<&str> = library.tracks.iter().map(|t| t.title.as_str()).collect(); + assert_eq!(titles, ["a"], "{}", last_message(&app)); +} + +/// The folder a property test's drives are mounted in, with [`sourcing`]'s plugin, and its +/// music folder: set up as the first case starts, for every case after it (see `plugins_once`). +fn sourcing_once() -> &'static (tempfile::TempDir, PathBuf) { + static DIR: std::sync::OnceLock<(tempfile::TempDir, PathBuf)> = std::sync::OnceLock::new(); + DIR.get_or_init(|| { + let dir = tempfile::tempdir().unwrap(); + let music = sourcing(dir.path()); + mounted(dir.path()); + (dir, music) + }) +} + +/// The folders a drawn build grants, as the form takes them. +#[derive(Debug, Clone, Copy, PartialEq)] +enum Granted { + Music, + /// The music folder, and again spelled another way. + MusicTwice, + /// A relative path, which the form takes from where the player runs: no folder there. + Relative, + Nowhere, + Nothing, +} + +/// From the palette or the drive's menu, with drawn folders, replacing or not and any page size: +/// with no folder the form stays open; a folder that isn't there, or a library on the drive not +/// replaced, is refused, saying so; esc while it's planned stops it. Otherwise the +/// confirmation names the source and the folders read, a no writes nothing, and a yes builds the +/// drive from the music folder. +#[hegel::test(test_cases = 12, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn a_drive_is_built_from_a_source_once_confirmed(tc: TestCase) { + static CASES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let (dir, music) = sourcing_once(); + let n = CASES.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let drive = Plugged(demo_drive_in(dir.path(), &format!("SRC{n}"))); + let titles = || -> Vec { + let library = locallibrary::Device::open(&drive.0).unwrap().library().unwrap(); + library.tracks.into_iter().map(|t| t.title).collect() + }; + let before = titles(); + let mut app = app_on(&drive.0); + if tc.draw(gs::booleans()) { + press(&mut app, ":build from audio files\n"); + } else { + let info = locallibrary::api::drive_info(&locallibrary::api::DriveRef { drive: drive.0.display().to_string() }); + app.drives = vec![info.unwrap()]; + app.drive_state.select(Some(0)); + app.view = Screen::Drives; + press(&mut app, "e"); + let at = picker(&app).items.iter().position(|i| i.label == "build from audio files…").unwrap(); + move_to(&mut app, at); + key(&mut app, KeyCode::Enter); + } + let granted = tc.draw(gs::sampled_from(vec![ + Granted::Music, + Granted::MusicTwice, + Granted::Relative, + Granted::Nowhere, + Granted::Nothing, + ])); + let replace = tc.draw(gs::booleans()); + let per_page = tc.draw(gs::sampled_from(vec!["", "1", "2"])); + let Some(Modal::Form(f)) = &mut app.modal else { panic!("no form: {}", last_message(&app)) }; + f.fields[0].text = match granted { + Granted::Music => music.display().to_string(), + Granted::MusicTwice => format!("{}, {}", music.display(), music.join(".").display()), + Granted::Relative => "music".into(), + Granted::Nowhere => dir.path().join("nowhere").display().to_string(), + Granted::Nothing => String::new(), + }; + f.fields[1].text = replace.to_string(); + f.fields[2].text = per_page.into(); + press(&mut app, "\n"); + if granted == Granted::Nothing { + assert!(matches!(app.modal, Some(Modal::Form(_))) && app.job.is_none(), "the form stays open"); + return; + } + assert_eq!(app.job.as_ref().map(|j| j.label.as_str()), Some("Planning")); + if tc.draw(gs::integers::().max_value(4)) == 0 { + press(&mut app, "\u{1b}"); + assert_eq!((app.job.is_none(), app.modal.is_none(), last_message(&app)), (true, true, "cancelled")); + assert_eq!(titles(), before); + return; + } + wait(&mut app, |a| a.job.is_none()); + let refused = match granted { + Granted::Relative | Granted::Nowhere => Some("not found: the folder"), + _ if !replace => Some("wasn't built from echo:folder; replace it"), + _ => None, + }; + if let Some(said) = refused { + assert!(app.modal.is_none() && last_message(&app).contains(said), "{said:?}: {}", last_message(&app)); + assert_eq!(titles(), before); + return; + } + let (operation, _) = confirming(&mut app); + assert_eq!(operation, "echo.source.folder"); + let Some(Modal::Confirm(c)) = &app.modal else { panic!() }; + let real = locallibrary::device::without_verbatim_prefix(music.canonicalize().unwrap()); + let read = format!("read the files Audio files finds in {}", real.display()); + assert!(c.plan.summary().contains(&read), "{:?}", c.plan.summary()); + let s = screen(&mut app, 120, 40); + assert!(s.contains("with the library source echo:folder?"), "{s}"); + settle(&mut app); + if tc.draw(gs::booleans()) { + press(&mut app, tc.draw(gs::sampled_from(vec!["n", "\u{1b}", "q"]))); + assert_eq!((last_message(&app), titles()), ("nothing written", before)); + return; + } + say_yes(&mut app); + assert_eq!(titles(), ["a"], "{}", last_message(&app)); +} diff --git a/apps/locallibrary/src/tui/ui.rs b/apps/locallibrary/src/tui/ui.rs index 6bfd1e71..fef7ea81 100644 --- a/apps/locallibrary/src/tui/ui.rs +++ b/apps/locallibrary/src/tui/ui.rs @@ -2553,6 +2553,7 @@ fn draw_form(f: &mut Frame, app: &mut App, area: Rect) { let then = match form.then { super::form::For::Run(_) | super::form::For::Export { .. } + | super::form::For::Source { .. } | super::form::For::ApplyFields { .. } | super::form::For::Fetch { .. } => "run", super::form::For::Settings(_) | super::form::For::ProviderSettings => "save", diff --git a/apps/locallibrary/tests/it/plugins.rs b/apps/locallibrary/tests/it/plugins.rs index 8dedfe72..c676f1dc 100644 --- a/apps/locallibrary/tests/it/plugins.rs +++ b/apps/locallibrary/tests/it/plugins.rs @@ -12,6 +12,7 @@ use crate::support::{Env, Rpc, echo_plugin, path, recording}; mod choose; mod props; mod providers; +mod sources; /// The echo manifest as these tests register it ([`echo_plugin`]): its `echo` command at /// `locallibrary say` taking flags, and its operation at `locallibrary playlist fresh`. diff --git a/apps/locallibrary/tests/it/plugins/props.rs b/apps/locallibrary/tests/it/plugins/props.rs index 36bc7093..5eca8435 100644 --- a/apps/locallibrary/tests/it/plugins/props.rs +++ b/apps/locallibrary/tests/it/plugins/props.rs @@ -31,7 +31,7 @@ fn toml(v: &Value) -> String { } /// Every capability a manifest can name but `net:`. -const CAPABILITIES: [&str; 16] = [ +const CAPABILITIES: [&str; 17] = [ "library:read", "analysis:read", "artwork:read", @@ -48,6 +48,7 @@ const CAPABILITIES: [&str; 16] = [ "tags:plan", "cues:plan", "artwork:plan", + "fs:read", ]; /// Text a person reads in a manifest: a title or a name. @@ -111,7 +112,8 @@ fn offered(tc: &TestCase, x: &mut Value) { /// operations (`op0`...), each at a place of its own on the command line or not and in drawn /// menus; analyzers of drawn fields and versions, and the test plugin's `size` (`bytes`) and /// `order` (`a`, `b`) it can run; export formats writing a file or a folder; providers of either -/// kind (granted reading tracks); and settings of every kind. +/// kind (granted reading tracks); library sources (granted reading folders); and settings of +/// every kind. #[hegel::test_helper] fn manifest(tc: &TestCase, id: &str) -> Value { let version = tc.draw(gs::sampled_from(&["0.1.0", "1.2.3-beta.1", "10.0.0"][..])); @@ -191,6 +193,12 @@ fn manifest(tc: &TestCase, id: &str) -> Value { if !providers.is_empty() && !capabilities.iter().any(|c| c == "library:read") { capabilities.push("library:read".into()); } + let sources: Vec = (0..tc.draw(gs::integers::().max_value(2))) + .map(|j| json!({ "name": format!("src{j}"), "title": title(tc) })) + .collect(); + if !sources.is_empty() && !capabilities.iter().any(|c| c == "fs:read") { + capabilities.push("fs:read".into()); + } let settings: Vec = (0..tc.draw(gs::integers::().max_value(4))).map(|j| setting(tc, &format!("k{j}"))).collect(); json!({ @@ -205,6 +213,7 @@ fn manifest(tc: &TestCase, id: &str) -> Value { "analyzers": analyzers, "exporters": exporters, "providers": providers, + "sources": sources, "settings": settings, }, }) @@ -265,6 +274,7 @@ fn listed_text(listed: &[Value]) -> String { ("analyzer", "analyzers"), ("export format", "exporters"), ("provider", "providers"), + ("library source", "sources"), ("setting", "settings"), ]; let adds: Vec = kinds @@ -344,6 +354,10 @@ fn described_text(d: &Value) -> String { }) .collect(); section("Providers:".into(), providers); + let sources = + list("sources").iter().map(|x| (format!("{id}:{}", str_of(&x["name"])), str_of(&x["title"]))).collect(); + let build = "locallibrary collection export DRIVE --from PLUGIN:SOURCE --grant FOLDER"; + section(format!("Library sources ({build}):"), sources); let settings = list("settings") .iter() .map(|s| { @@ -390,6 +404,7 @@ fn plugins_are_listed_and_described_in_text_as_in_json(tc: TestCase) { ("analyzers", "name"), ("exporters", "format"), ("providers", "name"), + ("sources", "name"), ("settings", "key"), ] { assert_eq!(p[key].as_array().unwrap(), &names(key, field), "{key}"); diff --git a/apps/locallibrary/tests/it/plugins/sources.rs b/apps/locallibrary/tests/it/plugins/sources.rs new file mode 100644 index 00000000..13f8c96b --- /dev/null +++ b/apps/locallibrary/tests/it/plugins/sources.rs @@ -0,0 +1,203 @@ +//! Library sources on the command line (`collection export --from PLUGIN:SOURCE --grant FOLDER`) +//! and in the sidecar (`plugins.source.plan` / `.apply`), with the test plugin reading the files +//! in the folders it's granted. + +use super::*; + +/// The echo manifest granted `fs:read`, with the library source `folder` (its `per_page` option). +fn sourcing() -> String { + let granted = echo_plugin().replace(r#""library:plan"]"#, r#""library:plan", "fs:read"]"#); + let options = "{ type = \"object\", properties = { per_page = { type = \"integer\" } } }"; + format!("{granted}\n[[contributes.sources]]\nname = \"folder\"\ntitle = \"Audio files\"\noptions = {options}\n") +} + +/// An [`Env`] with [`sourcing`]'s plugin and a folder of music: two WAV files (one titled, in a +/// playlist) and a text file. +fn music() -> (Env, std::path::PathBuf) { + let mut env = Env::new(); + env.plugins(&[&sourcing()]); + let music = env.outside("music"); + std::fs::create_dir_all(&music).unwrap(); + for name in ["a.wav", "b.wav"] { + std::fs::write(music.join(name), recording(1, 1.0, 44_100, 16, 0)).unwrap(); + } + std::fs::write(music.join("a.wav.json"), json!({ "title": "A", "artist": "Art" }).to_string()).unwrap(); + std::fs::write(music.join("notes.txt"), "not audio").unwrap(); + std::fs::write(music.join("playlists.json"), json!({ "Sets/Friday": ["a.wav"] }).to_string()).unwrap(); + (env, music) +} + +/// What a plan says the source read in `music`: the folder with its links resolved, as the host +/// gives it to the plugin (without Windows' `\\?\` prefix). +fn read_from(music: &std::path::Path) -> String { + let real = locallibrary::device::without_verbatim_prefix(music.canonicalize().unwrap()); + format!("read the files Audio files finds in {}", real.display()) +} + +#[test] +fn a_drive_is_built_from_a_plugins_library_source() { + let (env, music) = music(); + let stick = env.outside("stick"); + let build = |more: &[&str]| { + let mut args = vec!["collection", "export", path(&stick), "--from", "echo:folder", "--grant", path(&music)]; + args.extend(more); + env.run(&args) + }; + let dry = build(&["--dry-run", "--option", "per_page=1"]); + assert_eq!(dry.code, 0, "{}{}", dry.out, dry.err); + let said = format!("{}{}", dry.out, dry.err); + assert!(said.contains("the library source echo:folder -> "), "{said}"); + assert!(said.contains("2 track(s), 1 playlist(s)"), "{said}"); + assert!(said.contains("1 track(s) left out, e.g. ") && said.contains("notes.txt"), "{said}"); + assert!(said.contains(&read_from(&music)), "{said}"); + assert!(!stick.join("PIONEER").exists(), "a dry run writes nothing"); + + let done = build(&["--yes"]); + assert_eq!(done.code, 0, "{}{}", done.out, done.err); + assert!(done.out.contains("exported 2 track(s)"), "{}", done.out); + let tracks = env.json(&["tracks", path(&stick)]); + let mut titles: Vec<&str> = + tracks["tracks"].as_array().unwrap().iter().map(|t| t["title"].as_str().unwrap()).collect(); + titles.sort(); + assert_eq!(titles, ["A", "b"]); + + let refused = |args: &[&str]| { + let r = env.run(args); + assert_ne!(r.code, 0, "{args:?}"); + r.err + }; + let to = path(&stick); + let err = refused(&["collection", "export", to, "--from", "echo", "--grant", path(&music)]); + assert!(err.contains("--from takes PLUGIN:SOURCE, not \"echo\""), "{err}"); + let err = refused(&["collection", "export", to, "--from", "echo:nope", "--grant", path(&music)]); + assert!(err.contains("the echo plugin has no library source \"nope\""), "{err}"); + let err = refused(&["collection", "--db", "x.db", "export", to, "--from", "echo:folder", "--grant", path(&music)]); + assert!(err.contains("--db and --key are for rekordbox's collection"), "{err}"); + let err = + refused(&["collection", "export", to, "--from", "echo:folder", "--grant", path(&music), "--option", "x=1"]); + assert!(err.contains("this source has no option \"x\": its options are per_page"), "{err}"); + let err = refused(&["collection", "export", to, "--grant", path(&music)]); + assert!(err.contains("--from"), "--grant needs --from: {err}"); + let info = env.ok(&["plugin", "info", "echo"]); + assert!(info.contains("echo:folder") && info.contains("Library sources ("), "{info}"); +} + +#[test] +fn the_sidecar_builds_a_drive_from_a_library_source() { + let (env, music) = music(); + let stick = env.outside("stick"); + let mut s = Rpc::start(env.command()); + let req = json!({ "plugin": "echo", "source": "folder", "grants": [music], "drive": path(&stick) }); + let planned = s.call(1, "plugins.source.plan", req); + let planned = planned.get("result").cloned().unwrap_or_else(|| panic!("{planned}")); + assert_eq!((planned["operation"].as_str(), planned["tracks"].as_u64()), (Some("echo.source.folder"), Some(2))); + assert_eq!(planned["source"], "echo:folder"); + let wrong = s.call(2, "plugins.op.apply", json!({ "plan_id": planned["plan_id"] })); + assert_eq!(wrong["error"]["data"]["kind"], "invalid_params", "{wrong}"); + let applied = s.call(3, "plugins.source.apply", json!({ "plan_id": planned["plan_id"] })); + assert_eq!(applied["result"]["tracks"], 2, "{applied}"); + let listed = s.call(4, "plugins.list", json!({})); + let echo = listed["result"].as_array().unwrap().iter().find(|p| p["id"] == "echo").cloned().unwrap(); + assert_eq!(echo["sources"], json!(["folder"])); + assert!(echo["capabilities"].as_array().unwrap().iter().any(|c| c["capability"] == "fs:read"), "{echo}"); +} + +/// The titles of the tracks on the drive at `root`, sorted (none when there's no library). +fn titles(env: &Env, root: &std::path::Path) -> Vec { + let r = env.run(&["--json", "tracks", path(root)]); + let tracks: Value = serde_json::from_str(&r.out).unwrap_or_default(); + let mut titles: Vec = tracks["tracks"] + .as_array() + .into_iter() + .flatten() + .filter_map(|t| t["title"].as_str()) + .map(String::from) + .collect(); + titles.sort(); + titles +} + +/// For any `collection export --from` drawn (with at most one of these wrong: the source's name, +/// `--db` beside it, an option it doesn't have, no folder granted or one that isn't there; to a +/// new folder or a drive holding a library, replaced or not; a dry run, `--yes` or neither; in +/// text or `--json`): what's wrong, or a library not replaced, is refused, saying why, leaving +/// the drive as it was. +/// Otherwise a dry run or a run not confirmed writes nothing (the plan says what the source read), +/// and `--yes` builds the drive from the audio files granted. +#[hegel::test(test_cases = 16, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn collection_export_from_a_source_is_checked_then_written(tc: hegel::TestCase) { + use hegel::generators as gs; + let (env, music) = music(); + // At most one thing wrong, so most runs get as far as the plan. + let wrong = + tc.draw(gs::sampled_from(vec!["", "", "", "", "db", "echo", "echo:nope", "x=1", "no grant", "nowhere"])); + let from = if wrong.starts_with("echo") { wrong } else { "echo:folder" }; + let db = wrong == "db"; + let option = if wrong == "x=1" { Some("x=1") } else { tc.draw(gs::sampled_from(vec![None, Some("per_page=1")])) }; + let grant = match wrong { + "no grant" => None, + "nowhere" => Some("nowhere"), + _ => Some("music"), + }; + let on_demo = tc.draw(gs::booleans()); + let replace = tc.draw(gs::booleans()); + let mode = tc.draw(gs::sampled_from(vec!["--dry-run", "--yes", ""])); + let as_json = tc.draw(gs::integers::().max_value(2)) == 0; + let stick = if on_demo { env.demo("DEMO") } else { env.outside("stick") }; + let before = titles(&env, &stick); + let mut args: Vec = Vec::new(); + args.extend(as_json.then(|| "--json".to_string())); + args.push("collection".into()); + if db { + args.extend(["--db".into(), "x.db".into()]); + } + args.extend(["export".into(), path(&stick).into(), "--from".into(), from.into()]); + match grant { + Some("music") => args.extend(["--grant".into(), path(&music).into()]), + // Relative, as typed: made absolute against where it's run, where there's no such folder. + Some(other) => args.extend(["--grant".into(), other.into()]), + None => {} + } + if let Some(o) = option { + args.extend(["--option".into(), o.into()]); + } + args.extend(replace.then(|| "--replace".to_string())); + args.extend((!mode.is_empty()).then(|| mode.to_string())); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + let r = env.run(&args); + let refused = match () { + _ if db => Some("--db and --key are for rekordbox's collection"), + _ if from == "echo" => Some("--from takes PLUGIN:SOURCE"), + _ if from == "echo:nope" => Some("the echo plugin has no library source"), + _ if option == Some("x=1") => Some("this source has no option"), + _ if grant.is_none() => Some("grant the plugin a folder to read its library's files in"), + _ if grant == Some("nowhere") => Some("nowhere"), + _ if on_demo && !replace => Some("wasn't built from echo:folder; replace it"), + _ => None, + }; + if let Some(said) = refused { + // In `--json` mode the error is printed as JSON, on stdout. + let printed = if as_json { &r.out } else { &r.err }; + assert!(r.code != 0 && printed.contains(said), "{args:?} {said:?}: {}{}", r.out, r.err); + assert_eq!(titles(&env, &stick), before, "nothing written"); + return; + } + let wanted = match mode { + "--yes" => (0, vec!["A".to_string(), "b".to_string()]), + "--dry-run" => (0, before), + _ => (3, before), + }; + assert_eq!((r.code, titles(&env, &stick)), wanted, "{args:?}: {}{}", r.out, r.err); + if as_json { + let out: Value = serde_json::from_str(&r.out).unwrap_or_else(|e| panic!("{e}: {}", r.out)); + assert_eq!( + (out["plan"]["tracks"].as_u64(), out["applied"].as_bool()), + (Some(2), Some(mode == "--yes")), + "{out}" + ); + return; + } + let said = format!("{}{}", r.out, r.err); + assert!(said.contains("the library source echo:folder -> "), "{said}"); + assert!(said.contains(&read_from(&music)), "{said}"); +} diff --git a/crates/locallibrary/src/analysis_edit.rs b/crates/locallibrary/src/analysis_edit.rs index fbc35917..dd6cc1fe 100644 --- a/crates/locallibrary/src/analysis_edit.rs +++ b/crates/locallibrary/src/analysis_edit.rs @@ -24,7 +24,7 @@ use base64::prelude::*; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::anlz::{self, Bands, Beat, SongStructure}; +use crate::anlz::{self, Bands, Beat, CueFile, CuePoint, SongStructure}; use crate::commit::{self, Changes, NewFile, Options}; use crate::cue_edit::{clock, read_optional, sibling_key}; use crate::device::{Device, LineageOp}; @@ -256,6 +256,37 @@ fn rewrite(t: &Track, files: &Files, a: &NewAnalysis, bands: Option<&Bands>) -> Ok((changed, warnings)) } +/// New analysis files for `t`, a track without any, its audio at its `file_path` on the drive: +/// `.DAT` and `.EXT` holding `cues` and what `a` gives, its waveform `bands` (with no `.2EX`, the +/// 3-band ones are left out), each read back as given, and what to warn of. An error when they +/// can't be written so. +pub(crate) fn new_files( + t: &Track, + cues: &[CuePoint], + a: &NewAnalysis, + bands: Option<&Bands>, +) -> Result<([Vec; 2], Vec)> { + let (dat, ext) = anlz::new_analysis_files(&t.file_path)?; + let mut made = + [anlz::write_cue_points(&dat, CueFile::Dat, cues)?, anlz::write_cue_points(&ext, CueFile::Ext, cues)?]; + // Not on a drive yet: each file by its extension. + let keys = ["DAT", "EXT", "2EX"].map(String::from); + let files = Files { keys, bytes: [Some(made[0].clone()), Some(made[1].clone()), None] }; + let (changed, warnings) = rewrite(t, &files, a, bands)?; + for (key, new) in changed { + made[usize::from(key == "EXT")] = new; + } + let order = |c: &CuePoint| (c.hot_cue, c.start_ms, c.end_ms); + let mut read = anlz::read_cue_points(&made[0], Some(&made[1]))?; + let mut given = cues.to_vec(); + read.sort_by_key(order); + given.sort_by_key(order); + if read.len() != given.len() || read.iter().zip(&given).any(|(r, g)| !r.same_as(g)) { + return Err(Error::invalid(format!("{}'s cues don't read back as they were given", t.display_name()))); + } + Ok((made, warnings)) +} + /// The track of the audio file `file` (a device path) in `library`. fn track<'a>(library: &'a Library, file: &str) -> Result<&'a Track> { let key = paths::file_key(file); diff --git a/crates/locallibrary/src/api/collection.rs b/crates/locallibrary/src/api/collection.rs index 499c0322..ff7f8cdb 100644 --- a/crates/locallibrary/src/api/collection.rs +++ b/crates/locallibrary/src/api/collection.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use super::resolve_drive; use crate::collection_export::{self, CollectionExportOptions, CollectionExportPlan, CollectionExportReport}; +use crate::device::Device; use crate::masterdb::{self, CloudFolders, Collection, CollectionInfo, DriveDiff, TagCategory}; use crate::model::Library; use crate::op::{Operation, Planned, Probe, Touched, bytes}; @@ -102,10 +103,24 @@ pub fn collection_export_plan( ctx: &Ctx<'_>, ) -> Result> { let collection = open_collection(&req.collection)?; - let made = make_folder(&req.drive)?; - let planned = resolve_drive(&req.drive).and_then(|d| { - Ok((collection_export::plan(&collection, &CloudFolders::this_computer(), &d, &req.options, ctx)?, d)) - }); + let planned = export_planned("collection export", &req.drive, &req.options, |d| { + collection_export::plan(&collection, &CloudFolders::this_computer(), d, &req.options, ctx) + })?; + // rekordbox saving its collection in between makes the plan stale. + planned.touches(Touched::path(&collection.path, false, vec![Probe::Stamp(vec![String::new()])])).guard(None) +} + +/// An export (`operation`) to the drive or folder `drive` names, as `plan` plans it with `options`: +/// a folder that isn't there yet is made for it, and taken away again when planning fails. For the +/// caller to add what else it touches to, and guard. +pub(crate) fn export_planned( + operation: impl Into>, + drive: &str, + options: &CollectionExportOptions, + plan: impl FnOnce(&Device) -> Result, +) -> Result> { + let made = make_folder(drive)?; + let planned = resolve_drive(drive).and_then(|d| Ok((plan(&d)?, d))); let (mut plan, d) = match planned { Ok(p) => p, Err(e) => { @@ -117,7 +132,7 @@ pub fn collection_export_plan( }; plan.made_folder = made.is_some(); let what = match plan.playlists.as_slice() { - [] => "rekordbox's collection".to_string(), + [] => plan.what(), p => p.join(", "), }; let mut summary = vec![ @@ -139,16 +154,12 @@ pub fn collection_export_plan( } let destructive = !plan.deletes.is_empty() || plan.replaces_library; let warnings = plan.warnings.clone(); - // rekordbox saving its collection in between makes the plan stale. - let db = Touched::path(&collection.path, false, vec![Probe::Stamp(vec![String::new()])]); - Planned::new("collection export", plan, req.options.clone()) + Ok(Planned::new(operation, plan, options.clone()) .summary(summary) .warnings(warnings) .destructive(destructive) - .overwrite_archive(req.options.overwrite_archive) - .touches(Touched::library(&d, true)) - .touches(db) - .guard(None) + .overwrite_archive(options.overwrite_archive) + .touches(Touched::library(&d, true))) } /// A folder `spec` names that isn't there yet, made to export into; not the folders above it, as a diff --git a/crates/locallibrary/src/bin/test-plugin/main.rs b/crates/locallibrary/src/bin/test-plugin/main.rs index 1e2c4092..99a4e150 100644 --- a/crates/locallibrary/src/bin/test-plugin/main.rs +++ b/crates/locallibrary/src/bin/test-plugin/main.rs @@ -15,7 +15,8 @@ //! waveforms too, writing the files its `blobs` give first), and one named `order` answers fields //! `a` (null for a one-byte audio file) then `b`, or fails naming an empty audio file; and its //! exporter names the playlists it's given in a note, and would write to `outside`, or writes -//! `extra` too without saying so first. +//! `extra` too without saying so first. Any library source reads the files in the folders it's +//! granted, and what `.json` files beside them say (see `plugin::read_library`). //! `--manifest ` serves another manifest, `--protocol ` answers `hello` with //! another protocol and stops, `--silent` never answers, `--wrong-fields` answers fields of the //! wrong type, and `--without ` leaves out that feature's hooks, and the feature from diff --git a/crates/locallibrary/src/bin/test-plugin/plugin.rs b/crates/locallibrary/src/bin/test-plugin/plugin.rs index 07f96879..807da28d 100644 --- a/crates/locallibrary/src/bin/test-plugin/plugin.rs +++ b/crates/locallibrary/src/bin/test-plugin/plugin.rs @@ -15,7 +15,8 @@ use std::time::Duration; use loclib_plugin::{ Analysis, AnalyzeTrack, Candidate, Candidates, Choice, Context, Exported, ExportedFile, Fetch, FieldValue, Hooks, - Host, ListItem, OpAnswer, OpPlan, PlanOp, ProvideCandidates, RpcError, RunCommand, View, WriteExport, params, + Host, LibraryPage, ListItem, OpAnswer, OpPlan, PlanOp, ProvideCandidates, ReadLibrary, RpcError, RunCommand, + SourcePlaylist, SourceTrack, View, WriteExport, params, }; use serde_json::{Value, json}; @@ -63,6 +64,9 @@ impl Test { let test = test.clone(); hooks = hooks.on(move |req: PlanOp, host: &Host| test.plan_op(req, host)); } + if has("sources") { + hooks = hooks.on(|req: ReadLibrary, _: &Host| read_library(req)); + } if has("providers") { hooks = hooks.on(move |req: ProvideCandidates, host: &Host| test.provide(req, host)); } @@ -242,6 +246,68 @@ impl Test { } } +/// Any source: the files in its folders granted (and below), but `.json` files, each a track with +/// what the `.json` file named after it (`a.wav.json`) gives besides its path, once the files that +/// file's `blobs` give are in the scratch folder; and the playlists a folder's `playlists.json` +/// gives (`{"Sets/Friday": ["a.wav"]}`, files by their path in the folder). Its options: `also`, +/// paths to give as tracks as they are; `per_page`, how many tracks a page has (playlists come +/// on the last); `as_blob`, each page in a blob; `again`, every page says there's one after it. +fn read_library(req: ReadLibrary) -> Result { + let io = |e: std::io::Error| RpcError::failed("io", e.to_string()); + let (mut tracks, mut playlists) = (Vec::new(), Vec::new()); + for root in &req.roots { + let mut files = Vec::new(); + let mut folders = vec![root.clone()]; + while let Some(folder) = folders.pop() { + for entry in std::fs::read_dir(&folder).map_err(io)? { + let path = entry.map_err(io)?.path(); + match path.is_dir() { + true => folders.push(path), + false => files.push(path), + } + } + } + files.sort(); + for file in files.into_iter().filter(|f| f.extension().is_none_or(|e| e != "json")) { + let mut told: Value = match std::fs::read_to_string(format!("{}.json", file.display())) { + Ok(text) => serde_json::from_str(&text).map_err(|e| RpcError::failed("parse", e.to_string()))?, + Err(_) => json!({}), + }; + write_blobs(&req.context, &told)?; + told.as_object_mut().map(|o| o.remove("blobs")); + told["path"] = json!(file); + tracks.push(params::(told)?); + } + if let Ok(text) = std::fs::read_to_string(root.join("playlists.json")) { + let lists: indexmap::IndexMap> = + serde_json::from_str(&text).map_err(|e| RpcError::failed("parse", e.to_string()))?; + playlists.extend( + lists + .into_iter() + .map(|(path, names)| SourcePlaylist { path, tracks: names.iter().map(|n| root.join(n)).collect() }), + ); + } + } + let also = req.options["also"].as_array().into_iter().flatten().filter_map(Value::as_str); + tracks.extend(also.map(|p| SourceTrack { path: p.into(), ..SourceTrack::default() })); + let from: usize = req.cursor.as_deref().map_or(0, |c| c.parse().unwrap_or(0)); + let per_page = req.options["per_page"].as_u64().map_or(tracks.len().max(1), |n| n as usize); + let to = (from + per_page).min(tracks.len()); + let last = to >= tracks.len() && req.options["again"] != true; + let page = LibraryPage { + tracks: tracks.drain(from.min(to)..to).collect(), + playlists: if last { playlists } else { Vec::new() }, + blob: None, + next: (!last).then(|| to.to_string()), + }; + if req.options["as_blob"] != true { + return Ok(page); + } + let scratch = req.context.scratch.clone().ok_or_else(|| RpcError::failed("invalid", "no scratch folder"))?; + std::fs::write(scratch.join("page.json"), serde_json::to_vec(&page).expect("a page is JSON")).map_err(io)?; + Ok(LibraryPage { blob: Some("page.json".into()), next: page.next, ..LibraryPage::default() }) +} + /// The question `p["ask"]` asks, while `p` doesn't answer it (has no value of its param), or /// with `ask_again` whatever it has. fn asked(p: &Value) -> Option { diff --git a/crates/locallibrary/src/collection_export.rs b/crates/locallibrary/src/collection_export.rs index cbb71f8d..04c59e3c 100644 --- a/crates/locallibrary/src/collection_export.rs +++ b/crates/locallibrary/src/collection_export.rs @@ -1,7 +1,10 @@ //! Writing rekordbox's collection (`master.db` on this computer, see [`crate::masterdb`]) to a //! drive or a folder, as rekordbox's Export does, so a library can be had from this computer //! without a drive rekordbox exported to. A folder written this way is a drive like any other: -//! `pull` saves it in the store, and `library push` writes it to any drive. +//! `pull` saves it in the store, and `library push` writes it to any drive. A plugin's library +//! source is written the same way (a `Collected` library, filled by [`crate::plugin`]'s host): its +//! audio files hashed before they're copied, and checked as they are, and analysis files made for +//! tracks that come with cues, beat grids or waveforms. //! //! The export holds every track (or those of the chosen playlists), the playlists and folders //! (intelligent playlists, whose rules aren't read, are left out), cues and My Tags. Audio goes in @@ -26,15 +29,19 @@ //! recorded, changes since the last export) is saved in the store first, as [`crate::pack`] does. use indexmap::IndexMap; -use rustc_hash::FxHashSet; -use std::path::PathBuf; +use rustc_hash::{FxHashMap, FxHashSet}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; use chrono::{DateTime, Utc}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use crate::analysis_edit::{self, NewAnalysis}; +use crate::anlz::{Bands, Beat, CuePoint}; use crate::artwork::{image_key, medium_variant}; use crate::device::{Device, LineageEntry, LineageOp}; +use crate::error::IoContext; use crate::error::{Action, Reason}; use crate::hash::Hash; use crate::library_write::{self, FileCopy, Planned, Source, Written}; @@ -46,7 +53,7 @@ use crate::progress::{Ctx, Event, Phase}; use crate::store::{SnapshotInfo, Store}; use crate::sync::PullOptions; use crate::tags::DbTags; -use crate::{Error, Result, backup, convert, edit, fsutil, merge}; +use crate::{Error, Result, backup, convert, edit, fsutil, hash, merge}; const RECORD_FILE: &str = "collection.json"; @@ -75,8 +82,11 @@ pub struct CollectionExportOptions { /// What a target was exported from, kept on the target. #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct CollectionExportRecord { - /// The `master.db` it was exported from. + /// The `master.db` it was exported from (empty for a plugin's library source). pub collection: PathBuf, + /// The plugin's library source it was built from, `:`, instead. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, /// The playlists chosen; empty for the whole collection. pub playlists: Vec, pub at: DateTime, @@ -98,8 +108,11 @@ pub fn forget(device: &Device) -> Result<()> { #[derive(Debug, Clone, Serialize, JsonSchema)] pub struct CollectionExportPlan { - /// The `master.db` exported. + /// The `master.db` exported (empty for a plugin's library source). pub collection: PathBuf, + /// The plugin's library source written instead, `:`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, pub target: String, pub target_root: PathBuf, /// The playlists chosen; empty for the whole collection. @@ -124,7 +137,8 @@ pub struct CollectionExportPlan { /// Tracks whose analysis or artwork Cloud Library Sync hasn't finished bringing to this /// computer. pub syncing: Vec, - /// Tracks with an analysis file that can't be read: exported without it. + /// Tracks with an analysis file that can't be read, or (from a plugin's source) whose cues, + /// beat grid or waveform can't be written: exported without it. pub unreadable: Vec, /// The target holds a library that wasn't exported from the collection, or files the export /// removes though it holds no library. @@ -150,12 +164,91 @@ pub struct CollectionExportPlan { artwork: IndexMap, #[serde(skip)] written: Planned, + /// The analysis files made for the export, until they're copied. + #[serde(skip)] + _made: Option>, } impl CollectionExportPlan { pub fn is_noop(&self) -> bool { false // the databases are always rebuilt } + + /// What's exported, as messages name it: `rekordbox's collection`, or the source. + pub fn what(&self) -> String { + match &self.source { + None => "rekordbox's collection".into(), + Some(source) => format!("the library source {source}"), + } + } +} + +/// A library a drive is built from, whichever program it's read from: rekordbox's collection +/// ([`Collected::rekordbox`]), or a plugin's library source ([`Collected::checked`]). Its tracks' +/// paths are their audio files' on this computer. +pub(crate) struct Collected<'a> { + /// The plugin's library source it's from (`:`), if it's from one. + source: Option, + /// Tracks, playlists and folders. + library: Library, + /// My Tags: categories in order, each with its tags and their tracks. + tags: Vec, + /// Ids of intelligent playlists, whose rules aren't read: left out. + smart_playlists: Vec, + files: Files<'a>, +} + +/// Where a [`Collected`] library's tracks' files are. +enum Files<'a> { + /// rekordbox's: audio where its paths say (or in Cloud Library Sync's folders), analysis and + /// artwork in its `share` folder. + Rekordbox { collection: &'a Collection, cloud: &'a CloudFolders }, + /// Audio files the plugin host checked, by track id. + Checked(FxHashMap), +} + +/// A track's audio file as the plugin host checked it (in a folder the user granted, a file +/// players play), and what its analysis files are made with. +pub(crate) struct Checked { + /// Its size and modification time then: the export copies it as it was, hashed before it's + /// copied and checked as it is. + pub(crate) size: u64, + pub(crate) mtime: i64, + pub(crate) cues: Vec, + pub(crate) beats: Option>, + /// At 150 columns a second. + pub(crate) waveform: Option, +} + +impl Checked { + /// Whether it makes analysis files for its track. + fn analysed(&self) -> bool { + !self.cues.is_empty() || self.beats.is_some() || self.waveform.is_some() + } +} + +impl<'a> Collected<'a> { + /// rekordbox's collection, its tracks' files found through `cloud`. + pub(crate) fn rekordbox(collection: &'a Collection, cloud: &'a CloudFolders) -> Collected<'a> { + Collected { + source: None, + library: collection.library.clone(), + tags: collection.tags.clone(), + smart_playlists: collection.smart_playlists.clone(), + files: Files::Rekordbox { collection, cloud }, + } + } + + /// A plugin's library source (`:`): `library`'s tracks each at its audio file + /// (absolute) as `checked` has it, by id. + pub(crate) fn checked( + source: String, + library: Library, + tags: Vec, + checked: FxHashMap, + ) -> Collected<'static> { + Collected { source: Some(source), library, tags, smart_playlists: Vec::new(), files: Files::Checked(checked) } + } } #[derive(Debug, Clone, Default, Serialize, JsonSchema)] @@ -186,7 +279,19 @@ pub fn plan( opts: &CollectionExportOptions, ctx: &Ctx<'_>, ) -> Result { - let previous = record(target)?; + plan_from(&Collected::rekordbox(collection, cloud), target, opts, ctx) +} + +/// Plan writing `from` to `target`. An export from somewhere else than the target's last one +/// counts as none: the selection isn't carried over, and a library on the target is only replaced +/// when asked. +pub(crate) fn plan_from( + from: &Collected<'_>, + target: &Device, + opts: &CollectionExportOptions, + ctx: &Ctx<'_>, +) -> Result { + let previous = record(target)?.filter(|r| r.source == from.source); // An export cut off before it recorded itself counts as one. let interrupted = target.state().and_then(|s| s.pending.as_ref()).is_some_and(|p| p.op == LineageOp::Export); let exported = previous.is_some() || interrupted; @@ -196,7 +301,7 @@ pub fn plan( (p, _) => p.to_vec(), }; let mut warnings = Vec::new(); - let chosen = choose(collection, &playlists, &mut warnings)?; + let chosen = choose(from, &playlists, &mut warnings)?; // Each track's analysis files are read to write its path into them, which takes a while. let mut layout = Layout { artwork: previous.map(|r| r.artwork).unwrap_or_default(), ..Default::default() }; layout.used_keys = layout.artwork.values().map(|k| k.to_lowercase()).collect(); @@ -204,7 +309,7 @@ pub fn plan( for t in &chosen.tracks { ctx.check()?; ctx.emit(Event::Item { phase: Phase::Scan, path: t.file_path.clone(), bytes: 0 }); - layout.add(collection, cloud, t)?; + layout.add(from, t, target)?; } ctx.phase_end(Phase::Scan); let mut library = chosen; @@ -228,12 +333,18 @@ pub fn plan( )); } if let Some(first) = layout.unreadable.first() { - warnings.push(format!( - "{} track(s) have analysis files that can't be read (e.g. {first}); they're exported without them", - layout.unreadable.len() - )); + let what = match from.files { + Files::Rekordbox { .. } => "have analysis files that can't be read", + Files::Checked(_) => "have cues, beat grids or waveforms that can't be written", + }; + warnings + .push(format!("{} track(s) {what} (e.g. {first}); they're exported without them", layout.unreadable.len())); + } + if let Some(first) = layout.analysis_warnings.first() { + let more = layout.analysis_warnings.len() - 1; + warnings.push(if more > 0 { format!("{first} (and {more} more like it)") } else { first.clone() }); } - let tags = collection.tags.clone(); + let tags = from.tags.clone(); let target_lib = target.library().ok(); let scanned = target.scan_fingerprinted(target_lib.as_ref(), ctx)?; @@ -242,15 +353,17 @@ pub fn plan( let entries: usize = library.playlists.iter().map(|p| p.track_ids.len()).sum(); let database = 256 * 1024 + (library.tracks.len() + entries) as u64 * 1024; let databases = Written { pdb: Some(database), ext: None, plus: Some(database) }; - let written = library_write::plan(target, &scanned.tree, &layout.files, &[], databases, false); + let mut written = library_write::plan(target, &scanned.tree, &layout.files, &[], databases, false); + hash_checked(&mut written, &layout.checked, ctx)?; let downloads: Vec<&FileCopy> = written.copies.iter().filter(|c| layout.online_only.contains(&c.path)).collect(); // Without a record of an earlier export, whatever the target holds (a library, or loose audio // the export would remove) is only replaced when asked. let replaces_library = !exported && (target.has_export() || !written.deletes.is_empty()); if replaces_library && !opts.replace { - let what = match target.has_export() { - true => "a library that wasn't exported from rekordbox's collection", - false => "files the export would remove", + let what = match (target.has_export(), &from.source) { + (true, None) => "a library that wasn't exported from rekordbox's collection".to_string(), + (true, Some(source)) => format!("a library that wasn't built from {source}"), + (false, _) => "files the export would remove".to_string(), }; return Err(Error::conflict( Reason::Exists, @@ -272,7 +385,11 @@ pub fn plan( u.describe() )); } - warnings.extend(edit::journal::forget_warning(target, None, "exporting the collection")); + let exporting = match &from.source { + None => "exporting the collection".to_string(), + Some(source) => format!("writing {source}"), + }; + warnings.extend(edit::journal::forget_warning(target, None, &exporting)); if !written.fits { warnings.push(format!( "the export needs about {} bytes (its files, databases and backup) but only {} are free on {}", @@ -281,8 +398,13 @@ pub fn plan( target.name() )); } + let collection = match &from.files { + Files::Rekordbox { collection, .. } => collection.path.clone(), + Files::Checked(_) => PathBuf::new(), + }; Ok(CollectionExportPlan { - collection: collection.path.clone(), + collection, + source: from.source.clone(), target: target.name(), target_root: target.root().to_path_buf(), playlists, @@ -310,20 +432,47 @@ pub fn plan( tags, artwork: layout.artwork, written, + _made: layout.made.map(Arc::new), }) } -/// The collection's tracks and playlists to export: those of `playlists` (with the folders above +/// Hashes each audio file `written` copies that the plugin host checked (by key, `checked`), as +/// it is now, for the copy to be checked against: one whose size changed since is refused. +fn hash_checked(written: &mut Planned, checked: &FxHashSet, ctx: &Ctx<'_>) -> Result<()> { + let hashed: Vec<&FileCopy> = written.copies.iter().filter(|c| checked.contains(&c.path)).collect(); + if hashed.is_empty() { + return Ok(()); + } + ctx.phase_start(Phase::Hash, hashed.len() as u64, hashed.iter().map(|c| c.size).sum()); + let mut hashes = Vec::new(); + for c in hashed { + let src = &written.sources[&c.path]; + ctx.emit(Event::Item { phase: Phase::Hash, path: c.path.clone(), bytes: 0 }); + let now = std::fs::metadata(&src.path).at(&src.path)?.len(); + if now != src.size { + let changed = format!("{} changed while the export was planned; plan it again", src.path.display()); + return Err(Error::invalid(changed)); + } + hashes.push((c.path.clone(), hash::hash_file(&src.path, ctx, Phase::Hash)?)); + } + ctx.phase_end(Phase::Hash); + for (key, h) in hashes { + written.sources.get_mut(&key).expect("one of its copies").hash = Some(h); + } + Ok(()) +} + +/// The library's tracks and playlists to export: those of `playlists` (with the folders above /// them), or everything. Intelligent playlists and history lists are left out. -fn choose(collection: &Collection, playlists: &[String], warnings: &mut Vec) -> Result { - let lib = &collection.library; - let smart: FxHashSet = collection.smart_playlists.iter().copied().collect(); +fn choose(from: &Collected<'_>, playlists: &[String], warnings: &mut Vec) -> Result { + let lib = &from.library; + let smart: FxHashSet = from.smart_playlists.iter().copied().collect(); let keep = match playlists { [] => lib.playlists.iter().map(|p| p.id).collect(), specs => edit::chosen_nodes(lib, specs)?, }; let left_out: Vec = - collection.smart_playlists.iter().filter(|id| keep.contains(id)).map(|id| lib.playlist_path(*id)).collect(); + from.smart_playlists.iter().filter(|id| keep.contains(id)).map(|id| lib.playlist_path(*id)).collect(); if let Some(first) = left_out.first() { warnings.push(format!( "{} intelligent playlist(s) aren't exported (e.g. {first}): their rules aren't read", @@ -360,15 +509,28 @@ struct Layout { missing: Vec, syncing: Vec, unreadable: Vec, + /// What making analysis files warned of, a line each. + analysis_warnings: Vec, + /// Keys of the audio files the plugin host checked. + checked: FxHashSet, + /// The analysis files made, until they're copied. + made: Option, } impl Layout { - fn add(&mut self, collection: &Collection, cloud: &CloudFolders, t: &Track) -> Result<()> { - let audio = collection.audio_file(t, cloud).and_then(|p| std::fs::metadata(&p).ok().map(|m| (p, m))); - let Some((audio, meta)) = audio.filter(|(_, m)| m.is_file()) else { - self.missing.push(t.display_name()); - return Ok(()); - }; + fn add(&mut self, from: &Collected<'_>, t: &Track, target: &Device) -> Result<()> { + match &from.files { + Files::Rekordbox { collection, cloud } => self.add_rekordbox(collection, cloud, t), + Files::Checked(checked) => { + let c = checked.get(&t.id).ok_or_else(|| Error::invalid(format!("{} wasn't checked", t.file_path)))?; + self.add_checked(t, c, target) + } + } + } + + /// The audio file `audio` of `t` placed in `Contents///`, copied from + /// `source`: its key on the target. + fn place(&mut self, t: &Track, audio: &Path, source: Source) -> String { let folder = |s: &str, unknown: &str| match s.trim() { "" => unknown.to_string(), s => paths::sanitize_component(s), @@ -381,24 +543,42 @@ impl Layout { paths::sanitize_component(&name) ); let key = merge::unique_key(&wanted, &mut self.taken, &|_| false); + self.files.insert(key.clone(), source); + key + } + + /// A folder no other track's analysis files have, for those of the track at `device_path`. + fn analysis_dir(&self, device_path: &str) -> String { + let mut salt = 0; + let mut dir = merge::analysis_dir(device_path, salt); + while self.taken.contains(&format!("{dir}/anlz0000.dat").to_lowercase()) { + salt += 1; + dir = merge::analysis_dir(device_path, salt); + } + dir + } + + /// `t` as it goes on the target, its audio at `device_path`: no analysis or artwork yet. + fn placed(t: &Track, device_path: &str) -> Track { + Track { file_path: device_path.into(), analyze_path: String::new(), artwork_path: None, ..t.clone() } + } + + fn add_rekordbox(&mut self, collection: &Collection, cloud: &CloudFolders, t: &Track) -> Result<()> { + let audio = collection.audio_file(t, cloud).and_then(|p| std::fs::metadata(&p).ok().map(|m| (p, m))); + let Some((audio, meta)) = audio.filter(|(_, m)| m.is_file()) else { + self.missing.push(t.display_name()); + return Ok(()); + }; + let key = self.place(t, &audio, Source::file(audio.clone(), meta.len(), mtime(&meta))); let device_path = format!("/{key}"); if fsutil::is_online_only(&meta) { - self.online_only.insert(key.clone()); + self.online_only.insert(key); } - self.files.insert(key, Source::file(audio, meta.len(), mtime(&meta))); - let mut track = t.clone(); - track.file_path = device_path.clone(); - track.analyze_path = String::new(); - track.artwork_path = None; + let mut track = Layout::placed(t, &device_path); let (mut syncing, mut unreadable) = (false, false); if let Some((stem, _)) = t.analyze_path.rsplit_once('.').filter(|_| !t.analyze_path.is_empty()) { - let mut salt = 0; - let mut dir = merge::analysis_dir(&device_path, salt); - while self.taken.contains(&format!("{dir}/anlz0000.dat").to_lowercase()) { - salt += 1; - dir = merge::analysis_dir(&device_path, salt); - } + let dir = self.analysis_dir(&device_path); for ext in ANALYSIS_EXTENSIONS { let share = format!("{stem}.{ext}"); syncing |= collection.still_syncing(&share); @@ -429,6 +609,56 @@ impl Layout { Ok(()) } + /// A plugin's track, its audio file checked as `c` says (`t.file_path`), with analysis files + /// made for it when it brings cues, a beat grid or a waveform. + fn add_checked(&mut self, t: &Track, c: &Checked, target: &Device) -> Result<()> { + let audio = PathBuf::from(&t.file_path); + let key = self.place(t, &audio, Source::file(audio.clone(), c.size, c.mtime)); + let device_path = format!("/{key}"); + self.checked.insert(key); + let mut track = Layout::placed(t, &device_path); + if c.analysed() { + let a = NewAnalysis { beats: c.beats.clone(), ..NewAnalysis::default() }; + match analysis_edit::new_files(&track, &c.cues, &a, c.waveform.as_ref()) { + Ok((made, warnings)) => { + let dir = self.analysis_dir(&device_path); + for (ext, bytes) in ["DAT", "EXT"].into_iter().zip(made) { + let to = format!("{dir}/ANLZ0000.{ext}"); + let source = self.made(&to, &bytes, target)?; + self.taken.insert(to.to_lowercase()); + self.files.insert(to, source); + } + track.analyze_path = format!("/{dir}/ANLZ0000.DAT"); + self.analysis_warnings.extend(warnings); + } + Err(e) => self.unreadable.push(format!("{}: {e}", t.display_name())), + } + } + self.tracks.push(track); + Ok(()) + } + + /// An analysis file made for the export (`bytes`, to go at `key`), kept in a temporary folder + /// until it's copied: as current as the target's when that holds the same, else copied. + fn made(&mut self, key: &str, bytes: &[u8], target: &Device) -> Result { + let dir = match &self.made { + Some(dir) => dir.path().to_path_buf(), + None => { + let made = tempfile::tempdir().map_err(|e| Error::Io { path: std::env::temp_dir(), source: e })?; + self.made.insert(made).path().to_path_buf() + } + }; + let file = dir.join(format!("{}.{}", self.files.len(), key.rsplit_once('.').map_or("DAT", |(_, e)| e))); + std::fs::write(&file, bytes).at(&file)?; + let there = target.checked_path(key)?; + let same = std::fs::read(&there).is_ok_and(|b| b == bytes); + let at = match std::fs::metadata(&there) { + Ok(m) if same => mtime(&m), + _ => Utc::now().timestamp(), + }; + Ok(Source { force: !same, ..Source::file(file, bytes.len() as u64, at) }) + } + /// The key of the image at `image` in the `share` folder on the target (with its `_m` copy, /// when rekordbox made one), or `None` if it isn't there. fn artwork(&mut self, collection: &Collection, image: &str) -> Option { @@ -497,17 +727,17 @@ pub fn apply( })?; let opts = PullOptions { library: None, - message: Some(format!("{} before exporting rekordbox's collection to it: {}", plan.target, u.describe())), + message: Some(format!("{} before exporting {} to it: {}", plan.target, plan.what(), u.describe())), force: true, }; report.saved = crate::sync::pull(store, target, &opts, ctx)?.snapshot; } let _lock = crate::commit::lock(target, None)?; if target.has_export() { - let reason = "before exporting rekordbox's collection".to_string(); + let reason = format!("before exporting {}", plan.what()); report.backup = Some(backup::create(target, &reason, &plan.written.backed_up, None)?.id); } - target.begin(LineageOp::Export, "export rekordbox's collection", report.backup.clone())?; + target.begin(LineageOp::Export, format!("export {}", plan.what()), report.backup.clone())?; let identity = crate::player::stash(target, true)?; ctx.phase_start(Phase::Copy, plan.written.copies.len() as u64, plan.written.copy_bytes); (report.copied, report.copied_bytes) = library_write::copy(target, &plan.written, ctx)?; @@ -539,6 +769,7 @@ pub fn apply( }))?; let rec = CollectionExportRecord { collection: plan.collection.clone(), + source: plan.source.clone(), playlists: plan.playlists.clone(), at: Utc::now(), artwork: plan.artwork.clone(), diff --git a/crates/locallibrary/src/library_write.rs b/crates/locallibrary/src/library_write.rs index 0695d523..52d69c8a 100644 --- a/crates/locallibrary/src/library_write.rs +++ b/crates/locallibrary/src/library_write.rs @@ -20,6 +20,7 @@ use crate::anlz; use crate::commit::Swap; use crate::device::Device; use crate::error::IoContext; +use crate::hash::Hash; use crate::paths::{self, FileClass, fold}; use crate::progress::{Ctx, Event, Phase}; use crate::tree::FileTree; @@ -47,19 +48,22 @@ pub(crate) struct Source { /// Copied even when the drive's file looks current (same size and time): what's written into /// it changed in a way neither shows. pub(crate) force: bool, + /// What it holds, for a file copied as it is: the copy is refused when it doesn't (the file + /// changed since it was checked). + pub(crate) hash: Option, } impl Source { /// A file copied as it is. pub(crate) fn file(path: PathBuf, size: u64, mtime: i64) -> Source { - Source { path, size, mtime, audio_path: None, shift_ms: 0, force: false } + Source { path, size, mtime, audio_path: None, shift_ms: 0, force: false, hash: None } } /// An analysis file copied with `audio_path` written into it and its times `shift_ms` later: /// its size on the drive is worked out from its contents. pub(crate) fn analysis(path: PathBuf, mtime: i64, audio_path: &str, shift_ms: i32) -> Result { let size = analysis_bytes(&path, audio_path, shift_ms)?.len() as u64; - Ok(Source { path, size, mtime, audio_path: Some(audio_path.to_string()), shift_ms, force: false }) + Ok(Source { path, size, mtime, audio_path: Some(audio_path.to_string()), shift_ms, force: false, hash: None }) } } @@ -251,7 +255,7 @@ pub(crate) fn copy(target: &Device, planned: &Planned, ctx: &Ctx<'_>) -> Result< match &src.audio_path { Some(audio) => write_unsynced(&dst, &analysis_bytes(&src.path, audio, src.shift_ms)?)?, None => { - fsutil::copy_file(&src.path, &dst, None, None, false, ctx)?; + fsutil::copy_file(&src.path, &dst, None, src.hash, false, ctx)?; } } Ok((dst, src.mtime)) diff --git a/crates/locallibrary/src/plugin.rs b/crates/locallibrary/src/plugin.rs index 5d66691d..800b378a 100644 --- a/crates/locallibrary/src/plugin.rs +++ b/crates/locallibrary/src/plugin.rs @@ -1,6 +1,6 @@ //! The plugin host: plugins are programs locallibrary runs and talks to over their stdin and //! stdout ([`loclib_plugin`]'s protocol), adding commands, operations, track analyzers, exporters, -//! providers and settings. The CLI, the TUI and the RPC sidecar host them through [`Plugins`]. +//! providers, library sources and settings. The CLI, the TUI and the RPC sidecar host them through [`Plugins`]. //! //! Only plugins locallibrary ships run for now ([`PluginSpec::first_party`]): the app hands the //! host each one's manifest, compiled in, and how to start it (the locallibrary binary itself, @@ -18,7 +18,9 @@ //! files in a folder the host makes for each request (`scratch`), which it reads itself. //! Analyzers' values go to the [`crate::fields`] store, their beat grids, phrases and waveforms //! to its overlay store. What providers find for tracks, the host -//! offers the user and makes the edits of itself (see [`Plugins::provide`]). +//! offers the user and makes the edits of itself (see [`Plugins::provide`]). A library source's +//! tracks are copied to a drive only from the folders the user grants it, each file checked by the +//! host (see [`Plugins::plan_source_export`]). mod analysis; mod analyzers; @@ -33,6 +35,7 @@ mod process; mod providers; mod secrets; mod settings; +mod sources; use std::ffi::OsString; use std::path::PathBuf; @@ -61,6 +64,7 @@ pub use providers::{ AUTO_PICK, Found, Offer, Pick, PlanProvided, Provide, Provided, auto_pick, plan_provided, set_auto_pick, }; pub(crate) use settings::{host_settings, set_host_setting}; +pub use sources::SourceExport; use process::Slot; pub(crate) use process::THREAD_ELEVATED; diff --git a/crates/locallibrary/src/plugin/sources.rs b/crates/locallibrary/src/plugin/sources.rs new file mode 100644 index 00000000..9a82e3a1 --- /dev/null +++ b/crates/locallibrary/src/plugin/sources.rs @@ -0,0 +1,418 @@ +//! Plugins' library sources (`library/read`, the `sources` feature): a library a plugin reads +//! from somewhere on this computer, written to a drive as rekordbox's collection is (see +//! [`crate::collection_export`]). The plugin is given the folders the user grants it for the read +//! (`fs:read`) and answers pages of tracks and playlists until one has no `next`. None of its +//! paths is taken on its word: a track's audio file must be, its links resolved, a file in one +//! of those folders (not on the drive written to) that players play, which the host opens to read +//! its format and length itself; it's hashed before it's copied, and the copy checked against +//! that. A track that isn't is left out with a warning, and so is a playlist's entry for one. Cues, +//! beat grids and waveforms a track brings go into analysis files made for it, each read back +//! first; what can't be is left out with a warning. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use indexmap::IndexMap; +use loclib_model::audio::Format; +use loclib_plugin::{Context, LibraryPage, ReadLibrary, SourceCue, SourcePlaylist, SourceTrack, method}; +use rustc_hash::{FxHashMap, FxHashSet}; +use schemars::JsonSchema; +use serde::Deserialize; +use serde_json::Value; + +use super::Plugins; +use super::blobs::Scratch; +use crate::analysis_edit::NewAnalysis; +use crate::anlz::{self, Bands, CuePoint, DEFAULT_HOT_CUE_RGB}; +use crate::collection_export::{self, Checked, Collected, CollectionExportOptions, CollectionExportPlan}; +use crate::device::without_verbatim_prefix; +use crate::harmony::Camelot; +use crate::masterdb::{TagCategory, TagEntry}; +use crate::model::{Library, Playlist, Track}; +use crate::op::Planned; +use crate::progress::{Ctx, Phase}; +use crate::store::Store; +use crate::{Error, Result, api, cue_edit, probe}; + +/// Building a drive from a plugin's library source. +#[derive(Debug, Clone, Deserialize, JsonSchema)] +pub struct SourceExport { + pub plugin: String, + /// One of its library sources, by name. + pub source: String, + /// As the source's `options` schema says. + #[serde(default)] + pub options: Value, + /// The folders the plugin may read files in, absolute: granted for this read alone, and only + /// audio files in them are copied. + pub grants: Vec, + /// The drive or folder to write to. + pub drive: String, + #[serde(default, flatten)] + pub export: CollectionExportOptions, +} + +/// The most examples a warning about what was left out gives. +const EXAMPLES: usize = 3; + +/// What was left out of what a source read, by why: how many, and the first few. +#[derive(Default)] +struct LeftOut(IndexMap<&'static str, (usize, Vec)>); + +impl LeftOut { + fn add(&mut self, what: &'static str, example: String) { + let (n, examples) = self.0.entry(what).or_default(); + *n += 1; + if examples.len() < EXAMPLES { + examples.push(example); + } + } + + /// A warning for each kind, in the order they first came. + fn warnings(self) -> Vec { + let line = |(what, (n, examples)): (&str, (usize, Vec))| { + format!("{n} {what} left out, e.g. {}", examples.join("; ")) + }; + self.0.into_iter().map(line).collect() + } +} + +/// A track as a source answered it, with its waveform's bands read from its request's scratch +/// folder (or why they couldn't be). +struct Answered { + track: SourceTrack, + bands: Option>, +} + +impl Plugins { + /// Plans writing a plugin's library source to the drive or folder `req.drive` names, as + /// rekordbox's collection is written (`collection export`): the plugin reads it from + /// `req.grants`, a page at a time, and the plan (operation `.source.`) says + /// which folders it was granted, and what was left out. + pub fn plan_source_export( + &self, + req: &SourceExport, + store: Option<&Store>, + ctx: &Ctx<'_>, + ) -> Result> { + let slot = self.slot(&req.plugin)?; + let declared = slot.declared("library source", &req.source, &slot.manifest.contributes.sources, |s| &s.name)?; + let roots = granted(&req.grants)?; + let (context, _) = slot.context(Context::default(), store)?; + let options = super::fitting(declared.options.as_ref(), req.options.clone(), &context)?; + let source = format!("{}:{}", req.plugin, req.source); + let mut read = ReadLibrary { source: req.source.clone(), options, roots: roots.clone(), cursor: None, context }; + let operation = format!("{}.source.{}", req.plugin, req.source); + let mut planned = api::export_planned(operation, &req.drive, &req.export, |d| { + let drive = without_verbatim_prefix(d.root().canonicalize().unwrap_or_else(|_| d.root().to_path_buf())); + if let Some(root) = roots.iter().find(|r| r.starts_with(&drive)) { + return Err(Error::invalid(format!( + "{} is on {}, the drive written to: grant a folder elsewhere", + root.display(), + d.name() + ))); + } + let mut answered = Vec::new(); + let mut playlists = Vec::new(); + let mut cursors = FxHashSet::default(); + loop { + ctx.check()?; + let page = slot.call_with(method::READ_LIBRARY, &read, store, ctx, &|_| {}, |page, scratch, _| { + slot.page(page, scratch) + })?; + answered.extend(page.0); + playlists.extend(page.1); + read.cursor = match page.2 { + None => break, + Some(next) if !cursors.insert(next.clone()) => { + return Err(slot.misbehaved(format!("it answered the page after {next:?} twice"))); + } + next => next, + }; + } + let (collected, left_out) = collect(&source, answered, playlists, &roots, &drive, ctx)?; + let mut plan = collection_export::plan_from(&collected, d, &req.export, ctx)?; + plan.warnings.extend(left_out.warnings()); + Ok(plan) + })?; + let folders: Vec = roots.iter().map(|r| r.display().to_string()).collect(); + planned.summary.insert(1, format!("read the files {} finds in {}", declared.title, folders.join(", "))); + planned.guard(None) + } +} + +impl super::process::Slot { + /// A page as a source answered it: its tracks (with their waveforms) and playlists, those its + /// blob holds after them, and its `next`. + fn page( + &self, + mut page: LibraryPage, + scratch: &Scratch, + ) -> Result<(Vec, Vec, Option)> { + if let Some(name) = page.blob.take() { + let mut text = String::new(); + let read = scratch.blob(&name).and_then(|mut f| f.read_to_string(&mut text).map_err(|e| e.to_string())); + read.map_err(|why| self.misbehaved(format!("its page's blob: {why}")))?; + let more: LibraryPage = serde_json::from_str(&text) + .map_err(|e| self.misbehaved(format!("its page's blob isn't a page: {e}")))?; + page.tracks.extend(more.tracks); + page.playlists.extend(more.playlists); + } + let answered = page + .tracks + .into_iter() + .map(|track| { + let bands = track.waveform.as_ref().map(|w| { + let mut bytes = Vec::new(); + scratch.blob(&w.blob).and_then(|mut f| f.read_to_end(&mut bytes).map_err(|e| e.to_string()))?; + let bands = Bands::unpack(&bytes).filter(|b| !b.0.is_empty()); + let bands = bands.ok_or("its waveform isn't three bytes a column, at least one column")?; + match w.columns_per_sec.is_finite() && w.columns_per_sec > 0.0 { + true => Ok(anlz::at_detail_rate(&bands, w.columns_per_sec)), + false => { + Err(format!("its waveform's columns a second, {}, aren't more than 0", w.columns_per_sec)) + } + } + }); + Answered { track, bands } + }) + .collect(); + Ok((answered, page.playlists, page.next)) + } +} + +/// The folders granted, each absolute and there, as they are with their links resolved (each +/// once). +fn granted(grants: &[PathBuf]) -> Result> { + if grants.is_empty() { + return Err(Error::invalid("grant the plugin a folder to read its library's files in")); + } + let mut roots: Vec = Vec::new(); + for g in grants { + if !g.is_absolute() { + return Err(Error::invalid(format!("a folder granted must be an absolute path, not {}", g.display()))); + } + let real = g.canonicalize().map(without_verbatim_prefix).ok().filter(|r| r.is_dir()); + let real = real.ok_or_else(|| Error::NotFound(format!("the folder {}", g.display())))?; + if !roots.contains(&real) { + roots.push(real); + } + } + Ok(roots) +} + +/// `path` (as a plugin gave it) as it is with its links resolved, once it's an audio file players +/// play in one of `roots` and not below `drive`, with its size and modification time and its +/// format, or why it isn't. +fn checked( + path: &Path, + roots: &[PathBuf], + drive: &Path, +) -> std::result::Result<(PathBuf, u64, i64, probe::Format), &'static str> { + if !path.is_absolute() { + return Err("isn't an absolute path"); + } + let real = path.canonicalize().map(without_verbatim_prefix).map_err(|_| "isn't there")?; + if !roots.iter().any(|r| real.starts_with(r)) { + return Err("isn't in a folder granted (its links resolved)"); + } + if real.starts_with(drive) { + return Err("is on the drive written to"); + } + let meta = std::fs::metadata(&real).map_err(|_| "can't be read")?; + let name = real.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + if !meta.is_file() || Format::for_players(&name).is_none() { + return Err("isn't an audio file players play"); + } + if meta.len() == 0 { + return Err("is empty"); + } + let format = probe::format(&real).map_err(|_| "can't be read as audio")?; + Ok((real, meta.len(), crate::fsutil::mtime_secs(&meta), format)) +} + +/// What a source answered as a library to write: each track whose audio file is checked, its +/// playlists (with folders made for their paths) of those, and My Tags; and what was left out. +fn collect( + source: &str, + answered: Vec, + playlists: Vec, + roots: &[PathBuf], + drive: &Path, + ctx: &Ctx<'_>, +) -> Result<(Collected<'static>, LeftOut)> { + let found = probe::parallel( + &answered, + Phase::Scan, + |a| (a.track.path.display().to_string(), 0), + |a| checked(&a.track.path, roots, drive), + ctx, + )?; + let mut left_out = LeftOut::default(); + let (mut tracks, mut files) = (Vec::new(), FxHashMap::default()); + // Each track's id, by its path as the source gives it, and those taken by their files. + let (mut ids, mut real_paths) = (FxHashMap::default(), FxHashSet::default()); + let mut tags: IndexMap>> = IndexMap::new(); + for (a, found) in answered.into_iter().zip(found) { + let shown = a.track.path.display().to_string(); + let (real, size, mtime, format) = match found { + Ok(found) => found, + Err(why) => { + left_out.add("track(s)", format!("{shown}: {why}")); + continue; + } + }; + if !real_paths.insert(real.clone()) { + left_out.add("track(s)", format!("{shown}: is another track's audio file too")); + continue; + } + let id = tracks.len() as u32 + 1; + ids.insert(a.track.path.clone(), id); + for tag in &a.track.my_tags { + match tag.split_once('/').filter(|(c, t)| !c.trim().is_empty() && !t.trim().is_empty()) { + Some((category, tag)) => { + tags.entry(category.into()).or_default().entry(tag.into()).or_default().push(id) + } + None => left_out.add("My Tag(s)", format!("{shown}: {tag:?} isn't Category/Tag")), + } + } + let (track, check) = made(id, a, &real, size, mtime, format, &mut left_out); + tracks.push(track); + files.insert(id, check); + } + let mut library = Library { tracks, ..Library::default() }; + for p in playlists { + let id = folders(&mut library, &p.path); + let (known, unknown): (Vec<_>, Vec<_>) = p.tracks.iter().partition(|t| ids.contains_key(*t)); + for t in unknown { + left_out.add("playlist entr(ies)", format!("{} in {}: no track read", t.display(), p.path)); + } + let playlist = library.playlists.iter_mut().find(|x| x.id == id).expect("just made"); + playlist.track_ids = known.into_iter().map(|t| ids[t]).collect(); + } + let tags = tags + .into_iter() + .map(|(name, tags)| TagCategory { + name, + tags: tags.into_iter().map(|(name, track_ids)| TagEntry { name, track_ids }).collect(), + }) + .collect(); + Ok((Collected::checked(source.into(), library, tags, files), left_out)) +} + +/// The track `id` a source answered (`a`), its audio file checked at `real`, and how its files are +/// written; what of it is left out goes in `left_out`. +fn made( + id: u32, + a: Answered, + real: &Path, + size: u64, + mtime: i64, + format: probe::Format, + left_out: &mut LeftOut, +) -> (Track, Checked) { + let t = a.track; + let shown = t.path.display().to_string(); + let beats = match t.beats.as_slice() { + [] => None, + beats => anlz::checked_grid(beats).map_err(|e| left_out.add("beat grid(s)", format!("{shown}: {e}"))).ok(), + }; + let waveform = a.bands.and_then(|b| b.map_err(|why| left_out.add("waveform(s)", format!("{shown}: {why}"))).ok()); + let cues = cues(&t.cues, &shown, left_out); + let key = t.key.as_deref().filter(|k| !k.trim().is_empty()); + let key = key.and_then(|k| { + let note = Camelot::parse(k).map(Camelot::note); + note.ok_or_else(|| left_out.add("key(s)", format!("{shown}: {k:?} isn't a key"))).ok() + }); + let bpm = t.bpm.filter(|b| b.is_finite() && *b > 0.0); + let bpm = bpm.or_else(|| NewAnalysis { beats: beats.clone(), ..NewAnalysis::default() }.tempo()); + let secs = format.duration_secs.unwrap_or(0.0); + let title = match t.title.trim() { + "" => real.file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(), + title => title.to_string(), + }; + let track = Track { + id, + title, + artist: t.artist, + album: t.album, + album_artist: t.album_artist, + genre: t.genre, + label: t.label, + key: key.unwrap_or_default(), + remixer: t.remixer, + composer: t.composer, + comment: t.comment, + isrc: t.isrc, + bpm: bpm.unwrap_or(0.0), + duration_secs: secs.round() as u32, + year: t.year, + rating: t.rating.min(5), + bitrate: if secs > 0.0 { (size as f64 * 8.0 / secs / 1000.0).round() as u32 } else { 0 }, + sample_rate: format.sample_rate, + sample_depth: format.bit_depth, + file_size: size, + file_path: real.to_string_lossy().into_owned(), + ..Track::default() + }; + (track, Checked { size, mtime, cues, beats, waveform }) +} + +/// The cues a source gave a track (`shown`) as its analysis files hold them: a hot cue's slot +/// A-H, each slot once, a loop ending after it starts, colours as cue edits take them. +fn cues(given: &[SourceCue], shown: &str, left_out: &mut LeftOut) -> Vec { + let mut slots = FxHashSet::default(); + let mut out = Vec::new(); + for c in given { + let at = format!("{shown} at {} ms", c.start_ms); + let why = match c.hot_cue { + Some(slot) if slot > 7 => Some("its hot cue isn't A-H"), + Some(slot) if !slots.insert(slot) => Some("another hot cue has its slot"), + _ if c.end_ms.is_some_and(|end| end <= c.start_ms) => Some("its loop doesn't end after it starts"), + _ => None, + }; + if let Some(why) = why { + left_out.add("cue(s)", format!("{at}: {why}")); + continue; + } + let rgb = if c.hot_cue.is_some() { DEFAULT_HOT_CUE_RGB } else { [0; 3] }; + let mut p = CuePoint { + hot_cue: c.hot_cue, + start_ms: c.start_ms, + end_ms: c.end_ms, + comment: c.name.clone(), + color_rgb: rgb, + ..CuePoint::default() + }; + if let Some(Err(e)) = c.color.as_deref().map(|spec| cue_edit::set_color(&mut p, spec)) { + left_out.add("cue colour(s)", format!("{at}: {e}")); + } + out.push(p); + } + out +} + +/// The playlist `path` names (`Sets/Friday`) in `library`, made with the folders above it that +/// aren't there: its id. A playlist is always a new one; folders are shared. +fn folders(library: &mut Library, path: &str) -> u32 { + let mut parent = None; + let parts: Vec<&str> = path.split('/').filter(|p| !p.trim().is_empty()).collect(); + let (name, folders) = parts.split_last().map_or((path, &[][..]), |(n, f)| (*n, f)); + for folder in folders { + let there = library.playlists.iter().find(|p| p.is_folder && p.parent_id == parent && p.name == *folder); + parent = Some(match there { + Some(p) => p.id, + None => add(library, parent, folder, true), + }); + } + add(library, parent, name, false) +} + +/// A new playlist (or folder) `name` in `parent`, last there: its id. +fn add(library: &mut Library, parent: Option, name: &str, is_folder: bool) -> u32 { + let id = library.playlists.len() as u32 + 1; + let sort_order = library.playlists.iter().filter(|p| p.parent_id == parent).count() as u32; + let p = Playlist { id, parent_id: parent, name: name.into(), is_folder, sort_order, track_ids: Vec::new() }; + library.playlists.push(p); + id +} diff --git a/crates/locallibrary/tests/it/plugins.rs b/crates/locallibrary/tests/it/plugins.rs index c68e6482..88f5bc58 100644 --- a/crates/locallibrary/tests/it/plugins.rs +++ b/crates/locallibrary/tests/it/plugins.rs @@ -10,6 +10,7 @@ mod net; mod props; mod providers; mod roles; +mod sources; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/crates/locallibrary/tests/it/plugins/sources.rs b/crates/locallibrary/tests/it/plugins/sources.rs new file mode 100644 index 00000000..22f96aa3 --- /dev/null +++ b/crates/locallibrary/tests/it/plugins/sources.rs @@ -0,0 +1,827 @@ +//! Library sources (feature `sources`): a plugin reads a library from the folders it's granted, a +//! page at a time, and the host builds a drive from it as from rekordbox's collection, leaving out +//! what isn't an audio file in those folders, and making analysis files for the tracks that bring +//! cues, beat grids or waveforms. + +use hegel::TestCase; +use hegel::generators as gs; +use locallibrary::anlz::{self, Bands, Beat}; +use locallibrary::collection_export::{self, CollectionExportOptions, CollectionExportPlan}; +use locallibrary::op::Planned; +use locallibrary::paths; +use locallibrary::plugin::SourceExport; + +use super::*; + +/// The echo manifest granted `fs:read`, with the library source `folder` (any of the test plugin's +/// reads the files in its folders). +fn sourcing() -> String { + let options = "{ type = \"object\", properties = { per_page = { type = \"integer\" }, as_blob = { type = \ + \"boolean\" }, again = { type = \"boolean\" }, also = { type = \"array\", items = { type = \ + \"string\" } } } }"; + let source = + format!("\n[[contributes.sources]]\nname = \"folder\"\ntitle = \"Audio files\"\noptions = {options}\n"); + granting(&["fs:read"]) + &source +} + +/// `secs` of silence, as a WAV file. +fn wav(secs: f64) -> Vec { + common::media::wav_of(44_100, 16, 1, secs) +} + +fn beats(n: usize) -> Vec { + (0..n).map(|i| Beat { beat_number: (i % 4) as u16 + 1, bpm: 120.0, time_ms: 100 + i as u32 * 500 }).collect() +} + +impl Fixture { + fn sourcing() -> Fixture { + Fixture::new(|dir| vec![spec(dir, &sourcing(), &[])], Options::default()) + } + + /// A folder of its own, made. + fn folder(&self, name: &str) -> PathBuf { + let dir = self._dir.path().join(name); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// The folder the drive is built in. + fn stick(&self) -> PathBuf { + self._dir.path().join("stick") + } + + /// Building the stick from what the source reads in `grants`, with `options`. + fn request(&self, grants: &[&Path], options: Value) -> SourceExport { + SourceExport { + plugin: "echo".into(), + source: "folder".into(), + options, + grants: grants.iter().map(|g| g.to_path_buf()).collect(), + drive: self.stick().display().to_string(), + export: CollectionExportOptions::default(), + } + } + + /// Plans building the stick from what the source reads in `grants`, with `options`. + fn build( + &self, + grants: &[&Path], + options: Value, + ) -> Result> { + self.plugins.plan_source_export(&self.request(grants, options), Some(&self.store), &Ctx::silent()) + } +} + +/// Writes `bytes` at `path`, and what the source says of it besides (`told`) beside it. +fn put(path: &Path, bytes: &[u8], told: Option) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, bytes).unwrap(); + if let Some(told) = told { + std::fs::write(format!("{}.json", path.display()), told.to_string()).unwrap(); + } +} + +/// The library on the stick, by title: each track with its analysis there. +fn built(stick: &Path) -> Vec<(locallibrary::Track, anlz::TrackAnalysis)> { + let device = locallibrary::Device::open(stick).unwrap(); + let mut tracks: Vec<_> = device + .library() + .unwrap() + .tracks + .into_iter() + .map(|t| (anlz::load(&device, &t), t)) + .map(|(a, t)| (t, a)) + .collect(); + tracks.sort_by(|a, b| a.0.title.cmp(&b.0.title)); + tracks +} + +#[test] +fn a_source_builds_a_drive_from_the_audio_files_it_was_granted() { + let f = Fixture::sourcing(); + let music = f.folder("music"); + let bands = Bands((0..1500).map(|i| [(i % 256) as u8, 100, 200]).collect()); + let told = json!({ + "title": "A", "artist": "Art", "album": "Alb", "key": "8A", "bpm": 124.0, "rating": 9, + "my_tags": ["Mood/Dark", "untagged"], + "cues": [ + { "hot_cue": 0, "start_ms": 1000, "name": "drop", "color": "red" }, + { "start_ms": 2000 }, + { "start_ms": 3000, "end_ms": 4000 }, + { "hot_cue": 0, "start_ms": 5000 }, + { "hot_cue": 9, "start_ms": 6000 }, + ], + "beats": beats(16), + "waveform": { "blob": "wave.bands", "columns_per_sec": 150.0 }, + "blobs": { "wave.bands": bands.pack() }, + }); + put(&music.join("a.wav"), &wav(10.0), Some(told)); + put(&music.join("sub/b.wav"), &wav(2.0), None); + put(&music.join("notes.txt"), b"not audio", None); + put(&music.join("fake.mp3"), b"not audio either", None); + let lists = json!({ "Sets/Friday": ["a.wav", "sub/b.wav", "gone.wav"], "Sets/Empty": [] }); + std::fs::write(music.join("playlists.json"), lists.to_string()).unwrap(); + let outside = f.folder("outside").join("c.wav"); + put(&outside, &wav(1.0), None); + #[cfg(unix)] + std::os::unix::fs::symlink(&outside, music.join("link.wav")).unwrap(); + let options = json!({ "also": [outside, "relative.wav"] }); + + let planned = f.build(&[&music], options.clone()).unwrap(); + let plan = &planned.plan; + assert_eq!((plan.tracks, plan.playlist_count, plan.source.as_deref()), (2, 2, Some("echo:folder"))); + let real = real(&music); + assert!(planned.summary.iter().any(|s| *s == format!("read the files Audio files finds in {}", real.display()))); + let warned = |what: &str| planned.warnings.iter().find(|w| w.contains(what)).cloned().unwrap_or_default(); + // fake.mp3, link.wav (on Unix), notes.txt, c.wav and relative.wav, the first three shown. + let tracks = warned("track(s) left out"); + let left_out = if cfg!(unix) { 5 } else { 4 }; + assert!(tracks.starts_with(&format!("{left_out} track(s) left out, e.g. ")), "{:?}", planned.warnings); + let mut whys = vec!["fake.mp3: can't be read as audio", "notes.txt: isn't an audio file players play"]; + if cfg!(unix) { + whys.push("link.wav: isn't in a folder granted (its links resolved)"); + } + for why in whys { + assert!(tracks.contains(why), "{why}: {tracks}"); + } + assert!(warned("playlist entr(ies)").starts_with("1 playlist entr(ies) left out"), "{:?}", planned.warnings); + assert!(warned("My Tag(s)").contains("\"untagged\" isn't Category/Tag"), "{:?}", planned.warnings); + assert!(warned("cue(s)").starts_with("2 cue(s) left out"), "{:?}", planned.warnings); + + let report = planned.apply(Some(&f.store), &Ctx::silent()).unwrap(); + assert_eq!((report.tracks, report.playlists, report.tagged_tracks), (2, 2, 1)); + let tracks = built(&f.stick()); + let [(a, at), (b, bt)] = &tracks[..] else { panic!("two tracks: {tracks:?}") }; + assert_eq!((a.artist.as_str(), a.album.as_str(), a.key.as_str(), a.bpm, a.rating), ("Art", "Alb", "Am", 124.0, 5)); + assert_eq!((a.duration_secs, a.sample_rate, b.title.as_str(), b.duration_secs), (10, 44_100, "b", 2)); + assert!(a.file_path.starts_with("/Contents/Art/Alb/"), "{}", a.file_path); + let copied = paths::to_fs(&f.stick(), &paths::key(&a.file_path)); + assert_eq!(std::fs::read(copied).unwrap(), std::fs::read(music.join("a.wav")).unwrap()); + let mut cues: Vec<(Option, u32, Option, &str)> = + at.cues.iter().map(|c| (c.hot_cue, c.start_ms, c.end_ms, c.comment.as_str())).collect(); + cues.sort(); + assert_eq!(cues, [(None, 2000, None, ""), (None, 3000, Some(4000), ""), (Some(0), 1000, None, "drop")]); + assert_eq!(at.beats, beats(16)); + assert!(at.preview.is_some() && at.detail.is_some(), "its waveforms"); + assert!(b.analyze_path.is_empty() && bt.beats.is_empty(), "nothing to make analysis files of"); + let stick = locallibrary::Device::open(f.stick()).unwrap(); + let library = stick.library().unwrap(); + let friday = library.find_playlist("Sets/Friday").unwrap(); + let titles: Vec<&str> = friday.track_ids.iter().map(|id| library.track(*id).unwrap().title.as_str()).collect(); + assert_eq!(titles, ["A", "b"]); + assert_eq!(collection_export::record(&stick).unwrap().unwrap().source.as_deref(), Some("echo:folder")); + + // Built again, nothing has changed: nothing is copied. + let again = f.build(&[&music], options).unwrap(); + assert!(again.plan.copies.is_empty(), "{:?}", again.plan.copies); + assert!(!again.plan.replaces_library); +} + +/// Pages, in a message or in a blob, add up to the library; a source that never stops paging is +/// refused. +#[test] +fn a_source_answers_in_pages_and_blobs() { + let f = Fixture::sourcing(); + let music = f.folder("music"); + for name in ["a", "b", "c"] { + put(&music.join(format!("{name}.wav")), &wav(1.0), Some(json!({ "title": name }))); + } + for options in [json!({ "per_page": 1 }), json!({ "per_page": 2, "as_blob": true })] { + let planned = f.build(&[&music], options.clone()).unwrap(); + assert_eq!(planned.plan.tracks, 3, "{options}"); + } + let e = f.build(&[&music], json!({ "per_page": 2, "again": true })).unwrap_err(); + assert_eq!(failure(e), "the Echo plugin failed: it answered the page after \"3\" twice"); + let e = f.build(&[&music], json!({ "per_page": "two" })).unwrap_err(); + assert_eq!(e.code(), "invalid", "options are checked against the source's schema: {e}"); +} + +/// What a track brings that can't be written (a key that isn't one, a beat grid out of order, a +/// waveform that isn't one, a loop ending before it starts, a colour players don't have) is left +/// out with a warning, and the track is written without it. +#[test] +fn what_a_track_brings_that_cant_be_written_is_left_out() { + let f = Fixture::sourcing(); + let music = f.folder("music"); + let mut grid = beats(4); + grid.swap(1, 2); + let told = json!({ + "title": "D", "key": "x", + "beats": grid, + "waveform": { "blob": "none.bands", "columns_per_sec": 150.0 }, + "cues": [{ "start_ms": 500, "end_ms": 100 }, { "start_ms": 600, "color": "nope" }], + }); + put(&music.join("d.wav"), &wav(2.0), Some(told)); + let other = json!({ "title": "E", "waveform": { "blob": "e.bands", "columns_per_sec": 0.0 }, "blobs": { "e.bands": [1, 2, 3] } }); + put(&music.join("e.wav"), &wav(2.0), Some(other)); + let planned = f.build(&[&music], Value::Null).unwrap(); + let warned = |what: &str| planned.warnings.iter().find(|w| w.contains(what)).cloned().unwrap_or_default(); + assert!(warned("key(s) left out").contains("\"x\" isn't a key"), "{:?}", planned.warnings); + assert!(warned("beat grid(s) left out").contains("isn't after the one before it"), "{:?}", planned.warnings); + assert!(warned("2 waveform(s) left out").contains("aren't more than 0"), "{:?}", planned.warnings); + assert!(warned("cue(s) left out").contains("its loop doesn't end after it starts"), "{:?}", planned.warnings); + assert!(warned("cue colour(s) left out").contains("unknown colour \"nope\""), "{:?}", planned.warnings); + planned.apply(Some(&f.store), &Ctx::silent()).unwrap(); + let tracks = built(&f.stick()); + let [(d, da), (e, ea)] = &tracks[..] else { panic!("two tracks: {tracks:?}") }; + assert_eq!((d.title.as_str(), d.key.as_str(), d.bpm, e.title.as_str()), ("D", "", 0.0, "E")); + let cues: Vec = da.cues.iter().map(|c| c.start_ms).collect(); + assert_eq!((cues, da.beats.is_empty(), da.preview.is_none()), (vec![600], true, true)); + assert!(e.analyze_path.is_empty() && ea.preview.is_none(), "nothing left to write for it"); +} + +/// The folders granted are absolute folders off the drive written to, at least one; and a file +/// that changed after the export was planned isn't copied. +#[test] +fn grants_are_checked_and_files_copied_as_they_were_checked() { + let f = Fixture::sourcing(); + let music = f.folder("music"); + put(&music.join("a.wav"), &wav(1.0), None); + let refused = |grants: &[&Path]| f.build(grants, Value::Null).map(|_| ()).unwrap_err(); + assert_eq!(refused(&[]).code(), "invalid"); + assert_eq!(refused(&[Path::new("music")]).code(), "invalid"); + assert_eq!(refused(&[&f._dir.path().join("nowhere")]).code(), "not_found"); + std::fs::create_dir_all(f.stick().join("Contents")).unwrap(); + let e = refused(&[&f.stick().join("Contents")]); + assert!(e.to_string().contains("the drive written to"), "{e}"); + std::fs::remove_dir_all(f.stick()).unwrap(); + + let planned = f.build(&[&music], Value::Null).unwrap(); + let mut changed = wav(1.0); + *changed.last_mut().unwrap() = 1; + std::fs::write(music.join("a.wav"), changed).unwrap(); + let e = planned.apply(Some(&f.store), &Ctx::silent()).unwrap_err(); + assert!(matches!(e, Error::Corrupt { .. }), "{e}"); +} + +/// What a drawn track's file is. +#[derive(Debug, Clone, Copy, PartialEq)] +enum Placed { + /// A WAV file in the folder granted. + Granted, + /// One outside it. + Outside, + /// A link in the folder granted to one outside it. + LinkedOut, + /// Text named as an MP3, in the folder granted. + NotAudio, +} + +/// For any tracks a source reads, the drive holds exactly the WAV files in the folder granted, +/// each with its title and the cues it was given (a hot cue per slot, A-H), read back from the +/// analysis files made for it, its other cues and files left out with a warning. +#[hegel::test(test_cases = 8, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn the_drive_holds_exactly_the_granted_audio_files_and_their_cues(tc: TestCase) { + let f = Fixture::sourcing(); + let (music, outside) = (f.folder("music"), f.folder("outside")); + let mut places = vec![Placed::Granted, Placed::Outside, Placed::NotAudio]; + if cfg!(unix) { + places.push(Placed::LinkedOut); + } + let drawn: Vec = tc.draw(gs::vecs(gs::sampled_from(places)).max_size(5)); + let (mut also, mut wanted) = (Vec::new(), Vec::new()); + for (i, placed) in drawn.iter().enumerate() { + let slots: Vec = tc.draw(gs::vecs(gs::integers::().max_value(9)).max_size(4)); + let cues: Vec = + slots.iter().enumerate().map(|(j, s)| json!({ "hot_cue": s, "start_ms": 100 * j as u32 })).collect(); + let told = json!({ "title": format!("t{i}"), "cues": cues }); + match placed { + Placed::Granted => { + put(&music.join(format!("{i}.wav")), &wav(1.0), Some(told)); + let mut seen = Vec::new(); + let kept = slots.iter().enumerate().filter(|(_, s)| { + **s <= 7 && !seen.contains(*s) && { + seen.push(**s); + true + } + }); + let mut cues: Vec<(u8, u32)> = kept.map(|(j, s)| (*s, 100 * j as u32)).collect(); + cues.sort(); + wanted.push((format!("t{i}"), cues)); + } + Placed::Outside => { + let file = outside.join(format!("{i}.wav")); + put(&file, &wav(1.0), None); + also.push(file); + } + Placed::NotAudio => put(&music.join(format!("{i}.mp3")), b"text", Some(told)), + Placed::LinkedOut => { + let file = outside.join(format!("{i}.wav")); + put(&file, &wav(1.0), None); + #[cfg(unix)] + std::os::unix::fs::symlink(&file, music.join(format!("{i}-link.wav"))).unwrap(); + } + } + } + let planned = f.build(&[&music], json!({ "also": also })).unwrap(); + let refused = drawn.iter().filter(|p| **p != Placed::Granted).count(); + let said = planned.warnings.iter().any(|w| w.starts_with(&format!("{refused} track(s) left out"))); + assert_eq!(said, refused > 0, "{:?}", planned.warnings); + planned.apply(Some(&f.store), &Ctx::silent()).unwrap(); + let mut got: Vec<(String, Vec<(u8, u32)>)> = built(&f.stick()) + .into_iter() + .map(|(t, a)| { + let mut cues: Vec<(u8, u32)> = a.cues.iter().map(|c| (c.hot_cue.unwrap(), c.start_ms)).collect(); + cues.sort(); + (t.title, cues) + }) + .collect(); + got.sort(); + wanted.sort(); + assert_eq!(got, wanted); +} + +/// `path` with its links resolved, as the host gives it to a plugin. +fn real(path: &Path) -> PathBuf { + locallibrary::device::without_verbatim_prefix(path.canonicalize().unwrap()) +} + +/// What a source gives for a drawn track, and why the host leaves it out (if it does). +#[derive(Debug, Clone, Copy, PartialEq)] +enum Given { + /// A WAV file in the folder granted: written. + Audio, + /// A WAV file in the folder granted, given again by its path: the second is left out. + Twice, + /// Text named as an MP3, in the folder granted. + NotAudio, + /// A text file in the folder granted. + Text, + /// An empty WAV file in the folder granted. + Empty, + /// A WAV file on the drive written to, which is in the folder granted. + OnDrive, + /// A WAV file outside the folder granted, by its path. + Outside, + /// A path in the folder granted to nothing. + Gone, + /// A relative path. + Relative, + /// A link in the folder granted to a WAV file outside it. + LinkedOut, +} + +impl Given { + /// Why a track given so is left out. + fn why(self) -> Option<&'static str> { + match self { + Given::Audio => None, + Given::Twice => Some("is another track's audio file too"), + Given::NotAudio => Some("can't be read as audio"), + Given::Text => Some("isn't an audio file players play"), + Given::Empty => Some("is empty"), + Given::OnDrive => Some("is on the drive written to"), + Given::Outside | Given::LinkedOut => Some("isn't in a folder granted (its links resolved)"), + Given::Gone => Some("isn't there"), + Given::Relative => Some("isn't an absolute path"), + } + } +} + +/// For any tracks a source gives, in its folders or by their paths, the plan holds those whose +/// file is a WAV file in the folder granted, off the drive written to, each once; one warning +/// counts the rest, giving the first three as the source gave them, with why each is left out. +#[hegel::test(test_cases = 12, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn every_track_left_out_is_counted_with_why(tc: TestCase) { + let f = Fixture::sourcing(); + let (music, outside) = (f.folder("music"), f.folder("outside")); + let (real, drive) = (real(&music), music.join("stick")); + let mut kinds = vec![ + Given::Audio, + Given::Twice, + Given::NotAudio, + Given::Text, + Given::Empty, + Given::OnDrive, + Given::Outside, + Given::Gone, + Given::Relative, + ]; + if cfg!(unix) { + kinds.push(Given::LinkedOut); + } + let drawn: Vec = tc.draw(gs::vecs(gs::sampled_from(kinds)).max_size(6)); + // The files the source finds in the folder, as it gives them (sorted), then those it's told. + let (mut found, mut told) = (Vec::new(), Vec::new()); + for (i, given) in drawn.iter().enumerate() { + let name = format!("{i:02}.wav"); + let in_music = |name: &str, bytes: &[u8], found: &mut Vec<(PathBuf, Option<&str>)>| { + put(&music.join(name), bytes, None); + found.push((real.join(name), given.why())); + }; + match given { + Given::Audio => in_music(&name, &wav(1.0), &mut found), + Given::Empty => in_music(&name, b"", &mut found), + Given::Twice => { + in_music(&name, &wav(1.0), &mut found); + found.last_mut().unwrap().1 = None; + told.push((music.join(&name).display().to_string(), given.why())); + } + Given::NotAudio => in_music(&format!("{i:02}.mp3"), b"text", &mut found), + Given::Text => in_music(&format!("{i:02}.txt"), b"text", &mut found), + Given::OnDrive => in_music(&format!("stick/{name}"), &wav(1.0), &mut found), + Given::Outside => { + put(&outside.join(&name), &wav(1.0), None); + told.push((outside.join(&name).display().to_string(), given.why())); + } + Given::Gone => told.push((music.join(format!("{i:02}-gone.wav")).display().to_string(), given.why())), + Given::Relative => told.push((name, given.why())), + Given::LinkedOut => { + put(&outside.join(&name), &wav(1.0), None); + #[cfg(unix)] + std::os::unix::fs::symlink(outside.join(&name), music.join(format!("{i:02}-link.wav"))).unwrap(); + found.push((real.join(format!("{i:02}-link.wav")), given.why())); + } + } + } + found.sort(); + let found = found.into_iter().map(|(p, why)| (p.display().to_string(), why)); + let given: Vec<(String, Option<&str>)> = found.chain(told.iter().cloned()).collect(); + + let paths: Vec<&str> = told.iter().map(|(p, _)| p.as_str()).collect(); + let mut req = f.request(&[&music], json!({ "also": paths })); + req.drive = drive.display().to_string(); + req.export.replace = true; + let planned = f.plugins.plan_source_export(&req, Some(&f.store), &Ctx::silent()).unwrap(); + let left: Vec = given.iter().filter_map(|(p, why)| why.map(|why| format!("{p}: {why}"))).collect(); + assert_eq!(planned.plan.tracks, given.len() - left.len(), "{:?}", planned.warnings); + let said: Vec<&String> = planned.warnings.iter().filter(|w| w.contains("track(s) left out")).collect(); + let wanted = format!( + "{} track(s) left out, e.g. {}", + left.len(), + left.iter().take(3).cloned().collect::>().join("; ") + ); + assert_eq!(said, Vec::from_iter((!left.is_empty()).then_some(&wanted))); +} + +/// For any number of tracks read a page at a time (any page size, in messages or blobs), the +/// drive holds every one, and each playlist the source gives (folders made for its path) holds +/// those of its entries that are tracks read, in order, the rest counted in a warning. A source +/// whose every page says there's one after it is refused at the first page it gives twice, and +/// one that can't read what it's given fails. +#[hegel::test(test_cases = 10, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn pages_and_blobs_add_up_to_the_library(tc: TestCase) { + let f = Fixture::sourcing(); + let music = f.folder("music"); + let n = tc.draw(gs::integers::().max_value(5)); + for i in 0..n { + put(&music.join(format!("{i:02}.wav")), &wav(1.0), Some(json!({ "title": format!("t{i}") }))); + } + // Each playlist's entries by the track they name, `n` naming a file that isn't there. + let mut lists: Vec<(&str, Vec)> = Vec::new(); + for path in ["Sets/Friday", "Sets/Late/Peak", "Warmup"] { + if tc.draw(gs::booleans()) { + lists.push((path, tc.draw(gs::vecs(gs::integers::().max_value(n)).max_size(4)))); + } + } + let file = |i: usize| if i < n { format!("{i:02}.wav") } else { "gone.wav".into() }; + let named: serde_json::Map = + lists.iter().map(|(p, entries)| (p.to_string(), entries.iter().map(|i| json!(file(*i))).collect())).collect(); + std::fs::write(music.join("playlists.json"), Value::Object(named).to_string()).unwrap(); + let mut options = json!({ "as_blob": tc.draw(gs::booleans()) }); + if tc.draw(gs::booleans()) { + options["per_page"] = json!(tc.draw(gs::integers::().min_value(1).max_value(4))); + } + // What the source reads that isn't JSON (its playlists, or what it's told of a file) fails it. + let broken = match tc.draw(gs::integers::().max_value(7)) { + 0 => Some(music.join("playlists.json")), + 1 => Some(music.join("zz.wav.json")), + _ => None, + }; + if let Some(broken) = broken { + put(&music.join("zz.wav"), &wav(1.0), None); + std::fs::write(broken, "{").unwrap(); + let e = f.build(&[&music], options).map(|_| ()).unwrap_err(); + let said = failure(e); + assert!(said.starts_with("the Echo plugin failed: ") && said.contains("EOF"), "{said}"); + return; + } + if tc.draw(gs::integers::().max_value(3)) == 0 { + options["again"] = json!(true); + let e = f.build(&[&music], options).map(|_| ()).unwrap_err(); + assert_eq!(failure(e), format!("the Echo plugin failed: it answered the page after \"{n}\" twice")); + return; + } + let planned = f.build(&[&music], options.clone()).unwrap(); + assert_eq!((planned.plan.tracks, planned.plan.playlist_count), (n, lists.len()), "{options}"); + let gone: usize = lists.iter().map(|(_, e)| e.iter().filter(|i| **i == n).count()).sum(); + let said = planned.warnings.iter().find(|w| w.contains("playlist entr(ies) left out")); + assert_eq!(said.is_some(), gone > 0, "{:?}", planned.warnings); + assert!(said.is_none_or(|w| w.starts_with(&format!("{gone} playlist entr(ies) left out, e.g. "))), "{said:?}"); + planned.apply(Some(&f.store), &Ctx::silent()).unwrap(); + let library = locallibrary::Device::open(f.stick()).unwrap().library().unwrap(); + assert_eq!(library.tracks.len(), n); + for (path, entries) in &lists { + let list = library.find_playlist(path).unwrap_or_else(|| panic!("no {path}")); + let titles: Vec<&str> = list.track_ids.iter().map(|id| library.track(*id).unwrap().title.as_str()).collect(); + let wanted: Vec = entries.iter().filter(|i| **i < n).map(|i| format!("t{i}")).collect(); + assert_eq!(titles, wanted, "{path}"); + } +} + +/// What a drawn track brings besides its file, as the source tells it, and what the host keeps. +struct Brought { + told: Value, + /// Its title, key, tempo and rating as written. + written: (String, &'static str, f64, u8), + /// Its cues kept, beat grid kept, and whether its waveform is. + cues: Vec<(Option, u32, Option)>, + beats: bool, + waveform: bool, + /// How many of each kind of thing it brings are left out, by the warning's name for them. + left: [(&'static str, usize); 6], +} + +/// A track (the `i`th, its file named `name`) bringing a key, a beat grid, a waveform, My Tags +/// and cues, each drawn to be one players take or one they don't. +#[hegel::test_helper] +fn brought(tc: &TestCase, i: usize, name: &str) -> Brought { + let mut left = + [("key(s)", 0), ("beat grid(s)", 0), ("waveform(s)", 0), ("My Tag(s)", 0), ("cue(s)", 0), ("cue colour(s)", 0)]; + let (key, written_key) = match tc.draw(gs::integers::().max_value(3)) { + 0 => (Value::Null, ""), + 1 => (json!("8A"), "Am"), + 2 => (json!(" "), ""), + _ => { + left[0].1 += 1; + (json!("x"), "") + } + }; + let mut grid = beats(8); + let beats_kept = match tc.draw(gs::integers::().max_value(2)) { + 0 => { + grid.clear(); + false + } + 1 => true, + _ => { + grid.swap(2, 3); + left[1].1 += 1; + false + } + }; + // The tracks of a page share a scratch folder: each names its waveform's blob for itself. + let blob = format!("{i}.bands"); + let packed = Bands((0..300).map(|c| [(c % 256) as u8, 50, 9]).collect()).pack(); + let (columns_per_sec, bytes, waveform_kept) = match tc.draw(gs::integers::().max_value(5)) { + 0 => (None, None, false), + 1 => (Some(150.0), Some(json!(packed)), true), + 2 => (Some(0.0), Some(json!(packed)), false), + 3 => (Some(150.0), None, false), + 4 => (Some(150.0), Some(json!([1, 2])), false), + _ => (Some(150.0), Some(json!([])), false), + }; + let waveform = columns_per_sec.map(|c| json!({ "blob": blob, "columns_per_sec": c })); + let blobs = Value::Object(bytes.map(|b| (blob.clone(), b)).into_iter().collect()); + left[2].1 += usize::from(waveform.is_some() && !waveform_kept); + let tags: Vec<&str> = + tc.draw(gs::vecs(gs::sampled_from(vec!["Mood/Dark", "Genre/House", "untagged", "/x", "Mood/ "])).max_size(3)); + left[3].1 += tags.iter().filter(|t| !t.starts_with("Mood/D") && !t.starts_with("Genre/")).count(); + let (mut cues, mut kept, mut slots) = (Vec::new(), Vec::new(), Vec::new()); + for j in 0..tc.draw(gs::integers::().max_value(4)) { + let start = 1000 + j * 1000; + let hot: Option = tc.draw(gs::booleans()).then(|| tc.draw(gs::integers::().max_value(9))); + let end = match tc.draw(gs::integers::().max_value(2)) { + 0 => None, + 1 => Some(start + 500), + _ => Some(start), + }; + let colour = tc.draw(gs::sampled_from(vec![None, Some("red"), Some("nope")])); + cues.push(json!({ "hot_cue": hot, "start_ms": start, "end_ms": end, "color": colour })); + let refused = match hot { + Some(s) if s > 7 => true, + Some(s) if slots.contains(&s) => true, + _ => end == Some(start), + }; + if let Some(s) = hot.filter(|s| *s <= 7) { + slots.push(s); + } + if refused { + left[4].1 += 1; + continue; + } + left[5].1 += usize::from(colour == Some("nope")); + kept.push((hot, start, end)); + } + let bpm = tc.draw(gs::sampled_from(vec![None, Some(124.0), Some(-1.0)])); + let written_bpm = bpm.filter(|b: &f64| *b > 0.0).or(beats_kept.then_some(120.0)).unwrap_or(0.0); + let titled = tc.draw(gs::booleans()); + let title = if titled { format!("t{i}") } else { String::new() }; + let rating = tc.draw(gs::integers::().max_value(9)); + let told = json!({ + "title": title, "key": key, "beats": grid, "waveform": waveform, "blobs": blobs, "my_tags": tags, + "cues": cues, "bpm": bpm, "rating": rating, + }); + let written_title = if titled { title } else { name.trim_end_matches(".wav").to_string() }; + kept.sort(); + Brought { + told, + written: (written_title, written_key, written_bpm, rating.min(5)), + cues: kept, + beats: beats_kept, + waveform: waveform_kept, + left, + } +} + +/// For any tracks bringing keys, beat grids, waveforms, My Tags and cues players take or don't, +/// each is written with what players take, read back from the drive (its title its file's name +/// when it has none, its tempo the beat grid's when it gives none), and one warning a kind counts +/// what's left out. +#[hegel::test(test_cases = 12, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn what_tracks_bring_is_written_or_counted_as_left_out(tc: TestCase) { + let f = Fixture::sourcing(); + let music = f.folder("music"); + let n = tc.draw(gs::integers::().min_value(1).max_value(3)); + let brought: Vec = (0..n) + .map(|i| { + let name = format!("{i:02}.wav"); + let b = brought(&tc, i, &name); + // Long enough for its beat grid and cues. + put(&music.join(&name), &wav(5.0), Some(b.told.clone())); + b + }) + .collect(); + let planned = f.build(&[&music], Value::Null).unwrap(); + for (i, what) in brought[0].left.iter().map(|(what, _)| *what).enumerate() { + let left: usize = brought.iter().map(|b| b.left[i].1).sum(); + let said = planned.warnings.iter().find(|w| w.contains(&format!(" {what} left out, e.g. "))); + let count = said.and_then(|w| w.split(' ').next()?.parse::().ok()); + assert_eq!(count, (left > 0).then_some(left), "{what}: {:?}", planned.warnings); + } + planned.apply(Some(&f.store), &Ctx::silent()).unwrap(); + let mut wanted: Vec<&Brought> = brought.iter().collect(); + wanted.sort_by(|a, b| a.written.0.cmp(&b.written.0)); + let tracks = built(&f.stick()); + assert_eq!(tracks.len(), n); + for ((t, a), b) in tracks.iter().zip(wanted) { + assert_eq!((t.title.clone(), t.key.as_str(), t.bpm, t.rating), b.written); + let mut cues: Vec<(Option, u32, Option)> = + a.cues.iter().map(|c| (c.hot_cue, c.start_ms, c.end_ms)).collect(); + cues.sort(); + assert_eq!(cues, b.cues, "{}", t.title); + assert_eq!((a.beats.is_empty(), a.preview.is_some()), (!b.beats, b.waveform), "{}", t.title); + let nothing = b.cues.is_empty() && !b.beats && !b.waveform; + assert_eq!(t.analyze_path.is_empty(), nothing, "analysis files only for what there is: {}", t.title); + } +} + +/// Grows `file` by a byte as the export plan starts hashing the files it copies, as a file +/// being written while the export is planned does. +struct GrowsWhileHashed { + file: PathBuf, + grown: AtomicBool, +} + +impl locallibrary::Progress for GrowsWhileHashed { + fn on_event(&self, e: &locallibrary::Event) { + let hashing = matches!(e, locallibrary::Event::Item { phase: locallibrary::progress::Phase::Hash, .. }); + if hashing && !self.grown.swap(true, Ordering::SeqCst) { + grow(&self.file); + } + } +} + +/// Adds a byte to the end of `file`. +fn grow(file: &Path) { + use std::io::Write; + std::fs::OpenOptions::new().append(true).open(file).unwrap().write_all(&[0]).unwrap(); +} + +/// For any of a source's files changed while the export is planned, after its check, or after +/// the plan, or none: a size changed while it's planned refuses the plan, naming the file; any +/// change after it refuses the copy; with none, each file is copied as it is, and planned again +/// there's nothing to copy. +#[hegel::test(test_cases = 10, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn files_are_copied_only_as_they_were_checked(tc: TestCase) { + let f = Fixture::sourcing(); + let music = f.folder("music"); + let n = tc.draw(gs::integers::().min_value(1).max_value(3)); + let files: Vec = (0..n).map(|i| music.join(format!("{i:02}.wav"))).collect(); + for (i, file) in files.iter().enumerate() { + put(file, &wav(1.0 + i as f64), None); + } + let changed = &files[tc.draw(gs::integers::().max_value(n - 1))]; + let name = changed.file_name().unwrap().to_string_lossy().into_owned(); + let watch = GrowsWhileHashed { file: changed.clone(), grown: AtomicBool::new(false) }; + let req = f.request(&[&music], Value::Null); + match tc.draw(gs::integers::().max_value(3)) { + 0 => { + let e = f + .plugins + .plan_source_export(&req, Some(&f.store), &Ctx::new(&watch, locallibrary::CancelToken::new())) + .unwrap_err(); + assert_eq!(e.code(), "invalid", "{e}"); + assert!( + e.to_string().ends_with(&format!("{name} changed while the export was planned; plan it again")), + "{e}" + ); + } + changed_after @ (1 | 2) => { + let planned = f.build(&[&music], Value::Null).unwrap(); + match changed_after { + 1 => common::rot(changed), + _ => grow(changed), + } + let e = planned.apply(Some(&f.store), &Ctx::silent()).unwrap_err(); + let Error::Corrupt { path, .. } = &e else { panic!("{e}") }; + assert!(path.ends_with(&name), "{e}"); + } + _ => { + let planned = f.build(&[&music], Value::Null).unwrap(); + assert_eq!(planned.plan.copies.iter().filter(|c| c.path.ends_with(".wav")).count(), n); + planned.apply(Some(&f.store), &Ctx::silent()).unwrap(); + let mut copied: Vec> = built(&f.stick()) + .iter() + .map(|(t, _)| std::fs::read(paths::to_fs(&f.stick(), &paths::key(&t.file_path))).unwrap()) + .collect(); + let mut wanted: Vec> = files.iter().map(|p| std::fs::read(p).unwrap()).collect(); + copied.sort(); + wanted.sort(); + assert!(copied == wanted, "each file copied as it is"); + assert!(f.build(&[&music], Value::Null).unwrap().plan.copies.is_empty(), "nothing changed since"); + } + } +} + +/// A folder a drawn export grants the source. +#[derive(Debug, Clone, Copy, PartialEq)] +enum Grant { + /// The music folder, with one WAV file. + Music, + /// The music folder again, spelled another way. + MusicAgain, + /// Another folder, with one WAV file. + Other, + /// A relative path. + Relative, + /// A path to nothing. + Nowhere, + /// A file, not a folder. + File, + /// A folder on the drive written to. + OnDrive, +} + +/// For any folders granted, the export is refused with none, at the first that's relative (as +/// invalid) or isn't a folder (as not found), or for one on the drive written to; otherwise the +/// source reads each folder once, its links resolved, and the plan says which it read. +#[hegel::test(test_cases = 20, suppress_health_check = [hegel::HealthCheck::TooSlow])] +fn only_absolute_folders_off_the_drive_are_granted(tc: TestCase) { + let f = Fixture::sourcing(); + let (music, other) = (f.folder("music"), f.folder("other")); + put(&music.join("a.wav"), &wav(1.0), None); + put(&other.join("b.wav"), &wav(1.0), None); + let kinds = vec![ + Grant::Music, + Grant::MusicAgain, + Grant::Other, + Grant::Relative, + Grant::Nowhere, + Grant::File, + Grant::OnDrive, + ]; + let drawn: Vec = tc.draw(gs::vecs(gs::sampled_from(kinds)).max_size(4)); + if drawn.contains(&Grant::OnDrive) { + std::fs::create_dir_all(f.stick().join("Contents")).unwrap(); + } + let grants: Vec = drawn + .iter() + .map(|g| match g { + Grant::Music => music.clone(), + Grant::MusicAgain => music.join("."), + Grant::Other => other.clone(), + Grant::Relative => PathBuf::from("music"), + Grant::Nowhere => f._dir.path().join("nowhere"), + Grant::File => music.join("a.wav"), + Grant::OnDrive => f.stick().join("Contents"), + }) + .collect(); + let given: Vec<&Path> = grants.iter().map(PathBuf::as_path).collect(); + let got = f.build(&given, Value::Null); + let refused = drawn.iter().find(|g| matches!(g, Grant::Relative | Grant::Nowhere | Grant::File)); + let e = match (drawn.is_empty(), refused) { + (true, _) => Some(("invalid", "grant the plugin a folder")), + (_, Some(Grant::Relative)) => Some(("invalid", "must be an absolute path")), + (_, Some(_)) => Some(("not_found", "the folder")), + _ if drawn.contains(&Grant::OnDrive) => Some(("invalid", "the drive written to: grant a folder elsewhere")), + _ => None, + }; + if let Some((code, said)) = e { + let e = got.map(|_| ()).unwrap_err(); + assert!(e.code() == code && e.to_string().contains(said), "{drawn:?}: {e}"); + return; + } + let planned = got.unwrap(); + let mut read: Vec = Vec::new(); + for g in &grants { + let real = real(g); + if !read.contains(&real) { + read.push(real); + } + } + assert_eq!(planned.plan.tracks, read.len(), "a track a folder"); + let folders: Vec = read.iter().map(|r| r.display().to_string()).collect(); + let summary = format!("read the files Audio files finds in {}", folders.join(", ")); + assert!(planned.summary.contains(&summary), "{:?}", planned.summary); +} diff --git a/crates/loclib-plugin/Cargo.toml b/crates/loclib-plugin/Cargo.toml index d1dfbce0..f72c1144 100644 --- a/crates/loclib-plugin/Cargo.toml +++ b/crates/loclib-plugin/Cargo.toml @@ -13,7 +13,7 @@ repository.workspace = true [dependencies] indexmap.workspace = true jsonschema.workspace = true -loclib-model.workspace = true +loclib-model = { workspace = true, features = ["schemars"] } loclib-progress = { workspace = true, features = ["schemars"] } rustc-hash.workspace = true schemars = { version = "1.2", default-features = false, features = ["derive", "std", "indexmap2"] } diff --git a/crates/loclib-plugin/README.md b/crates/loclib-plugin/README.md index 1c71bb2c..6b414d7d 100644 --- a/crates/loclib-plugin/README.md +++ b/crates/loclib-plugin/README.md @@ -120,7 +120,7 @@ assert!(manifest.validate(false).is_err()); - `capabilities`: `library:read`, `analysis:read`, `artwork:read`, `drive:info`, `health:read`, `store:read`, `settings:read`, `plays:read`, `sets:read`, `account:status`, `collection:read`, `library:plan`, `library:delete`, `tags:plan`, `cues:plan`, - `artwork:plan`, and `net:`, or `net:*.` for any host below the domain: what + `artwork:plan`, `analysis:plan`, `fs:read`, and `net:`, or `net:*.` for any host below the domain: what `net/fetch` may reach (shown to the user; a native plugin can't be held to it otherwise). Any other is refused, and each is granted once. - `[[net]]`: `host` (one of its `net:` capabilities, as written) and `per_second`, how many @@ -239,6 +239,27 @@ to 512 MiB (in base64), leaving out the rest with a warning. for tracks (`provide/candidates`). It's given their tags, so it needs `library:read`, and never a `:plan` capability for what it finds: the host plans that. +### sources + +`library/read {source, options, roots, cursor?, context}` -> `{"tracks": [...], "playlists": +[{path, tracks}], "blob"?, "next"?}`: a page of a library the plugin reads from somewhere on the +computer (another program's collection, a folder of tagged files), which the host builds a drive +from. `roots` are the folders the user granted it for this read (absolute); the host asks for +pages, passing each one's `next` back as `cursor`, until one has none. A track is `{path, title, +artist, album, album_artist, genre, label, remixer, composer, comment, isrc, year, rating, key?, +bpm?, cues?, beats?, waveform?, my_tags?}`: `path` its audio file (absolute), `key` a Camelot code +or a key as rekordbox names it, `cues` `[{hot_cue?, start_ms, end_ms?, name?, color?}]` (`hot_cue` +0 for A to 7 for H; `color` as cue edits take one), `beats` a beat grid as an analyzer's `beats` +field is, `waveform` a blob as its `waveform` field is, `my_tags` `Category/Tag`. A playlist's +`path` is `Folder/Name`, its `tracks` their `path`s. A page too big for a message is a blob, `{"blob": +"page.json"}`, holding a page as JSON. The host trusts no path: it copies only a regular audio file +in one of `roots` (links resolved), not on the drive written to, reading its length and format +itself and hashing it before copying, and leaves out (with a warning) the tracks that aren't and +what it can't write of the rest; cues, beat grids and waveforms go into analysis files it makes. + +- `[[contributes.sources]]`: `name`, `title` and `options`. It reads files in the folders it's + given, so it needs `fs:read`. + ## Writing a plugin in Rust Register a function for each kind of hook the manifest needs in `Hooks` (`.on(run_command)`, for @@ -318,7 +339,7 @@ served.join().expect("the plugin stops"); # Ok::<(), Box>(()) ``` -`examples/echo` is a whole plugin, with one of each kind of contribution. +`examples/echo` is a whole plugin, with one of each kind of contribution but a library source. ## Licence diff --git a/crates/loclib-plugin/src/features.rs b/crates/loclib-plugin/src/features.rs index 2bab3176..bac8f059 100644 --- a/crates/loclib-plugin/src/features.rs +++ b/crates/loclib-plugin/src/features.rs @@ -29,6 +29,7 @@ pub const FEATURES: &[Feature] = &[ hooks::analyzers::FEATURE, hooks::exporters::FEATURE, hooks::providers::FEATURE, + hooks::sources::FEATURE, // Every hook's `context.scratch`, `{"blob": name}` in answers, and a view's rows in parts. Feature { name: "blobs", methods: &[], sends: &[method::PARTIAL], contributed: |_| false }, Feature { name: "net", methods: &[], sends: &[method::NET_FETCH], contributed: |m| !m.net.is_empty() }, diff --git a/crates/loclib-plugin/src/hooks.rs b/crates/loclib-plugin/src/hooks.rs index 0d7919a4..bfaa1bcb 100644 --- a/crates/loclib-plugin/src/hooks.rs +++ b/crates/loclib-plugin/src/hooks.rs @@ -16,3 +16,4 @@ pub(crate) mod commands; pub(crate) mod exporters; pub(crate) mod operations; pub(crate) mod providers; +pub(crate) mod sources; diff --git a/crates/loclib-plugin/src/hooks/analysis.rs b/crates/loclib-plugin/src/hooks/analysis.rs index 0db0dc84..1667326f 100644 --- a/crates/loclib-plugin/src/hooks/analysis.rs +++ b/crates/loclib-plugin/src/hooks/analysis.rs @@ -7,6 +7,7 @@ //! manifest declaring such a field. use loclib_model::analysis::{Beat, SongStructure}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::manifest::invalid; @@ -22,7 +23,7 @@ pub(crate) const FEATURE: Feature = Feature { /// A `waveform` field's value: a file in the request's scratch folder holding three bytes a /// column, its lows, mids and highs, 0-255 ([`loclib_model::analysis::Bands`] packed), and how /// many columns a second of audio it has. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct WaveformBlob { pub blob: String, pub columns_per_sec: f64, diff --git a/crates/loclib-plugin/src/hooks/sources.rs b/crates/loclib-plugin/src/hooks/sources.rs new file mode 100644 index 00000000..9dda2e43 --- /dev/null +++ b/crates/loclib-plugin/src/hooks/sources.rs @@ -0,0 +1,179 @@ +//! Feature `sources`: a library a plugin reads from somewhere on this computer (another DJ +//! program's collection, a folder of tagged files), which the host builds a drive from: its +//! tracks, each an audio file in a folder the user granted the plugin (`fs:read`), with their tags, +//! cues and beat grids, and its playlists. + +use std::path::PathBuf; + +use loclib_model::analysis::Beat; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::manifest::{invalid, object_schema, shown, unique_names}; +use crate::sdk::declared; +use crate::{Capability, Context, Contributions, Feature, Hook, Manifest, ManifestError, RpcError, WaveformBlob}; + +/// Host to plugin: [`ReadLibrary`] -> [`LibraryPage`]. +pub const READ_LIBRARY: &str = "library/read"; + +pub(crate) const FEATURE: Feature = Feature { + name: "sources", + methods: &[READ_LIBRARY], + sends: &[], + contributed: |m| !m.contributes.sources.is_empty(), +}; + +/// `library/read`: a page of the source's library, the first or the one after `cursor`. The host +/// asks for pages until one has no `next`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct ReadLibrary { + /// One of the manifest's sources. + pub source: String, + /// What the source's `options` schema describes. + #[serde(default)] + pub options: Value, + /// The folders the user granted the plugin for this read (`fs:read`), absolute: each track's + /// audio file is in one of them, or the host leaves the track out. + pub roots: Vec, + /// The `next` of the page before; none for the first. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, + #[serde(default)] + pub context: Context, +} + +/// What `library/read` answers: some of the library's tracks and playlists, in any order across +/// pages (a playlist names its tracks by path, on this page or another). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LibraryPage { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub tracks: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub playlists: Vec, + /// A file in the request's scratch folder holding more of the page, a page as JSON (its own + /// `blob` and `next` are ignored): for a page too big for a message. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub blob: Option, + /// The cursor of the page after this one; none on the last. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub next: Option, +} + +/// A track of a source's library: its audio file and what's known of it, each left out when +/// there's nothing. The host reads the file itself (its length, format and size). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SourceTrack { + /// Its audio file, absolute, in one of the request's `roots`. + pub path: PathBuf, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub title: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub artist: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub album: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub album_artist: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub genre: String, + /// The record label. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub label: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub remixer: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub composer: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub comment: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub isrc: String, + #[serde(default, skip_serializing_if = "is_zero")] + pub year: u16, + /// 0 to 5 stars. + #[serde(default, skip_serializing_if = "is_zero")] + pub rating: u8, + /// A Camelot code (`8A`), or a key as rekordbox names it (`Am`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub key: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bpm: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub cues: Vec, + /// Its beat grid: each beat's number in its bar (1-4), the tempo there and its time. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub beats: Vec, + /// Its waveform, three bands in a file in the request's scratch folder (as an analyzer's + /// `waveform` field gives one). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub waveform: Option, + /// Its My Tags, each `Category/Tag`. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub my_tags: Vec, +} + +fn is_zero(n: &N) -> bool { + *n == N::default() +} + +/// A hot cue, a memory cue or a loop. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SourceCue { + /// Its hot cue's slot, 0 for A to 7 for H; a memory cue without one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hot_cue: Option, + pub start_ms: u32, + /// Where a loop ends. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub end_ms: Option, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + /// As cue edits take a colour: a name (`red`, `aqua`), `#rrggbb` (the nearest players have), + /// or a number (a hot cue's palette code, 1-62; a memory cue's colour, 1 pink to 8 purple). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, +} + +/// A playlist of a source's library. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SourcePlaylist { + /// Its folders and name, `/` between them (`Sets/Friday`): the folders are made for it. + pub path: String, + /// Its tracks in order, by their `path`. + pub tracks: Vec, +} + +impl Hook for ReadLibrary { + const METHOD: &'static str = READ_LIBRARY; + type Answer = LibraryPage; + + fn declared(&self, c: &Contributions) -> Result<(), RpcError> { + declared("source", &self.source, c.sources.iter().map(|x| &x.name)) + } +} + +/// Read with `library/read`, a page at a time, for the host to build a drive from. It's given the +/// folders the user grants it, which takes `fs:read`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct LibrarySource { + pub name: String, + pub title: String, + /// A JSON Schema for its options (an object). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub options: Option, +} + +/// `[[contributes.sources]]`: names, titles and options, and reading the folders granted. +pub(crate) fn validate(m: &Manifest) -> Result<(), ManifestError> { + let (at, c) = ("contributes.sources", &m.contributes); + unique_names(at, "name", c.sources.iter().map(|x| x.name.as_str()).enumerate())?; + for (i, x) in c.sources.iter().enumerate() { + shown(format!("{at}[{i}].title"), &x.title)?; + object_schema(format!("{at}[{i}].options"), x.options.as_ref())?; + } + if !c.sources.is_empty() && !m.capabilities.contains(&Capability::FsRead) { + let message = + format!("reads files in the folders it's granted, which needs the {} capability", Capability::FsRead); + return Err(invalid(format!("{at}[0]"), message)); + } + Ok(()) +} diff --git a/crates/loclib-plugin/src/lib.rs b/crates/loclib-plugin/src/lib.rs index aee07f53..f27537c7 100644 --- a/crates/loclib-plugin/src/lib.rs +++ b/crates/loclib-plugin/src/lib.rs @@ -27,6 +27,7 @@ pub use hooks::operations::{OpAnswer, OpPlan, Operation, PlanOp}; pub use hooks::providers::{ Candidate, Candidates, ProvideCandidates, Provider, ProviderKind, TrackMetadata, TrackTags, }; +pub use hooks::sources::{LibraryPage, LibrarySource, ReadLibrary, SourceCue, SourcePlaylist, SourceTrack}; /// The types [`DriveReads`] answers with (`Track`, `Library`, `TrackAnalysis`), for a plugin to /// name without depending on loclib-model itself. pub use loclib_model; diff --git a/crates/loclib-plugin/src/manifest.rs b/crates/loclib-plugin/src/manifest.rs index 5fcf74f1..2e5b8afc 100644 --- a/crates/loclib-plugin/src/manifest.rs +++ b/crates/loclib-plugin/src/manifest.rs @@ -49,6 +49,7 @@ pub struct Contributions { pub analyzers: Vec, pub exporters: Vec, pub providers: Vec, + pub sources: Vec, pub settings: Vec, } @@ -227,6 +228,9 @@ pub enum Capability { ArtworkPlan, /// Plan writing beat grids, phrases and waveforms into tracks' analysis files. AnalysisPlan, + /// Read files in the folders the user grants it for a read of its library sources: the host + /// copies only audio files in them (links resolved), whatever the plugin says. + FsRead, /// `net:`: it talks to this server through the host (`net/fetch`), or with /// `net:*.` to any server below that domain. Shown to the user; a native plugin can't /// be held to it. @@ -235,7 +239,7 @@ pub enum Capability { impl Capability { /// Every capability but `net:`. - pub const NAMED: [Capability; 17] = [ + pub const NAMED: [Capability; 18] = [ Capability::LibraryRead, Capability::AnalysisRead, Capability::ArtworkRead, @@ -253,6 +257,7 @@ impl Capability { Capability::CuesPlan, Capability::ArtworkPlan, Capability::AnalysisPlan, + Capability::FsRead, ]; /// Its name, unless it's `net:`. @@ -275,6 +280,7 @@ impl Capability { Capability::CuesPlan => "cues:plan", Capability::ArtworkPlan => "artwork:plan", Capability::AnalysisPlan => "analysis:plan", + Capability::FsRead => "fs:read", Capability::Net(_) => return None, }) } @@ -576,6 +582,7 @@ impl Manifest { hooks::analyzers::validate(self)?; hooks::exporters::validate(self)?; hooks::providers::validate(self)?; + hooks::sources::validate(self)?; let at = "contributes.settings"; unique_names(at, "key", c.settings.iter().map(|x| x.key.as_str()).enumerate())?; for (i, x) in c.settings.iter().enumerate() { diff --git a/crates/loclib-plugin/src/manifest/tests.rs b/crates/loclib-plugin/src/manifest/tests.rs index 5b9adba8..ec9101f2 100644 --- a/crates/loclib-plugin/src/manifest/tests.rs +++ b/crates/loclib-plugin/src/manifest/tests.rs @@ -4,7 +4,10 @@ use indexmap::IndexMap; use serde_json::{Map, json}; use super::*; -use crate::{Analyzer, Command, Exporter, FieldType, FieldValue, Mode, Operation, Provider, ProviderKind, Role, code}; +use crate::{ + Analyzer, Command, Exporter, FieldType, FieldValue, LibrarySource, Mode, Operation, Provider, ProviderKind, Role, + code, +}; const HEAD: &str = "id = \"demo\"\nname = \"Demo\"\nversion = \"0.1.0\"\nprotocol = \"^0.2\"\n"; @@ -344,6 +347,20 @@ fn a_provider_is_granted_reading_tracks() { assert!(provider("", "lyrics").contains("unknown variant `lyrics`")); } +/// A library source reads files in the folders the user grants it: it's granted that. +#[test] +fn a_source_is_granted_reading_folders() { + let source = |grants: &str| { + problem(&format!("capabilities = [{grants}]\n[[contributes.sources]]\nname = \"s\"\ntitle = \"S\"\n")) + }; + assert_eq!(source("\"fs:read\""), ""); + let said = "contributes.sources[0]: reads files in the folders it's granted, which needs the fs:read capability"; + assert_eq!(source("\"library:read\""), said); + assert_eq!(source(""), said); + assert_eq!("fs:read".parse::().unwrap(), Capability::FsRead); + assert_eq!(Capability::FsRead.to_string(), "fs:read"); +} + #[test] fn exporters_write_a_file_or_a_folder() { let both = "[[contributes.exporters]]\nformat = \"f\"\ntitle = \"F\"\nextension = \"x\"\nfolder = true\n"; @@ -405,6 +422,7 @@ fn names_and_titles_arent_empty() { ("operations", "name = \"o\""), ("analyzers", "name = \"a\"\nversion = \"1\"\nfields = { x = \"number\" }"), ("exporters", "format = \"f\"\nfolder = true"), + ("sources", "name = \"s\""), ("settings", "key = \"s\"\ntype = \"string\""), ] { let e = problem(&format!("[[contributes.{kind}]]\ntitle = \"\"\n{rest}\n")); @@ -929,6 +947,14 @@ fn manifest(tc: &TestCase, first_party: bool) -> Manifest { if !providers.is_empty() && !capabilities.contains(&Capability::LibraryRead) { capabilities.push(Capability::LibraryRead); } + let sources: Vec = drawn_names(tc, 3) + .into_iter() + .map(|name| LibrarySource { name, title: title(tc), options: maybe_schema(tc) }) + .collect(); + // Granted reading the folders they're given. + if !sources.is_empty() && !capabilities.contains(&Capability::FsRead) { + capabilities.push(Capability::FsRead); + } let hosts: Vec = tc.draw(gs::vecs(gs::from_regex(r"(\*\.)?[A-Za-z0-9.-]{1,16}").fullmatch(true)).max_size(2).unique(true)); let limited = tc.draw(gs::subsequences(hosts.clone())); @@ -949,7 +975,7 @@ fn manifest(tc: &TestCase, first_party: bool) -> Manifest { version: format!("{major}.{minor}.{patch}{pre}"), protocol: tc.draw(gs::sampled_from(protocols)).to_string(), capabilities: tc.draw(gs::permutations(capabilities)), - contributes: Contributions { commands, operations, analyzers, exporters, providers, settings }, + contributes: Contributions { commands, operations, analyzers, exporters, providers, sources, settings }, net, unknown: Vec::new(), } @@ -1013,7 +1039,8 @@ fn a_drawn_manifest_is_valid_and_reads_back_as_it_was_written(tc: TestCase) { } } -const CONTRIBUTIONS: [&str; 6] = ["commands", "operations", "analyzers", "exporters", "providers", "settings"]; +const CONTRIBUTIONS: [&str; 7] = + ["commands", "operations", "analyzers", "exporters", "providers", "sources", "settings"]; /// What names a contribution of `kind`. fn name_key(kind: &str) -> &'static str { @@ -1031,6 +1058,7 @@ fn len(c: &Contributions, kind: &str) -> usize { "analyzers" => c.analyzers.len(), "exporters" => c.exporters.len(), "providers" => c.providers.len(), + "sources" => c.sources.len(), _ => c.settings.len(), } } @@ -1058,6 +1086,8 @@ fn add(c: &mut Contributions, kind: &str, n: usize) { } // Granted library:read by the caller. "providers" => c.providers.push(Provider { name, title, kind: ProviderKind::Metadata }), + // Granted fs:read by the caller. + "sources" => c.sources.push(LibrarySource { name, title, options: None }), _ => c.settings.push(Setting { key: name, title, @@ -1077,6 +1107,7 @@ fn names_and_titles<'a>(c: &'a mut Contributions, kind: &str) -> Vec<(&'a mut St "analyzers" => c.analyzers.iter_mut().map(|x| (&mut x.name, &mut x.title)).collect(), "exporters" => c.exporters.iter_mut().map(|x| (&mut x.format, &mut x.title)).collect(), "providers" => c.providers.iter_mut().map(|x| (&mut x.name, &mut x.title)).collect(), + "sources" => c.sources.iter_mut().map(|x| (&mut x.name, &mut x.title)).collect(), _ => c.settings.iter_mut().map(|x| (&mut x.key, &mut x.title)).collect(), } } @@ -1089,6 +1120,7 @@ fn duplicate(c: &mut Contributions, kind: &str, i: usize) { "analyzers" => c.analyzers.push(c.analyzers[i].clone()), "exporters" => c.exporters.push(c.exporters[i].clone()), "providers" => c.providers.push(c.providers[i].clone()), + "sources" => c.sources.push(c.sources[i].clone()), _ => c.settings.push(c.settings[i].clone()), } } @@ -1098,6 +1130,7 @@ fn schema_of<'a>(c: &'a mut Contributions, kind: &str, i: usize) -> (&'a mut Opt match kind { "commands" => (&mut c.commands[i].params, format!("contributes.commands[{i}].params")), "operations" => (&mut c.operations[i].params, format!("contributes.operations[{i}].params")), + "sources" => (&mut c.sources[i].options, format!("contributes.sources[{i}].options")), _ => (&mut c.exporters[i].options, format!("contributes.exporters[{i}].options")), } } @@ -1147,6 +1180,7 @@ fn each_kind_of_mistake_is_refused_saying_where(tc: TestCase) { "net twice", "net rate", "provider grant", + "source grant", ]; let mistake = tc.draw(gs::sampled_from(mistakes)); let c = &mut m.contributes; @@ -1241,6 +1275,9 @@ fn each_kind_of_mistake_is_refused_saying_where(tc: TestCase) { if kind == "providers" && !m.capabilities.contains(&Capability::LibraryRead) { m.capabilities.push(Capability::LibraryRead); } + if kind == "sources" && !m.capabilities.contains(&Capability::FsRead) { + m.capabilities.push(Capability::FsRead); + } let i = index(&tc, len(c, kind)); let key = name_key(kind); match mistake { @@ -1261,8 +1298,11 @@ fn each_kind_of_mistake_is_refused_saying_where(tc: TestCase) { } } "not an object" | "bad schema" | "param name" | "x-cli" | "positional twice" | "x-path" => { - let kind = tc.draw(gs::sampled_from(vec!["commands", "operations", "exporters"])); + let kind = tc.draw(gs::sampled_from(vec!["commands", "operations", "exporters", "sources"])); add(c, kind, 1); + if kind == "sources" && !m.capabilities.contains(&Capability::FsRead) { + m.capabilities.push(Capability::FsRead); + } let i = index(&tc, len(c, kind)); let (slot, at) = schema_of(c, kind, i); let mut drawn = params(&tc); @@ -1476,6 +1516,13 @@ fn each_kind_of_mistake_is_refused_saying_where(tc: TestCase) { m.capabilities.retain(|g| *g != Capability::LibraryRead); ("contributes.providers[0]: is given tracks' tags, which needs the library:read capability".into(), false) } + "source grant" => { + add(c, "sources", 1); + m.capabilities.retain(|g| *g != Capability::FsRead); + let said = + "contributes.sources[0]: reads files in the folders it's granted, which needs the fs:read capability"; + (said.into(), false) + } _ => { add(c, "settings", 1); let i = index(&tc, c.settings.len()); @@ -1797,6 +1844,7 @@ fn capabilities_read_back_from_their_names_and_nothing_else(tc: TestCase) { "cues:plan", "artwork:plan", "analysis:plan", + "fs:read", ]; let s = match tc.draw(gs::integers::().max_value(3)) { 0 => tc.draw(gs::sampled_from(named.to_vec())).to_string(), diff --git a/crates/loclib-plugin/src/protocol.rs b/crates/loclib-plugin/src/protocol.rs index 614a94cf..5dca2827 100644 --- a/crates/loclib-plugin/src/protocol.rs +++ b/crates/loclib-plugin/src/protocol.rs @@ -23,6 +23,7 @@ pub mod method { pub use crate::hooks::exporters::WRITE_EXPORT; pub use crate::hooks::operations::PLAN_OP; pub use crate::hooks::providers::PROVIDE; + pub use crate::hooks::sources::READ_LIBRARY; /// Host to plugin, first: [`super::Hello`] -> [`super::HelloReply`]. pub const HELLO: &str = "hello"; diff --git a/crates/loclib-plugin/src/sdk/tests.rs b/crates/loclib-plugin/src/sdk/tests.rs index 08b25d6a..10972898 100644 --- a/crates/loclib-plugin/src/sdk/tests.rs +++ b/crates/loclib-plugin/src/sdk/tests.rs @@ -11,8 +11,8 @@ use serde_json::json; use super::*; use crate::{ - Analysis, AnalyzeTrack, Candidate, Candidates, Exported, ExportedFile, FieldValue, OpAnswer, OpPlan, Pending, - PlanOp, ProvideCandidates, RunCommand, View, WriteExport, code, + Analysis, AnalyzeTrack, Candidate, Candidates, Exported, ExportedFile, FieldValue, LibraryPage, OpAnswer, OpPlan, + Pending, PlanOp, ProvideCandidates, ReadLibrary, RunCommand, SourceTrack, View, WriteExport, code, }; #[path = "../../examples/echo/plugin.rs"] @@ -300,6 +300,38 @@ fn hooks_run_only_for_what_the_manifest_declares() { s.stop(); } +/// A library source answers its library a page at a time, from the folders it's granted, and +/// only for a source the manifest declares. +#[test] +fn a_source_answers_its_library_a_page_at_a_time() { + let granted = echo::MANIFEST.replace(r#""library:plan"]"#, r#""library:plan", "fs:read"]"#); + let manifest = format!("{granted}\n[[contributes.sources]]\nname = \"files\"\ntitle = \"Files\"\n"); + assert_eq!(Manifest::parse(&manifest).unwrap().validate(false).map_err(|e| e.to_string()), Ok(())); + let read = |req: ReadLibrary, _: &Host| { + let page: usize = req.cursor.as_deref().map_or(Ok(0), str::parse).map_err(|_| RpcError::invalid_params("x"))?; + let track = SourceTrack { path: req.roots[0].join(format!("{page}.mp3")), ..SourceTrack::default() }; + Ok(LibraryPage { tracks: vec![track], next: (page == 0).then(|| "1".into()), ..LibraryPage::default() }) + }; + let s = start_with(manifest, Hooks::new().on(read)); + let reply: HelloReply = s.host.request(method::HELLO, &hello(PROTOCOL_VERSION, &[])).unwrap(); + assert_eq!(reply.features, ["analysis", "sources", "blobs", "net", "choose"]); + let mut paths = Vec::new(); + let mut cursor: Option = None; + loop { + let req = json!({ "source": "files", "roots": ["/music"], "cursor": cursor }); + let page: LibraryPage = s.host.request(method::READ_LIBRARY, &req).unwrap(); + paths.extend(page.tracks.into_iter().map(|t| t.path)); + cursor = page.next; + if cursor.is_none() { + break; + } + } + assert_eq!(paths, [Path::new("/music/0.mp3"), Path::new("/music/1.mp3")]); + let e = s.call(method::READ_LIBRARY, json!({ "source": "nope", "roots": [] })).unwrap_err(); + assert_eq!(e.message, "invalid params: the manifest declares no source \"nope\""); + s.stop(); +} + /// A plugin has the features of the hooks it implements (and the SDK's own), and a hook it /// doesn't implement isn't found. #[test] From 1a5059c50c9ab7434ee2d4a814d60da40a384a0a Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:56:28 -0400 Subject: [PATCH 2/3] plugins: the sources test expects a file on the drive written to with the system's separator The host names a file it finds in a granted folder with the system's separator, so on Windows the warning says stick\00.wav; the test built stick/00.wav and failed there. --- crates/locallibrary/tests/it/plugins/sources.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/locallibrary/tests/it/plugins/sources.rs b/crates/locallibrary/tests/it/plugins/sources.rs index 22f96aa3..09e4d21a 100644 --- a/crates/locallibrary/tests/it/plugins/sources.rs +++ b/crates/locallibrary/tests/it/plugins/sources.rs @@ -419,7 +419,8 @@ fn every_track_left_out_is_counted_with_why(tc: TestCase) { } Given::NotAudio => in_music(&format!("{i:02}.mp3"), b"text", &mut found), Given::Text => in_music(&format!("{i:02}.txt"), b"text", &mut found), - Given::OnDrive => in_music(&format!("stick/{name}"), &wav(1.0), &mut found), + // The host names a file it finds with this system's separator. + Given::OnDrive => in_music(&format!("stick{}{name}", std::path::MAIN_SEPARATOR), &wav(1.0), &mut found), Given::Outside => { put(&outside.join(&name), &wav(1.0), None); told.push((outside.join(&name).display().to_string(), given.why())); From 73b112fb73f34335a8bb6789ea2575fc055cdeae Mon Sep 17 00:00:00 2001 From: botandbang <337765928+botandbang@users.noreply.github.com> Date: Thu, 22 Oct 2026 10:28:11 -0400 Subject: [PATCH 3/3] locallibrary: the account command and analysis folders' counter are named for what they are, so CodeQL takes neither for an account's data or a salt --- apps/locallibrary/src/main.rs | 5 +++-- crates/locallibrary/src/collection_export.rs | 8 ++++---- crates/locallibrary/src/merge.rs | 12 ++++++------ 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/locallibrary/src/main.rs b/apps/locallibrary/src/main.rs index 61ce9dcf..857c25d8 100644 --- a/apps/locallibrary/src/main.rs +++ b/apps/locallibrary/src/main.rs @@ -406,7 +406,8 @@ locallibrary pack Home /Volumes/TONIGHT --playlist Friday --transcode --players Profile(profile_cmd::ProfileArgs), /// Sign in with your Atmosphere account (the one you use for Bluesky and other AT Protocol apps), or sign out (alpha) #[cfg(feature = "account")] - Account(account_cmd::AccountArgs), + #[command(name = "account")] + Atmosphere(account_cmd::AccountArgs), /// Report a bug in locallibrary, with this version and system filled in #[command(after_help = BUG_HELP)] Bug(bug_cmd::BugArgs), @@ -1481,7 +1482,7 @@ fn run(cli: Cli, out: Output, contributed: Option) -> R Command::History(args) => return plays_cmd::history(args, &cli.store, out, &ctx), Command::Stats(args) => return plays_cmd::stats(args, &cli.store, out), #[cfg(feature = "account")] - Command::Account(args) => return account_cmd::run(args, &cli.store, out, &ctx), + Command::Atmosphere(args) => return account_cmd::run(args, &cli.store, out, &ctx), Command::Profile(args) => return profile_cmd::run(args, &cli.store, out, &ctx), Command::Export(args) => return export_cmd::run(args, &cli.store, out, &ctx), Command::Import(args) => return plugin_cmd::import(args, &cli.store, out, &ctx), diff --git a/crates/locallibrary/src/collection_export.rs b/crates/locallibrary/src/collection_export.rs index e713cea0..76a6b93e 100644 --- a/crates/locallibrary/src/collection_export.rs +++ b/crates/locallibrary/src/collection_export.rs @@ -549,11 +549,11 @@ impl Layout { /// A folder no other track's analysis files have, for those of the track at `device_path`. fn analysis_dir(&self, device_path: &str) -> String { - let mut salt = 0; - let mut dir = merge::analysis_dir(device_path, salt); + let mut attempt = 0; + let mut dir = merge::analysis_dir(device_path, attempt); while self.taken.contains(&format!("{dir}/anlz0000.dat").to_lowercase()) { - salt += 1; - dir = merge::analysis_dir(device_path, salt); + attempt += 1; + dir = merge::analysis_dir(device_path, attempt); } dir } diff --git a/crates/locallibrary/src/merge.rs b/crates/locallibrary/src/merge.rs index 350e43b0..90c6120d 100644 --- a/crates/locallibrary/src/merge.rs +++ b/crates/locallibrary/src/merge.rs @@ -612,10 +612,10 @@ fn plan_analysis( let dir_taken = |d: &str, taken: &FxHashSet| siblings.iter().any(|s| taken.contains(&fold(&s.replacen(dir, d, 1)))); let mut target_dir = dir.to_string(); - let mut salt = 0u32; + let mut attempt = 0u32; while dir_taken(&target_dir, taken) { - target_dir = analysis_dir(target_audio, salt); - salt += 1; + target_dir = analysis_dir(target_audio, attempt); + attempt += 1; } let pairs: Vec<(String, String)> = siblings.iter().map(|s| (s.to_string(), s.replacen(dir, &target_dir, 1))).collect(); @@ -627,9 +627,9 @@ fn plan_analysis( } /// An analysis folder for the audio file `audio` (a key), as rekordbox lays them out -/// (`PIONEER/USBANLZ/Pxxx/xxxxxxxx`): from a hash of its path, another for each `salt`. -pub(crate) fn analysis_dir(audio: &str, salt: u32) -> String { - let h = Hash::of(format!("{audio}\0{salt}").as_bytes()); +/// (`PIONEER/USBANLZ/Pxxx/xxxxxxxx`): from a hash of its path, another for each `attempt`. +pub(crate) fn analysis_dir(audio: &str, attempt: u32) -> String { + let h = Hash::of(format!("{audio}\0{attempt}").as_bytes()); let b = h.as_bytes(); let p = u16::from_be_bytes([b[0], b[1]]) & 0x0fff; let q = u32::from_be_bytes([b[2], b[3], b[4], b[5]]);