From 16d46c96654f1ad3829b0e4b242ed4abfdd76d81 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Wed, 14 Oct 2026 18:07:55 +0200 Subject: [PATCH 1/3] test(privacy): reproduce private feed redaction follow-ups --- src/createPodcastNote.privacy.test.ts | 46 +++++++++++++++ src/services/privateFeeds.test.ts | 21 +++++++ src/settingsTransfer.test.ts | 32 +++++++++++ src/utility/privateFeedUrl.test.ts | 83 +++++++++++++++++++++++++++ tests/e2e/private-notes.test.ts | 76 +++++++++++++++++++++++- 5 files changed, 257 insertions(+), 1 deletion(-) diff --git a/src/createPodcastNote.privacy.test.ts b/src/createPodcastNote.privacy.test.ts index d49a4370..d456b7b0 100644 --- a/src/createPodcastNote.privacy.test.ts +++ b/src/createPodcastNote.privacy.test.ts @@ -92,6 +92,52 @@ test("an unsubscribed private episode writes no feed address and keeps its live expect(episode.feedUrl).toBe(address); }); +test("the default note omits an empty resume label and keeps public resume links", () => { + noteVault(false); + const publicEpisode = { ...episode, feedUrl: "https://public.example.com/rss" }; + expect(NoteTemplateEngine(DEFAULT_SETTINGS.note.template, publicEpisode)).toContain( + `[Resume in PodNotes](${buildEpisodeResumeLink(publicEpisode)})`, + ); + expect(NoteTemplateEngine(DEFAULT_SETTINGS.note.template, episode)).not.toContain( + "Resume in PodNotes", + ); +}); + +test("an unsubscribed feed note names its safe placeholder and leaves a public feed alone", () => { + noteVault(false); + const feed = { + title: "Paid", + url: address, + artworkUrl: "", + description: `Your feed: ${address}.`, + }; + const publicUrl = "https://public.example.com/rss"; + expect( + FeedNoteTemplateEngine("{{description}}", { + ...feed, + url: publicUrl, + description: `Your feed: ${publicUrl}.`, + }), + ).toBe(`Your feed: ${publicUrl}.`); + expect(FeedNoteTemplateEngine("{{description}}", feed)).toBe( + "Your feed: podnotes-private-feed:Paid.", + ); +}); + +test.each([address, address.replace(":", ":"), encodeURIComponent(address)])( + "an unsubscribed feed placeholder never reflects its private address from its name: %s", + (title) => { + noteVault(false); + const rendered = FeedNoteTemplateEngine("{{description}}", { + title, + url: address, + artworkUrl: "", + description: `Your feed: ${address}.`, + }); + expect(rendered).toBe("Your feed: podnotes-private-feed:."); + }, +); + test.each([ address, "https://user:API_SENTINEL@private.example.com/rss", diff --git a/src/services/privateFeeds.test.ts b/src/services/privateFeeds.test.ts index d55bc373..5c51cf27 100644 --- a/src/services/privateFeeds.test.ts +++ b/src/services/privateFeeds.test.ts @@ -164,6 +164,27 @@ function episode(title: string, feedUrl: string) { } describe("protectPrivateFeeds", () => { + it("redacts entity-escaped descriptions before persistence without changing public bytes", () => { + const { feedUrls } = repository(); + const address = "https://private.example.com/rss?auth=PERSIST_ENTITY_SENTINEL"; + const escaped = address.replace(":", ":").replaceAll("/", "/"); + const publicText = "Public https://private.example.com/public?a=1&b=2"; + const input = settings({ + savedFeeds: { + Paid: feed({ + title: "Paid", + url: address, + description: `${escaped}. ${publicText}`, + }), + Public: feed({ description: publicText }), + }, + }); + const { settings: safe } = protectPrivateFeeds(input, feedUrls); + expect(safe.savedFeeds.Public).toEqual(input.savedFeeds.Public); + expect(safe.savedFeeds.Paid.description).toBe(`podnotes-private-feed:Paid. ${publicText}`); + expect(JSON.stringify(encodePodNotesData(safe))).not.toContain("PERSIST_ENTITY_SENTINEL"); + }); + it("moves only credential-bearing feeds", () => { const { feedUrls, values } = repository(); const { settings: safe, failed } = protectPrivateFeeds( diff --git a/src/settingsTransfer.test.ts b/src/settingsTransfer.test.ts index ea3ce558..234ef9b6 100644 --- a/src/settingsTransfer.test.ts +++ b/src/settingsTransfer.test.ts @@ -18,6 +18,38 @@ function makeSettings(overrides: Partial = {}): IPodNotesSett const NOW = "2026-06-14T00:00:00.000Z"; describe("private feed reference transfer", () => { + it("redacts entity-escaped descriptions at export without changing public bytes", () => { + const address = "https://private.example.com/rss?auth=EXPORT_ENTITY_SENTINEL"; + const publicText = "Public https://private.example.com/public?a=1&b=2"; + const settings = makeSettings({ + savedFeeds: { + Paid: { + title: "Paid", + url: "podnotes-private-feed:Paid", + artworkUrl: "", + description: `${address.replace(":", ":").replaceAll("/", "/")}. ${publicText}`, + }, + Public: { + title: "Public", + url: "https://public.example/rss", + artworkUrl: "", + description: publicText, + }, + }, + }); + const { settings: exported } = serializeSettings( + settings, + {}, + "2.21.1", + NOW, + new Map([[address, "podnotes-private-feed:Paid"]]), + ); + expect(exported.savedFeeds?.Public).toEqual(settings.savedFeeds.Public); + expect(exported.savedFeeds?.Paid.description).toBe( + `podnotes-private-feed:Paid. ${publicText}`, + ); + }); + it("puts no private feed address in a settings export, including quoted episode text", () => { const address = "https://private.example/feed?auth=EXPORT_SENTINEL"; const placeholder = "podnotes-private-feed:Private"; diff --git a/src/utility/privateFeedUrl.test.ts b/src/utility/privateFeedUrl.test.ts index 2a20a0bc..31e6db38 100644 --- a/src/utility/privateFeedUrl.test.ts +++ b/src/utility/privateFeedUrl.test.ts @@ -125,6 +125,89 @@ describe("isCredentialBearingUrl for publishers that put the secret in the addre }); describe("redactPrivateFeedAddresses", () => { + it.each([ + ["feed scheme", "feed://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["podcast scheme", "podcast://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["HTTP", "http://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["scheme-less", "private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["entity escaped", "https://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["named entities", "https://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["entity in hostname", "https://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + [ + "encoded query", + "https://redirect.example/?url=https%3A%2F%2Fprivate.example.com%2Frss%3Fauth%3DFOLLOWUP_SENTINEL", + ], + ])("redacts the %s alias while preserving public text byte for byte", (_, alias) => { + const known = new Map([ + [ + "https://private.example.com/rss?auth=FOLLOWUP_SENTINEL", + "podnotes-private-feed:Paid", + ], + ]); + const publicText = alias + .replace("rss?auth=FOLLOWUP_SENTINEL", "public?show=1") + .replace("rss%3Fauth%3DFOLLOWUP_SENTINEL", "public%3Fshow%3D1"); + const media = "https://private.example.com/audio.mp3?token=MEDIA_SIGNATURE"; + const document = { quoted: `Your feed: ${alias}.`, publicText, media }; + const safe = redactPrivateFeedAddresses(document, known); + expect(safe.publicText).toBe(publicText); + expect(safe.media).toBe(media); + expect(safe.quoted).not.toContain("FOLLOWUP_SENTINEL"); + expect(safe.quoted).toContain("podnotes-private-feed"); + expect(safe.quoted.endsWith(".")).toBe(true); + }); + + it("redacts entity-encoded secret characters without decoding surrounding public markup", () => { + const address = "https://private.example.com/rss?auth=FOLLOWUP_SENTINEL"; + const publicText = + "<b>Public</b> & https://private.example.com/audio.mp3?token=MEDIA_SIGNATURE"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const text = `${publicText} ${address.replace("_", "_")}.`; + expect(redactPrivateFeedAddresses(text, known)).toBe( + `${publicText} podnotes-private-feed:Paid.`, + ); + }); + + it("preserves sentence punctuation after a host-only private address", () => { + const known = new Map([["https://private.example.com", "podnotes-private-feed:Paid"]]); + expect(redactPrivateFeedAddresses("Visit https://public.example.com.", known)).toBe( + "Visit https://public.example.com.", + ); + expect(redactPrivateFeedAddresses("Visit https://private.example.com.", known)).toBe( + "Visit podnotes-private-feed:Paid.", + ); + }); + + it("preserves the outer query bytes when its encoded private feed value is replaced", () => { + const address = "https://private.example.com/rss?auth=NESTED_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const outer = `https://redirect.example/?url=${encodeURIComponent(address)}&keep=a%2fb+z`; + expect(redactPrivateFeedAddresses(outer, known)).toBe( + "https://redirect.example/?url=podnotes-private-feed%3APaid&keep=a%2fb+z", + ); + }); + + it("keeps public path escapes and host-name suffixes distinct from a private address", () => { + const address = "https://private.example.com/rss/private?auth=PATH_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const publicUrls = [ + "https://private.example.com/rss%2Fprivate?auth=PATH_SENTINEL", + "https://notprivate.example.com/rss/private?auth=PATH_SENTINEL", + "https://public.example/path/private.example.com/rss/private?auth=PATH_SENTINEL", + ]; + expect(redactPrivateFeedAddresses(publicUrls, known)).toBe(publicUrls); + }); + + it("redacts a private feed with literal path escapes without changing the distinct decoded path", () => { + const address = "https://private.example.com/rss%2Fprivate?auth=PATH_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + expect(redactPrivateFeedAddresses(`Feed ${address}.`, known)).toBe( + "Feed podnotes-private-feed:Paid.", + ); + const publicUrl = "https://private.example.com/rss/private?auth=PATH_SENTINEL"; + expect(redactPrivateFeedAddresses(publicUrl, known)).toBe(publicUrl); + }); + it.each([ "https://private.example.com./rss?auth=ALIAS_SENTINEL", "https:\\\\private.example.com/rss?auth=ALIAS_SENTINEL", diff --git a/tests/e2e/private-notes.test.ts b/tests/e2e/private-notes.test.ts index b65597c8..548b2e81 100644 --- a/tests/e2e/private-notes.test.ts +++ b/tests/e2e/private-notes.test.ts @@ -1,4 +1,7 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; import { expect, onTestFinished, test } from "vitest"; +import { DEFAULT_SETTINGS } from "../../src/constants"; import { SUPERCAST_FIXTURE_HOST } from "../../scripts/e2e-instance.mjs"; import { createPodNotesE2EHarness } from "./harness"; import { @@ -24,6 +27,72 @@ import { import { jsLiteral, RELOAD_OPTIONS } from "./runtime"; const getContext = createPodNotesE2EHarness("private-notes"); +const evidence = path.resolve(".obsidian-e2e-artifacts/private-notes"); + +test("entity-escaped private descriptions stay out of data.json and the settings export", async () => { + const context = getContext(); + const { obsidian, plugin, sandbox } = context; + const episode = { title: "Entity episode", seconds: 60, description: "" }; + const show: FixtureFeed = { + title: "Entity Private", + slug: "entity-private", + host: SUPERCAST_FIXTURE_HOST, + path: "/feeds/PRIV5_ENTITY_SENTINEL/feed.xml", + artwork: false, + episodes: [episode], + }; + const server = await startFixtureServer([show]); + onTestFinished(() => server.stop()); + const address = server.feedUrl(show); + const encoded = address.replaceAll(":", ":").replaceAll("/", "/"); + const publicText = "Public https://priv3.supercast.com/public?x=1&y=2."; + episode.description = `Your feed: ${encoded}. ${publicText}`; + server.retitle(show, episode, episode.title); + await seedLibrary(context, server, { noteTemplate: DEFAULT_SETTINGS.note.template }); + await showPodNotes(obsidian); + await showAddPodcast(obsidian); + await typeInto(obsidian, ui.podcastSearch, address); + await click(obsidian, ui.addPodcast(show.title)); + await waitForLibrary(plugin, (library) => library.subscriptions.length === 1, "subscribed"); + const secretId = await obsidian.dev.evalJson( + 'app.plugins.plugins.podnotes.settings.savedFeeds["Entity Private"].urlSecretId', + ); + onTestFinished(() => + obsidian.dev.evalJson(`app.secretStorage.setSecret(${jsLiteral(secretId)}, ""), true`), + ); + await showLibrary(obsidian); + await click(obsidian, ui.podcastCard(show.title)); + await click(obsidian, ui.episodeMenu(episode.title)); + await click(obsidian, ui.menuItem("Add to Favorites")); + await click(obsidian, ui.episode(episode.title)); + await waitForPlayback(obsidian, (state) => state.playing, "playing"); + await click(obsidian, ui.playbackToggle); + await runCommand(obsidian, "podnotes:create-podcast-note"); + const notePath = `PodNotes/${show.title}/${episode.title}.md`; + await sandbox.waitForExists(notePath, { timeoutMs: 10_000 }); + await obsidian.dev.evalJsonAsync( + "(async () => { await app.plugins.plugins.podnotes.flushSaves(); return true; })()", + ); + const persisted = JSON.stringify(await plugin.data().read()); + await openPodNotesSettings(obsidian); + const name = "entity-settings.json"; + await typeInto(obsidian, ui.settingControl("Export settings", "input"), sandbox.path(name), { + replace: true, + }); + await click(obsidian, ui.settingControl("Export settings", "button")); + await sandbox.waitForExists(name, { timeoutMs: 10_000 }); + await closeSettings(obsidian); + const exported = await readVaultText(obsidian, sandbox.path(name)); + const note = await readVaultText(obsidian, sandbox.path(notePath)); + await mkdir(evidence, { recursive: true }); + for (const [name, text] of Object.entries({ persisted, exported, note })) { + await writeFile(path.join(evidence, `${name}.txt`), text); + expect.soft(text, name).not.toContain("PRIV5_ENTITY_SENTINEL"); + expect.soft(text, name).toContain("podnotes-private-feed:Entity%20Private"); + } + expect(persisted).toContain(publicText); + expect(exported).toContain(publicText); +}); test("private descriptions and aliases stay out of cold and warm-cache episode notes", async () => { const context = getContext(); @@ -153,7 +222,7 @@ test("an unsubscribed private link plays, creates a secret-free note and capture }; const server = await startFixtureServer([show]); onTestFinished(() => server.stop()); - await seedLibrary(context, server); + await seedLibrary(context, server, { noteTemplate: DEFAULT_SETTINGS.note.template }); const address = server.feedUrl(show); const uri = `obsidian://podnotes?episodeName=${encodeURIComponent(episode.title)}&url=${encodeURIComponent(address)}&time=12`; const legacyPath = sandbox.path("legacy-link.md"); @@ -179,9 +248,14 @@ test("an unsubscribed private link plays, creates a secret-free note and capture { timeoutMs: 10_000, intervalMs: 100, message: "plain timestamp and segment saved" }, ); const note = await readVaultText(obsidian, sandbox.path(notePath)); + await openInReadingView(obsidian, sandbox.path(notePath)); + await mkdir(evidence, { recursive: true }); + await writeFile(path.join(evidence, "unsubscribed-note.md"), note); + await obsidian.dev.screenshot(path.join(evidence, "unsubscribed-note.png")); expect(note).toContain("# Unsubscribed episode"); expect(note).not.toContain("UNSUBSCRIBED_NOTE_SENTINEL"); expect(note).not.toContain("obsidian://podnotes"); + expect(note).not.toContain("Resume in PodNotes"); expect(note).not.toMatch(/^feedUrl:/m); expect( await obsidian.dev.evalJson("app.plugins.plugins.podnotes.api.podcast.feedUrl"), From 949b554259e78b9a0fede35039ad731e94665649 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Wed, 14 Oct 2026 19:49:52 +0200 Subject: [PATCH 2/3] fix(privacy): protect escaped feed addresses and note placeholders --- obsidian-e2e.config.mjs | 2 +- src/TemplateEngine.ts | 27 +++- src/constants.ts | 2 +- src/createPodcastNote.privacy.test.ts | 44 +++++- src/services/privateFeeds.test.ts | 8 +- src/services/publishedTranscript/parse.ts | 14 +- src/utility/htmlEntities.ts | 8 + src/utility/privateFeedUrl.test.ts | 120 ++++++++++++++- src/utility/privateFeedUrl.ts | 177 +++++++++++++++++----- tests/e2e/private-notes.test.ts | 8 +- 10 files changed, 344 insertions(+), 66 deletions(-) create mode 100644 src/utility/htmlEntities.ts diff --git a/obsidian-e2e.config.mjs b/obsidian-e2e.config.mjs index f40a6327..6086ac8b 100644 --- a/obsidian-e2e.config.mjs +++ b/obsidian-e2e.config.mjs @@ -80,7 +80,7 @@ export default { "---\n" + "# {{title}}\n\n" + "![]({{artwork}})\n\n" + - "[Resume in PodNotes]({{episodelink}})\n\n" + + "{{episodelink:Resume in PodNotes}}\n\n" + "{{url}}\n\n" + "{{description}}\n", }, diff --git a/src/TemplateEngine.ts b/src/TemplateEngine.ts index 8d498592..ecc6363a 100644 --- a/src/TemplateEngine.ts +++ b/src/TemplateEngine.ts @@ -47,6 +47,18 @@ export function templateHasTag(template: string, tag: Lowercase): boolea ); } +// A title names the placeholder only when it cannot spell the feed's address: +// it holds no character reference or percent escape, and redaction leaves it +// alone as text, where trailing punctuation cannot pass an address off as a +// distinct whole-field URL. +function feedRedactionContext(feed: PodcastFeed) { + const title = ` ${feed.title} `; + const named = + !/&#|&\w+;|%[\da-f]{2}/i.test(title) && + withoutPrivateFeedAddresses([title, { feedUrl: feed.url }] as const)[0] === title; + return { feedUrl: feed.url, podcastName: named ? feed.title : "" }; +} + function withoutFeedAddresses(feed: PodcastFeed): PodcastFeed { return withoutPrivateFeedAddresses([feed, { feedUrl: feed.url }] as const)[0]; } @@ -373,7 +385,10 @@ export function NoteTemplateEngine( // never been played). The resume point is resolved at click time, not baked // in here, so a single templated link always jumps to where you left off. // Empty when the episode has no feed/file URL to address it by. See issue #35. - addTag("episodelink", buildEpisodeResumeLink(liveEpisode)); + addTag("episodelink", (label?: string) => { + const link = buildEpisodeResumeLink(liveEpisode); + return label === undefined || !link ? link : `[${escapeMarkdownText(label)}](${link})`; + }); return replacer(template); } @@ -487,7 +502,8 @@ export function TranscriptTemplateEngine( export function FeedNoteTemplateEngine(template: string, liveFeed: PodcastFeed) { const [replacer, addTag] = useTemplateEngine(); - const feed = withoutFeedAddresses(liveFeed); + const context = feedRedactionContext(liveFeed); + const [feed] = withoutPrivateFeedAddresses([liveFeed, context] as const); const safeTitle = replaceIllegalFileNameCharactersInString(feed.title); @@ -507,9 +523,10 @@ export function FeedNoteTemplateEngine(template: string, liveFeed: PodcastFeed) addTag("feedartwork", sanitizePortableResourceUrl(feed.artworkUrl ?? "")); addTag("author", escapeMarkdownBodyText(feed.author ?? "")); addTag("description", (prependToLines?: string) => { - const sanitizeDescription = feedHtmlToMarkdown(feed.description ?? "", { - feedUrl: liveFeed.url, - }).replace(/\n{3,}/g, "\n\n"); + const sanitizeDescription = feedHtmlToMarkdown(feed.description ?? "", context).replace( + /\n{3,}/g, + "\n\n", + ); if (prependToLines) { return sanitizeDescription .split("\n") diff --git a/src/constants.ts b/src/constants.ts index 028ef384..cdd1cdb8 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -115,7 +115,7 @@ export const DEFAULT_SETTINGS: IPodNotesSettings = { "---\n" + "# {{title}}\n\n" + "![]({{artwork}})\n\n" + - "[Resume in PodNotes]({{episodelink}})\n\n" + + "{{episodelink:Resume in PodNotes}}\n\n" + "{{url}}\n\n" + "{{description}}\n", }, diff --git a/src/createPodcastNote.privacy.test.ts b/src/createPodcastNote.privacy.test.ts index d456b7b0..11e720e5 100644 --- a/src/createPodcastNote.privacy.test.ts +++ b/src/createPodcastNote.privacy.test.ts @@ -103,6 +103,19 @@ test("the default note omits an empty resume label and keeps public resume links ); }); +test("a saved template keeps its resume line, and a resume label is plain text", () => { + noteVault(false); + const saved = "[Resume in PodNotes]({{episodelink}})"; + const publicEpisode = { ...episode, feedUrl: "https://public.example.com/rss" }; + const link = + "obsidian://podnotes?episodeName=Private%20episode&url=https%3A%2F%2Fpublic.example.com%2Frss&guid=episode-1"; + expect(NoteTemplateEngine(saved, episode)).toBe("[Resume in PodNotes]()"); + expect(NoteTemplateEngine(saved, publicEpisode)).toBe(`[Resume in PodNotes](${link})`); + expect(NoteTemplateEngine("{{episodelink:Resume [here]}}", publicEpisode)).toBe( + `[Resume \\[here\\]](${link})`, + ); +}); + test("an unsubscribed feed note names its safe placeholder and leaves a public feed alone", () => { noteVault(false); const feed = { @@ -122,9 +135,20 @@ test("an unsubscribed feed note names its safe placeholder and leaves a public f expect(FeedNoteTemplateEngine("{{description}}", feed)).toBe( "Your feed: podnotes-private-feed:Paid.", ); + const doubleEscaped = `Your feed: ${address.replace(":", "&#58;")}.`; + expect(FeedNoteTemplateEngine("{{description}}", { ...feed, description: doubleEscaped })).toBe( + "Your feed: podnotes-private-feed:Paid.", + ); }); -test.each([address, address.replace(":", ":"), encodeURIComponent(address)])( +test.each([ + address, + `${address}.`, + address.replace(":", ":"), + address.replace(":", "&#58;"), + address.replace(":", "%253A"), + encodeURIComponent(address), +])( "an unsubscribed feed placeholder never reflects its private address from its name: %s", (title) => { noteVault(false); @@ -138,6 +162,20 @@ test.each([address, address.replace(":", ":"), encodeURIComponent(address)]) }, ); +test.each(["Paid", "Q&A", "99% Invisible"])( + "an unsubscribed feed placeholder takes the plain title %s as its name", + (title) => { + noteVault(false); + const rendered = FeedNoteTemplateEngine("{{description}}", { + title, + url: address, + artworkUrl: "", + description: `Your feed: ${address}.`, + }); + expect(rendered).toBe(`Your feed: podnotes-private-feed:${encodeURIComponent(title)}.`); + }, +); + test.each([ address, "https://user:API_SENTINEL@private.example.com/rss", @@ -162,7 +200,7 @@ test.each([ await createPodcastNoteFileIfNotExists(quoted); expect(written).toHaveLength(1); expect(written[0].path).toBe(path); - expect(written[0].content).toContain("[Resume in PodNotes]()"); + expect(written[0].content).not.toContain("Resume in PodNotes"); expect(written[0].content).not.toMatch(/^(guid|feedUrl):/m); expect(written[0].content).not.toContain("NOTE_SENTINEL"); }, @@ -228,7 +266,7 @@ test("a feed note for an unsubscribed private feed writes no feed address", () = description: `Your feed: ${address}.`, }; expect(FeedNoteTemplateEngine("{{feedurl}}|{{description}}", feed)).toBe( - "|Your feed: podnotes-private-feed:.", + "|Your feed: podnotes-private-feed:Paid.", ); expect(FeedFilePathTemplateEngine("{{title}}", { ...feed, title: `Paid ${address}` })).toBe( "Paid podnotes-private-feed", diff --git a/src/services/privateFeeds.test.ts b/src/services/privateFeeds.test.ts index 5c51cf27..2997880c 100644 --- a/src/services/privateFeeds.test.ts +++ b/src/services/privateFeeds.test.ts @@ -169,20 +169,24 @@ describe("protectPrivateFeeds", () => { const address = "https://private.example.com/rss?auth=PERSIST_ENTITY_SENTINEL"; const escaped = address.replace(":", ":").replaceAll("/", "/"); const publicText = "Public https://private.example.com/public?a=1&b=2"; + const nested = `https://redirect.example/?u=${encodeURIComponent(address)}&x=1`; const input = settings({ savedFeeds: { Paid: feed({ title: "Paid", url: address, - description: `${escaped}. ${publicText}`, + description: `${escaped}. ${nested} ${publicText}`, }), Public: feed({ description: publicText }), }, }); const { settings: safe } = protectPrivateFeeds(input, feedUrls); expect(safe.savedFeeds.Public).toEqual(input.savedFeeds.Public); - expect(safe.savedFeeds.Paid.description).toBe(`podnotes-private-feed:Paid. ${publicText}`); + expect(safe.savedFeeds.Paid.description).toBe( + `podnotes-private-feed:Paid. https://redirect.example/?u=podnotes-private-feed%3APaid&x=1 ${publicText}`, + ); expect(JSON.stringify(encodePodNotesData(safe))).not.toContain("PERSIST_ENTITY_SENTINEL"); + expect(protectPrivateFeeds(safe, feedUrls).settings).toBe(safe); }); it("moves only credential-bearing feeds", () => { diff --git a/src/services/publishedTranscript/parse.ts b/src/services/publishedTranscript/parse.ts index 428538b1..932800d3 100644 --- a/src/services/publishedTranscript/parse.ts +++ b/src/services/publishedTranscript/parse.ts @@ -1,4 +1,5 @@ import { isPlainObject } from "src/persistence/codecUtils"; +import { COMMON_NAMED_ENTITIES } from "src/utility/htmlEntities"; export type TranscriptFormat = "vtt" | "srt" | "json" | "html" | "text"; @@ -165,16 +166,9 @@ function toSeconds(value: unknown): number | undefined { return Number.isFinite(seconds) && seconds >= 0 ? seconds : undefined; } -const NAMED_ENTITIES = new Map([ - ["amp", "&"], - ["lt", "<"], - ["gt", ">"], - ["quot", '"'], - ["apos", "'"], - ["nbsp", "\u00a0"], - ["lrm", "\u200e"], - ["rlm", "\u200f"], -]); +const NAMED_ENTITIES = new Map( + Object.entries({ ...COMMON_NAMED_ENTITIES, lrm: "\u200e", rlm: "\u200f" }), +); function decodeEntities(text: string): string { return text.replace(/&(?:#(\d+)|#x([\da-f]+)|([a-z]+));/gi, (match, decimal, hex, name) => { diff --git a/src/utility/htmlEntities.ts b/src/utility/htmlEntities.ts new file mode 100644 index 00000000..823d69b6 --- /dev/null +++ b/src/utility/htmlEntities.ts @@ -0,0 +1,8 @@ +export const COMMON_NAMED_ENTITIES = { + amp: "&", + lt: "<", + gt: ">", + quot: '"', + apos: "'", + nbsp: "\u00a0", +}; diff --git a/src/utility/privateFeedUrl.test.ts b/src/utility/privateFeedUrl.test.ts index 31e6db38..964283de 100644 --- a/src/utility/privateFeedUrl.test.ts +++ b/src/utility/privateFeedUrl.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import checkStringIsUrl from "./checkStringIsUrl"; import { isCredentialBearingUrl, isPrivateFeedPlaceholder, @@ -7,6 +8,11 @@ import { redactPrivateFeedAddresses, } from "./privateFeedUrl"; +vi.mock("./checkStringIsUrl", async (importOriginal) => { + const actual = await importOriginal(); + return { default: vi.fn(actual.default) }; +}); + describe("isCredentialBearingUrl", () => { it("detects userinfo credentials", () => { expect(isCredentialBearingUrl("https://user:pass@feeds.example.com/rss")).toBe(true); @@ -128,6 +134,7 @@ describe("redactPrivateFeedAddresses", () => { it.each([ ["feed scheme", "feed://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], ["podcast scheme", "podcast://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], + ["feed-wrapped URL", "feed:https://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], ["HTTP", "http://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], ["scheme-less", "private.example.com/rss?auth=FOLLOWUP_SENTINEL"], ["entity escaped", "https://private.example.com/rss?auth=FOLLOWUP_SENTINEL"], @@ -208,6 +215,78 @@ describe("redactPrivateFeedAddresses", () => { expect(redactPrivateFeedAddresses(publicUrl, known)).toBe(publicUrl); }); + it("decodes an encoded query value once, so its path escapes stay significant", () => { + const address = "https://private.example.com/rss%2Fprivate?auth=PATH_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + expect(redactPrivateFeedAddresses(`?u=${encodeURIComponent(address)}`, known)).toBe( + "?u=podnotes-private-feed%3APaid", + ); + const decodedPath = `?u=${encodeURIComponent(decodeURIComponent(address))}`; + expect(redactPrivateFeedAddresses(decodedPath, known)).toBe(decodedPath); + }); + + it.each([ + ["scheme letters", "https://private.example.com/rss?auth=ENTITY_SENTINEL"], + [ + "hex scheme", + "https://private.example.com/rss?auth=ENTITY_SENTINEL", + ], + [ + "padded numbers", + "https://private.example.com/rss?auth=ENTITY_SENTINEL", + ], + ["semicolon-free numbers", "https://private.example.com/rss?auth=ENTITY_SENTINEL"], + ["scheme-less host", "private.example.com/rss?auth=ENTITY_SENTINEL"], + ])("redacts an address spelled with %s", (_, spelled) => { + const known = new Map([ + ["https://private.example.com/rss?auth=ENTITY_SENTINEL", "podnotes-private-feed:Paid"], + ]); + expect(redactPrivateFeedAddresses(`Feed: ${spelled}. &c`, known)).toBe( + "Feed: podnotes-private-feed:Paid. &c", + ); + }); + + it("never takes a known host inside another host name, path or user name", () => { + const known = new Map([ + ["https://private.example.com/rss?auth=HOST_SENTINEL", "podnotes-private-feed:Paid"], + ]); + const publicText = [ + "notprivate.example.com/rss?auth=HOST_SENTINEL", + "www.private.example.com/rss?auth=HOST_SENTINEL", + "https://user@private.example.com/rss?auth=HOST_SENTINEL", + "https://public.example/private.example.com/rss?auth=HOST_SENTINEL", + "https://public.example\\private.example.com/rss?auth=HOST_SENTINEL", + "feed://notprivate.example.com/rss?auth=HOST_SENTINEL", + "https%3A%2F%2Fnotprivate.example.com%2Frss%3Fauth%3DHOST_SENTINEL", + ]; + expect(redactPrivateFeedAddresses(publicText, known)).toBe(publicText); + expect( + redactPrivateFeedAddresses(`?u=private.example.com/rss?auth=HOST_SENTINEL`, known), + ).toBe("?u=podnotes-private-feed:Paid"); + }); + + it.each([ + ["percent escape", "https%3A%2F%2Fprivate.example.com%2Frss%3Fauth%3D%E0%A4%A"], + ["character reference", "� � � private.example.com"], + ])("skips a malformed %s and keeps redacting the rest", (_, malformed) => { + const address = "https://private.example.com/rss?auth=ESCAPE_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + expect(redactPrivateFeedAddresses({ malformed }, known)).toEqual({ malformed }); + expect(redactPrivateFeedAddresses(`${malformed} ${address}`, known)).toBe( + `${malformed} podnotes-private-feed:Paid`, + ); + }); + + it("leaves a saved placeholder alone even when its encoded name holds the address", () => { + const address = "https://private.example.com/rss?auth=NAME_SENTINEL"; + const placeholder = privateFeedPlaceholder(address); + const document = { url: placeholder, quoted: encodeURIComponent(address) }; + expect(redactPrivateFeedAddresses(document, new Map([[address, placeholder]]))).toEqual({ + url: placeholder, + quoted: encodeURIComponent(placeholder), + }); + }); + it.each([ "https://private.example.com./rss?auth=ALIAS_SENTINEL", "https:\\\\private.example.com/rss?auth=ALIAS_SENTINEL", @@ -246,6 +325,45 @@ describe("redactPrivateFeedAddresses", () => { }); }); + it("redacts addresses in markup escaped with the uppercase entities HTML defines", () => { + const known = new Map([ + [ + "https://private.example.com/rss?auth=CASE_SENTINEL&show=1", + "podnotes-private-feed:Paid", + ], + ]); + const alias = "https://private.example.com/rss?auth=CASE_SENTINEL&show=1"; + const document = { + attribute: ``, + escaped: `<a href="${alias}">${alias}</a> <${alias}>`, + }; + expect(redactPrivateFeedAddresses(document, known)).toEqual({ + attribute: '', + escaped: + "<a href="podnotes-private-feed:Paid">podnotes-private-feed:Paid</a> <podnotes-private-feed:Paid>", + }); + }); + + it.each([ + ["schemes", "https://private.example.com/x"], + ["encoded schemes", "https%3A%2F%2Fprivate.example.com%2Fx"], + ["feed wrappers", "feed://private.example.com/x"], + ["host query values", "=private.example.com/x"], + ["trailing dots", "."], + ])("parses a long run of repeated %s a bounded number of times", (_, unit) => { + const address = "https://private.example.com/rss?auth=WORK_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const run = `https://private.example.com/rss${unit.repeat(2000)}`; + const parse = vi.mocked(checkStringIsUrl); + parse.mockClear(); + expect(redactPrivateFeedAddresses(`Feed ${run} ${address}`, known)).toBe( + `Feed ${run} podnotes-private-feed:Paid`, + ); + const parsed = parse.mock.calls.reduce((total, [url]) => total + url.length, 0); + expect(parse).toHaveBeenCalled(); + expect(parsed).toBeLessThan(10 * run.length); + }); + it.each([ ["https://private.example.com", "https://%70rivate.example.com"], ["https://xn--bcher-kva.example.com", "https://bücher.example.com"], diff --git a/src/utility/privateFeedUrl.ts b/src/utility/privateFeedUrl.ts index ca407498..309d2bff 100644 --- a/src/utility/privateFeedUrl.ts +++ b/src/utility/privateFeedUrl.ts @@ -12,6 +12,7 @@ */ import checkStringIsUrl from "./checkStringIsUrl"; +import { COMMON_NAMED_ENTITIES } from "./htmlEntities"; const PLACEHOLDER_SCHEME = "podnotes-private-feed:"; @@ -98,25 +99,85 @@ export function canonicalUrl(url: string): string { return parsed.href; } -/** - * `document` with no private feed address left in it. A snapshot's - * credential-bearing `feedUrl` is a private feed address, as is each key of - * `known` (saved private feeds' real URLs, mapped to their placeholders). Every - * string anywhere that is one of these addresses, or quotes one as written, - * gets its placeholder instead, so the address cannot stay behind in a - * description or a field this version does not know. Signed media and an - * episode's own link are other URLs and stay as they are. Returns `document` - * itself when nothing changed; never mutates it. - */ +function redactionKey(address: string): string | undefined { + const url = checkStringIsUrl(address); + if (!url || !/^https?:$/.test(url.protocol)) return undefined; + url.protocol = "https:"; + url.hostname = url.hostname.replace(/\.$/, ""); + url.hash = ""; + return url.href; +} + +const HTML_ENTITY = /&(?:#(x[\da-f]+|\d+);?|([a-z]+);)/gi; +// These four common entity names also have uppercase HTML forms. +const NAMED_ENTITIES = new Map( + Object.entries({ + ...COMMON_NAMED_ENTITIES, + AMP: "&", + LT: "<", + GT: ">", + QUOT: '"', + colon: ":", + sol: "/", + period: ".", + quest: "?", + equals: "=", + num: "#", + percnt: "%", + lowbar: "_", + commat: "@", + plus: "+", + }), +); + +type Entity = [decodedStart: number, decodedEnd: number, sourceStart: number, sourceEnd: number]; + +function decodeHtml(value: string): [text: string, entities: Entity[]] { + const entities: Entity[] = []; + if (!value.includes("&")) return [value, entities]; + let shift = 0; + const text = value.replace( + HTML_ENTITY, + (entity: string, code: string | undefined, name: string | undefined, from: number) => { + const char = code + ? String.fromCodePoint( + Math.min(Number(/^x/i.test(code) ? `0${code}` : code), 0x10ffff), + ) + : NAMED_ENTITIES.get(name ?? ""); + if (char === undefined) return entity; + entities.push([from - shift, from - shift + char.length, from, from + entity.length]); + shift += entity.length - char.length; + return char; + }, + ); + return [text, entities]; +} + +function sourceIndex(entities: Entity[], index: number): number { + let shift = 0; + for (const [at, end, from, to] of entities) { + if (index <= at) break; + if (index < end) return from; + shift = to - end; + } + return index + shift; +} + +const ADDRESS_START = + /(podnotes-private-feed:[\w.!~*'()%-]*)|(https?%3a)|(?:feed|podcast|https?):|(?:^|[^\w.@/\\:%-])(?=([\w.-]+)[/?#:])/gi; +const ADDRESS = /[^\s<>"'`]+/y; +const ENCODED_ADDRESS = /[^\s<>"'`&#]+/y; +// Encoded, numeric and non-ASCII authorities can normalize to a known host too. +const UNUSUAL_AUTHORITY = /^https?:[/\\]*(?:[^/?#]*[%\\[\u0080-\uFFFF]|(?:[^/?#@]*@)?\d)/i; + export function redactPrivateFeedAddresses( document: T, known: ReadonlyMap = new Map(), ): T { const placeholders = new Map(); const add = (url: string, placeholder: string) => { - const canonical = canonicalUrl(url).split("#")[0]; - const kept = placeholders.get(canonical) ?? placeholder; - placeholders.set(canonical, kept); + const key = redactionKey(url); + if (key && !placeholders.has(key)) placeholders.set(key, placeholder); }; for (const [url, placeholder] of known) add(url, placeholder); const credentialed = new Map(); @@ -131,36 +192,74 @@ export function redactPrivateFeedAddresses( add(feedUrl, privateFeedPlaceholder(typeof podcastName === "string" ? podcastName : "")); }); if (placeholders.size === 0) return document; - const hosts = [ - ...new Set([...placeholders.keys()].map((address) => new URL(address).hostname)), - ]; - // Encoded, numeric and non-ASCII authorities can normalize to a known host too. - // A tagged template anywhere in main.js prevents V8 from caching the eval-compiled - // plugin, so keep this untagged to let Obsidian reuse it across reloads. - const candidate = new RegExp( - hosts.map((host) => host.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|") + - "|https?:[/\\\\]*[^\\s/?#]*[%\\\\\\[\\u0080-\\uFFFF]|https?:[/\\\\]*(?:[^\\s/?#@]*@)?\\d", - "i", - ); + const hosts = [...new Set([...placeholders.keys()].map((key) => new URL(key).hostname))]; + // URL normalization keeps trailing punctuation, so a run longer than the + // longest address cannot end one. + const longest = Math.max(...[...placeholders.keys()].map((key) => key.length)); const matches = new Map(); const lookup = (address: string) => { - if (matches.has(address)) return matches.get(address); - const replacement = placeholders.get( - canonicalUrl(address.replace(/&/gi, "&")).split("#")[0], - ); - matches.set(address, replacement); - return replacement; + if (!matches.has(address)) { + const key = redactionKey(address); + matches.set(address, key && placeholders.get(key)); + } + return matches.get(address); + }; + const spelledAddress = (quote: string, encoded: boolean) => { + if (!encoded) + return quote.replace(/^(?:feed|podcast):/i, "").replace(/^(?!https?:)\/*/i, "https://"); + try { + return decodeURIComponent(quote); + } catch { + return ""; + } + }; + const matchAddress = (quote: string, encoded: boolean, isWholeString: boolean) => { + const spelled = spelledAddress(quote, encoded); + const lower = spelled.toLowerCase(); + if (!hosts.some((host) => lower.includes(host)) && !UNUSUAL_AUTHORITY.test(spelled)) return; + let bare = quote.length; + while (!isWholeString && bare > 0 && ".,;:!?)]}".includes(quote[bare - 1])) bare--; + for (let end = bare; end <= Math.min(quote.length, bare + longest); end++) { + const placeholder = lookup(spelledAddress(quote.slice(0, end), encoded)); + if (placeholder) + return { end, text: encoded ? encodeURIComponent(placeholder) : placeholder }; + } }; return replaceStrings(document, (value) => { - if (!/https?:/i.test(value) || !candidate.test(value)) return value; - if (/^https?:[^\s<>"'`]+$/i.test(value)) return lookup(value) ?? value; - return value.replace(/https?:[^\s<>"'`]+/gi, (address) => { - const whole = lookup(address); - if (whole) return whole; - const bare = address.replace(/[.,;:!?)\]}]+$/, ""); - const replacement = lookup(bare); - return replacement ? replacement + address.slice(bare.length) : address; - }); + const [text, entities] = decodeHtml(value); + let redacted = ""; + let copied = 0; + // A failed address still gets searched for an encoded one in its query. + let failedUntil = 0; + ADDRESS_START.lastIndex = 0; + for (let match; (match = ADDRESS_START.exec(text));) { + const [start, placeholder, encodedScheme, host] = match; + // A host name at the very start matches empty. + if (!start) ADDRESS_START.lastIndex++; + const at = match.index + (host === undefined ? 0 : start.length); + if (placeholder || (host !== undefined && !hosts.includes(host.toLowerCase()))) + continue; + if (!encodedScheme && at < failedUntil) continue; + const extent = encodedScheme ? ENCODED_ADDRESS : ADDRESS; + extent.lastIndex = at; + extent.test(text); + const end = extent.lastIndex; + const found = matchAddress( + text.slice(at, end), + Boolean(encodedScheme), + at === 0 && end === text.length, + ); + if (found) { + redacted += value.slice(copied, sourceIndex(entities, at)) + found.text; + copied = sourceIndex(entities, at + found.end); + ADDRESS_START.lastIndex = at + found.end; + } else if (encodedScheme) { + ADDRESS_START.lastIndex = end; + } else { + failedUntil = end; + } + } + return copied ? redacted + value.slice(copied) : value; }); } diff --git a/tests/e2e/private-notes.test.ts b/tests/e2e/private-notes.test.ts index 548b2e81..a35d6e2c 100644 --- a/tests/e2e/private-notes.test.ts +++ b/tests/e2e/private-notes.test.ts @@ -85,10 +85,10 @@ test("entity-escaped private descriptions stay out of data.json and the settings const exported = await readVaultText(obsidian, sandbox.path(name)); const note = await readVaultText(obsidian, sandbox.path(notePath)); await mkdir(evidence, { recursive: true }); - for (const [name, text] of Object.entries({ persisted, exported, note })) { - await writeFile(path.join(evidence, `${name}.txt`), text); - expect.soft(text, name).not.toContain("PRIV5_ENTITY_SENTINEL"); - expect.soft(text, name).toContain("podnotes-private-feed:Entity%20Private"); + for (const [artifact, text] of Object.entries({ persisted, exported, note })) { + await writeFile(path.join(evidence, `${artifact}.txt`), text); + expect.soft(text, artifact).not.toContain("PRIV5_ENTITY_SENTINEL"); + expect.soft(text, artifact).toContain("podnotes-private-feed:Entity%20Private"); } expect(persisted).toContain(publicText); expect(exported).toContain(publicText); From 4f3ef5f55199b2dc1bf2814b7b8298ba7f86c750 Mon Sep 17 00:00:00 2001 From: Christian Bager Bach Houmann Date: Wed, 14 Oct 2026 22:15:30 +0200 Subject: [PATCH 3/3] fix(privacy): avoid repeated redaction work during saves --- src/main.persistence.test.ts | 85 +++++++++++++ src/main.ts | 15 ++- src/utility/privateFeedUrl.test.ts | 194 +++++++++++++++++++++++++++++ src/utility/privateFeedUrl.ts | 34 ++++- 4 files changed, 320 insertions(+), 8 deletions(-) diff --git a/src/main.persistence.test.ts b/src/main.persistence.test.ts index bff21997..16c72c74 100644 --- a/src/main.persistence.test.ts +++ b/src/main.persistence.test.ts @@ -5,6 +5,8 @@ import { Notice, type SecretStorage } from "obsidian"; import * as obsidian from "obsidian"; import { DEFAULT_SETTINGS } from "./constants"; import PodNotes from "./main"; +import * as privateFeedUrls from "./utility/privateFeedUrl"; +import * as persistenceMerge from "./persistence/merge"; import { encodePodNotesData } from "./persistence/podNotesData"; import { PlayerController } from "./player/PlayerController"; import { downloadedEpisodes, localFiles, plugin as pluginStore } from "./store"; @@ -988,6 +990,89 @@ describe("PodNotes persistence integration", () => { expect(saveData.mock.lastCall?.[0].queue.episodes).toHaveLength(2); }); + it("does not rescan an equal repair just because restoration copied it", async () => { + const saveData = vi.fn().mockResolvedValue(undefined); + const plugin = makePlugin(saveData, { dataFileExists: true }); + const rawUrl = "https://private.example/feed.xml?auth=EQUAL_REPAIR_SENTINEL"; + const initial = JSON.parse(JSON.stringify(encodePodNotesData(plugin.settings))); + initial.savedFeeds = { Private: { title: "Private", url: rawUrl, artworkUrl: "" } }; + await plugin.saveData(initial); + await plugin.loadSettings(); + const synced = JSON.parse(JSON.stringify(encodePodNotesData(plugin.settings))); + synced.savedFeeds.Private.url = rawUrl; + await plugin.saveData(synced); + saveData.mockClear(); + const repairs = vi.spyOn(persistenceMerge, "restoreDropped"); + const redactions = vi.spyOn(privateFeedUrls, "redactPrivateFeedAddresses"); + + plugin.settings.defaultVolume = 0.4; + await plugin.saveSettingsStrict(); + + expect( + repairs.mock.results.some( + (result, index) => + result.type === "return" && + result.value !== repairs.mock.calls[index][1] && + persistenceMerge.jsonValuesEqual(result.value, repairs.mock.calls[index][1]), + ), + ).toBe(true); + expect(redactions).toHaveBeenCalledTimes(5); + expect(saveData).toHaveBeenCalledTimes(1); + expect(saveData.mock.lastCall?.[0]).toMatchObject({ + defaultVolume: 0.4, + savedFeeds: { Private: { url: "podnotes-private-feed:Private" } }, + }); + expect(JSON.stringify(saveData.mock.calls)).not.toContain("EQUAL_REPAIR_SENTINEL"); + }); + + describe("when a synced private feed arrives with a queue that had to be repaired", () => { + const rawUrl = "https://private.example/feed.xml?auth=RESTORED_SENTINEL"; + const episode = { + title: "Ep", + podcastName: "Pod", + streamUrl: "https://example.com/ep.mp3", + url: "", + content: "", + }; + + async function saveAfterSync(description: string) { + const saveData = vi.fn().mockResolvedValue(undefined); + const plugin = makePlugin(saveData, { dataFileExists: true }); + await plugin.loadSettings(); + plugin.settings.queue.episodes = [{ ...episode, description }]; + attachDataFileWriter(plugin); + const synced = JSON.parse(JSON.stringify(encodePodNotesData(plugin.settings))); + synced.queue = "not a playlist"; + synced.savedFeeds = { Private: { title: "Private", url: rawUrl, artworkUrl: "" } }; + await plugin.saveData(synced); + saveData.mockClear(); + + plugin.settings.defaultVolume = 0.4; + await plugin.saveSettingsStrict(); + return saveData; + } + + it("redacts the address in the entries it keeps from this device", async () => { + const saveData = await saveAfterSync(`Your feed: ${rawUrl}`); + + const written = saveData.mock.lastCall?.[0]; + expect(written.queue.episodes).toEqual([ + { ...episode, description: "Your feed: podnotes-private-feed:Private" }, + ]); + expect(written.savedFeeds.Private.url).toBe("podnotes-private-feed:Private"); + expect(JSON.stringify(saveData.mock.calls)).not.toContain("RESTORED_SENTINEL"); + }); + + it("keeps the entries unchanged when they quote no private address", async () => { + const saveData = await saveAfterSync("Public notes"); + + const written = saveData.mock.lastCall?.[0]; + expect(written.queue.episodes).toEqual([{ ...episode, description: "Public notes" }]); + expect(written.savedFeeds.Private.url).toBe("podnotes-private-feed:Private"); + expect(written.defaultVolume).toBe(0.4); + }); + }); + it("drops a saved feed cache preference at the next save, even one an older device syncs back", async () => { const saveData = vi.fn().mockResolvedValue(undefined); const plugin = makePlugin(saveData, { dataFileExists: true }); diff --git a/src/main.ts b/src/main.ts index d58d4ed8..d30a1677 100644 --- a/src/main.ts +++ b/src/main.ts @@ -567,14 +567,21 @@ export default class PodNotes extends Plugin implements IPodNotes { // An entry the codec dropped while repairing a value (an invalid episode, a // collection reset) is not a deletion on the other device. Keep it. const raw = data as Record; + let restored = false; for (const [key, value] of Object.entries(base ?? {})) { - if (!jsonValuesEqual(raw[key], doc[key])) { - doc[key] = restoreDropped(value, doc[key], raw[key], [key]); - } + if (jsonValuesEqual(raw[key], doc[key])) continue; + const kept = restoreDropped(value, doc[key], raw[key], [key]); + if (jsonValuesEqual(kept, doc[key])) continue; + doc[key] = kept; + restored = true; } + // toDocument redacted the rest. A restored entry can quote a private address, + // or name a private feed whose address other fields quote. return { kind: "file", - doc: redactPrivateFeedAddresses(doc, addresses) as PersistedPodNotesDataV2, + doc: (restored + ? redactPrivateFeedAddresses(doc, addresses) + : doc) as PersistedPodNotesDataV2, }; }, write: (doc) => diff --git a/src/utility/privateFeedUrl.test.ts b/src/utility/privateFeedUrl.test.ts index 964283de..365aa4aa 100644 --- a/src/utility/privateFeedUrl.test.ts +++ b/src/utility/privateFeedUrl.test.ts @@ -425,3 +425,197 @@ describe("private feed placeholder", () => { expect(parsePrivateFeedPlaceholder("podnotes-private-feed:%E0%A4%A")).toBeNull(); }); }); + +describe("candidate filtering preserves every covered address spelling", () => { + const address = "https://private.example/rss?auth=FILTER_SENTINEL"; + const numeric = "https://127.0.0.1/rss?auth=FILTER_SENTINEL"; + const spellings: [name: string, knownAddress: string, spelling: string][] = [ + ["plain", address, address], + ["HTTP", address, "http://private.example/rss?auth=FILTER_SENTINEL"], + ["feed wrapper", address, `feed:${address}`], + ["podcast wrapper", address, `podcast:${address}`], + ["scheme-less", address, "private.example/rss?auth=FILTER_SENTINEL"], + ["host case", address, "https://PRIVATE.EXAMPLE/rss?auth=FILTER_SENTINEL"], + [ + "numeric entities", + address, + "https://private.example/rss?auth=FILTER_SENTINEL", + ], + [ + "hex entities", + address, + "https://private.example/rss?auth=FILTER_SENTINEL", + ], + [ + "named entities", + address, + "https://private.example/rss?auth=FILTER_SENTINEL", + ], + ["percent-encoded host", address, "https://%70rivate.example/rss?auth=FILTER_SENTINEL"], + ["Unicode host", address, "https://ｐｒｉｖａｔｅ.example/rss?auth=FILTER_SENTINEL"], + ["backslashes", address, "https:\\\\private.example/rss?auth=FILTER_SENTINEL"], + [ + "numeric feed authority", + "https://127.0.0.1/rss?auth=FILTER_SENTINEL", + "feed://2130706433/rss?auth=FILTER_SENTINEL", + ], + [ + "numeric podcast authority", + "https://127.0.0.1/rss?auth=FILTER_SENTINEL", + "podcast://0x7f000001/rss?auth=FILTER_SENTINEL", + ], + [ + "IPv6", + "https://[::1]/rss?auth=FILTER_SENTINEL", + "https://[0:0:0:0:0:0:0:1]/rss?auth=FILTER_SENTINEL", + ], + [ + "ampersand in the host", + "https://a&b.example/rss?auth=FILTER_SENTINEL", + "https://a&b.example/rss?auth=FILTER_SENTINEL", + ], + ["malformed mixed-case amp", `${address}&show=1`, `${address}&aMp;show=1`], + [ + "malformed mixed-case amp in the host", + "https://a&b.example/rss?auth=FILTER_SENTINEL", + "https://a&aMp;b.example/rss?auth=FILTER_SENTINEL", + ], + ["numeric HTTP authority", numeric, "http://0x7f.1/rss?auth=FILTER_SENTINEL"], + [ + "numeric authority with backslashes", + numeric, + "https:\\\\2130706433/rss?auth=FILTER_SENTINEL", + ], + [ + "numeric authority after user info", + "https://user@127.0.0.1/rss?auth=FILTER_SENTINEL", + "https://user@2130706433/rss?auth=FILTER_SENTINEL", + ], + [ + "uppercase punycode", + "https://xn--bcher-kva.example/rss?auth=FILTER_SENTINEL", + "https://XN--BCHER-KVA.EXAMPLE/rss?auth=FILTER_SENTINEL", + ], + ]; + it.each(spellings)( + "redacts %s and preserves its distinct public address", + (_name, knownAddress, spelling) => { + const known = new Map([[knownAddress, "podnotes-private-feed:Paid"]]); + const input = `Subscribe at ${spelling}.`; + const publicText = input.replaceAll("FILTER_SENTINEL", "PUBLIC_VALUE"); + expect(redactPrivateFeedAddresses(input, known)).toBe( + "Subscribe at podnotes-private-feed:Paid.", + ); + expect(redactPrivateFeedAddresses(publicText, known)).toBe(publicText); + }, + ); + + it("redacts an encoded query with no literal hostname and preserves a public query", () => { + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const input = + "Visit https://redirect.example/?feed=https%3A%2F%2Fprivate%2Eexample%2Frss%3Fauth%3DFILTER_SENTINEL&next=1."; + expect(redactPrivateFeedAddresses(input, known)).toBe( + "Visit https://redirect.example/?feed=podnotes-private-feed%3APaid&next=1.", + ); + const publicText = input.replaceAll("FILTER_SENTINEL", "PUBLIC_VALUE"); + expect(redactPrivateFeedAddresses(publicText, known)).toBe(publicText); + }); + + it("redacts an object key and leaves its own output unchanged", () => { + for (const [, knownAddress, spelling] of spellings) { + const known = new Map([[knownAddress, "podnotes-private-feed:Paid"]]); + const publicText = spelling.replaceAll("FILTER_SENTINEL", "PUBLIC_VALUE"); + const safe = redactPrivateFeedAddresses( + { [`Subscribe at ${spelling}.`]: publicText }, + known, + ); + expect(safe).toEqual({ "Subscribe at podnotes-private-feed:Paid.": publicText }); + expect(redactPrivateFeedAddresses(safe, known)).toBe(safe); + } + }); + + it("decodes amp in any case, but no other entity name HTML lacks in capitals", () => { + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const capitals = "Subscribe at https&COLON;//private.example/rss?auth=FILTER_SENTINEL."; + expect(redactPrivateFeedAddresses(capitals, known)).toBe(capitals); + }); +}); + +describe("redacting a document with repeated text", () => { + const address = "https://private.example/rss?auth=REPEAT_SENTINEL"; + const known = new Map([[address, "podnotes-private-feed:Paid"]]); + const prose = `Feed ${address}.`; + // A whole field ending in a period is a distinct resource address. + const field = `${address}.`; + + it("redacts every copy and keeps a whole field apart from prose quoting the same text", () => { + expect(redactPrivateFeedAddresses([field, prose, field, prose], known)).toEqual([ + field, + "Feed podnotes-private-feed:Paid.", + field, + "Feed podnotes-private-feed:Paid.", + ]); + expect(redactPrivateFeedAddresses({ a: prose, b: field, [prose]: field }, known)).toEqual({ + a: "Feed podnotes-private-feed:Paid.", + b: field, + "Feed podnotes-private-feed:Paid.": field, + }); + }); + + it("returns an unchanged document itself and never mutates one it copies", () => { + const publicText = prose.replace("REPEAT_SENTINEL", "PUBLIC_VALUE"); + const unchanged = { a: [publicText, publicText], b: { [publicText]: publicText } }; + expect(redactPrivateFeedAddresses(unchanged, known)).toBe(unchanged); + + const document = { a: [prose, prose], b: { [prose]: prose }, c: publicText }; + const before = structuredClone(document); + expect(redactPrivateFeedAddresses(document, known)).toEqual({ + a: ["Feed podnotes-private-feed:Paid.", "Feed podnotes-private-feed:Paid."], + b: { "Feed podnotes-private-feed:Paid.": "Feed podnotes-private-feed:Paid." }, + c: publicText, + }); + expect(document).toEqual(before); + }); + + it("refuses a key collision on a repeated key", () => { + expect(() => + redactPrivateFeedAddresses( + { a: { [prose]: 1 }, b: { [prose]: 2, "Feed podnotes-private-feed:Paid.": 3 } }, + known, + ), + ).toThrow("Private feed address protection would overwrite a saved field."); + expect(redactPrivateFeedAddresses({ a: { [prose]: 1 }, b: { [prose]: 2 } }, known)).toEqual( + { + a: { "Feed podnotes-private-feed:Paid.": 1 }, + b: { "Feed podnotes-private-feed:Paid.": 2 }, + }, + ); + }); + + it("takes each call's own addresses and placeholders, never an earlier call's", () => { + const other = "https://other.example/rss?auth=OTHER_SENTINEL"; + const placeholder = (name: string) => new Map([[address, `podnotes-private-feed:${name}`]]); + expect(redactPrivateFeedAddresses(prose, placeholder("Old"))).toBe( + "Feed podnotes-private-feed:Old.", + ); + expect(redactPrivateFeedAddresses(prose, placeholder("New"))).toBe( + "Feed podnotes-private-feed:New.", + ); + expect( + redactPrivateFeedAddresses(prose, new Map([[other, "podnotes-private-feed:Other"]])), + ).toBe(prose); + + const snapshot = { podcastName: "Found", feedUrl: address, description: prose }; + expect(redactPrivateFeedAddresses([snapshot, prose])).toEqual([ + { + podcastName: "Found", + feedUrl: "podnotes-private-feed:Found", + description: "Feed podnotes-private-feed:Found.", + }, + "Feed podnotes-private-feed:Found.", + ]); + expect( + redactPrivateFeedAddresses([prose], new Map([[other, "podnotes-private-feed:Other"]])), + ).toEqual([prose]); + }); +}); diff --git a/src/utility/privateFeedUrl.ts b/src/utility/privateFeedUrl.ts index 309d2bff..6f48bf25 100644 --- a/src/utility/privateFeedUrl.ts +++ b/src/utility/privateFeedUrl.ts @@ -109,11 +109,11 @@ function redactionKey(address: string): string | undefined { } const HTML_ENTITY = /&(?:#(x[\da-f]+|\d+);?|([a-z]+);)/gi; -// These four common entity names also have uppercase HTML forms. +// HTML also spells lt, gt and quot in capitals. decodeHtml takes amp in any case, so +// an address escaped with a malformed &aMp; still matches its known address. const NAMED_ENTITIES = new Map( Object.entries({ ...COMMON_NAMED_ENTITIES, - AMP: "&", LT: "<", GT: ">", QUOT: '"', @@ -143,7 +143,7 @@ function decodeHtml(value: string): [text: string, entities: Entity[]] { ? String.fromCodePoint( Math.min(Number(/^x/i.test(code) ? `0${code}` : code), 0x10ffff), ) - : NAMED_ENTITIES.get(name ?? ""); + : NAMED_ENTITIES.get(name?.toLowerCase() === "amp" ? "amp" : (name ?? "")); if (char === undefined) return entity; entities.push([from - shift, from - shift + char.length, from, from + entity.length]); shift += entity.length - char.length; @@ -169,7 +169,19 @@ const ADDRESS = /[^\s<>"'`]+/y; const ENCODED_ADDRESS = /[^\s<>"'`&#]+/y; // Encoded, numeric and non-ASCII authorities can normalize to a known host too. const UNUSUAL_AUTHORITY = /^https?:[/\\]*(?:[^/?#]*[%\\[\u0080-\uFFFF]|(?:[^/?#@]*@)?\d)/i; +// A string the scan can redact holds one of these, or a known host. +const ESCAPED_OR_UNUSUAL = /[&%@\\[\u0080-\uFFFF]|(?:https?|feed|podcast):\/*\d/i; +/** + * `document` with no private feed address left in it. A snapshot's + * credential-bearing `feedUrl` is a private feed address, as is each key of + * `known` (saved private feeds' real URLs, mapped to their placeholders). Every + * string anywhere that is one of these addresses, or quotes one as written, + * gets its placeholder instead, so the address cannot stay behind in a + * description or a field this version does not know. Signed media and an + * episode's own link are other URLs and stay as they are. Returns `document` + * itself when nothing changed; never mutates it. + */ export function redactPrivateFeedAddresses( document: T, known: ReadonlyMap = new Map(), @@ -225,7 +237,7 @@ export function redactPrivateFeedAddresses( return { end, text: encoded ? encodeURIComponent(placeholder) : placeholder }; } }; - return replaceStrings(document, (value) => { + const redactText = (value: string) => { const [text, entities] = decodeHtml(value); let redacted = ""; let copied = 0; @@ -260,6 +272,20 @@ export function redactPrivateFeedAddresses( } } return copied ? redacted + value.slice(copied) : value; + }; + const mayQuote = (value: string) => { + if (ESCAPED_OR_UNUSUAL.test(value)) return true; + const lower = value.toLowerCase(); + return hosts.some((host) => lower.includes(host)); + }; + const results = new Map(); + return replaceStrings(document, (value) => { + let result = results.get(value); + if (result === undefined) { + result = mayQuote(value) ? redactText(value) : value; + results.set(value, result); + } + return result; }); }