From 5a0e8c739ec1b58e7647dd7e5b6e73ade3c1f63f Mon Sep 17 00:00:00 2001 From: David Bond Date: Tue, 6 Oct 2026 01:19:54 +0100 Subject: [PATCH 1/2] MS-26473 PDChat: wake phrase for Voice Mode, and the demo exposes every voice option With IChatService.WakePhrases set, a microphone that hears no word for VoiceIdleTimeout (default 10 seconds) goes Dormant: nothing is typed or sent until a wake phrase is heard, matched ignoring case and punctuation, and only the words after it are typed. It never goes dormant while the text box has focus or a send is pending. PDChatVoiceState.Dormant is appended to the enum. DumbChatService makes its voice endpoints, agents, models, wake phrases, idle timeout, auto-send delay, read-aloud and selections settable, each raising OnConfigurationChanged, and the PDChat demo page gains a Voice and Assistant section to drive them. The simulated microphone starts every other utterance with "Hey DumbBot," so the wake phrase can be seen working. Co-Authored-By: Claude Opus 5.5 --- .../Pages/PDChatDocumentation.razor | 30 +++ .../Pages/PDChatPage.razor | 59 +++++ .../Pages/PDChatPage.razor.cs | 61 ++++- .../Components/PDChatTests.WakePhrase.cs | 250 ++++++++++++++++++ .../DumbChatServiceVoiceTests.cs | 73 +++++ .../Interfaces/ChatServiceDefaultState.cs | 4 + .../Interfaces/IChatService.cs | 26 ++ PanoramicData.Blazor/PDChat.Voice.cs | 48 +++- PanoramicData.Blazor/PDChat.WakePhrase.cs | 136 ++++++++++ PanoramicData.Blazor/PDChat.razor.css | 5 + .../Services/DumbChatService.Voice.cs | 89 ++++++- .../wwwroot/js/pdchat-voice-simulated.js | 14 +- 12 files changed, 778 insertions(+), 17 deletions(-) create mode 100644 PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs create mode 100644 PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs create mode 100644 PanoramicData.Blazor/PDChat.WakePhrase.cs diff --git a/PanoramicData.Blazor.Demo/Pages/PDChatDocumentation.razor b/PanoramicData.Blazor.Demo/Pages/PDChatDocumentation.razor index 892500bd5..f8d88de02 100644 --- a/PanoramicData.Blazor.Demo/Pages/PDChatDocumentation.razor +++ b/PanoramicData.Blazor.Demo/Pages/PDChatDocumentation.razor @@ -127,4 +127,34 @@ ToastEnabled / ToastDisplayDurationSeconds for backward compatibility.

+
+

Input toolbar, voice and wake phrase

+

+ A thin toolbar sits above the text box. It holds an agent picker and a model picker (each shown only when + the service lists two or more), the notification sounds bell (IsMuted), and, when the service + supplies VoiceEndpoints, a read-aloud speaker and a microphone. Dictated words are typed into + the text box and sent after a pause unless the user is editing. All of these are configured on + IChatService and have defaults, so an existing service needs no change. +

+ + + + + + + + + + + +
IChatService memberTypeDefaultDescription
Agents / SelectedAgentIdIReadOnlyList<PDChatAgentOption>? / string?nullWho the user is talking to.
Models / SelectedModelIdIReadOnlyList<PDChatModelOption>? / string?nullWhich model answers.
VoiceEndpointsPDChatVoiceEndpoints?nullWhere speech is sent and answers fetched; null hides the voice controls. PDChatVoiceEndpoints.Simulated needs no speech service.
IsReadAloudEnabledboolfalseReads the answer to each sent message aloud. Independent of the microphone and of notification sounds.
VoiceAutoSendDelayTimeSpan1 secondHow long after a pause dictation is sent, when the user is not editing.
WakePhrasesIReadOnlyList<string>?null (off)Phrases such as "Hey DumbBot" that wake a dormant microphone. Case and punctuation are ignored, and the phrase itself is never typed.
VoiceIdleTimeoutTimeSpan10 secondsHow long the microphone may hear nothing before it goes dormant. Applies only with wake phrases.
+

+ With wake phrases set, Voice Mode goes Dormant after the idle timeout (never while the user is + editing or a send is pending). While dormant nothing is typed or sent until a wake phrase is heard; the words + after it are typed as usual. The microphone keeps streaming to the speech service while dormant, since it + must hear the phrase, so tell your users that a dormant microphone is still listening. Try it with the + Voice and Assistant settings on this page: the simulated microphone starts every other + utterance with "Hey DumbBot". +

+
diff --git a/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor b/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor index 2edfe1dd8..83f63f3fa 100644 --- a/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor +++ b/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor @@ -288,6 +288,65 @@
Uncheck "Enable Message Input" above to see this. Leave blank for pure toast-only mode (the input simply disappears with no message).
+ +
+
Voice and Assistant
+
The input toolbar above the text box. The simulated microphone makes up what it hears, and starts every other utterance with "Hey DumbBot".
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+
+ + +
+
+ + +
+
+ +
+ + +
With Wake Phrase on, the microphone goes dormant after the idle time and types nothing until it hears one of these. It is still streaming audio while dormant.
+
+ +
+ Agent: @(ChatService.SelectedAgentId ?? "(default)"), + Model: @(ChatService.SelectedModelId ?? "(default)") +
diff --git a/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs b/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs index e7fae61b1..cc44b2603 100644 --- a/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs +++ b/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs @@ -1,4 +1,6 @@ -namespace PanoramicData.Blazor.Demo.Pages; +using PanoramicData.Blazor.Services; + +namespace PanoramicData.Blazor.Demo.Pages; public partial class PDChatPage : IDisposable { @@ -23,6 +25,63 @@ private PDChatDockMode CurrentDockMode private readonly Action _onDockModeChanged; + private string _wakePhraseText = string.Join(", ", DumbChatService.DemoWakePhrases); + + // Voice endpoints, agents and models are read-only on IChatService, so the demo sets them on its own service. + private DumbChatService? DemoService => ChatService as DumbChatService; + + private bool IsVoiceOffered + { + get => ChatService.VoiceEndpoints is not null; + set => DemoService?.VoiceEndpoints = value ? PDChatVoiceEndpoints.Simulated : null; + } + + private bool IsAgentPickerShown + { + get => ChatService.Agents is { Count: > 1 }; + set => DemoService?.Agents = value ? DumbChatService.DemoAgents : null; + } + + private bool IsModelPickerShown + { + get => ChatService.Models is { Count: > 1 }; + set => DemoService?.Models = value ? DumbChatService.DemoModels : null; + } + + private int AutoSendDelayMs + { + get => (int)ChatService.VoiceAutoSendDelay.TotalMilliseconds; + set => ChatService.VoiceAutoSendDelay = TimeSpan.FromMilliseconds(Math.Max(0, value)); + } + + private double IdleTimeoutSeconds + { + get => ChatService.VoiceIdleTimeout.TotalSeconds; + set => ChatService.VoiceIdleTimeout = TimeSpan.FromSeconds(Math.Max(1, value)); + } + + private bool IsWakePhraseOn + { + get => ChatService.WakePhrases is { Count: > 0 }; + set => ChatService.WakePhrases = value ? ParseWakePhrases(_wakePhraseText) : null; + } + + private string WakePhraseText + { + get => _wakePhraseText; + set + { + _wakePhraseText = value; + if (IsWakePhraseOn) + { + ChatService.WakePhrases = ParseWakePhrases(value); + } + } + } + + private static string[] ParseWakePhrases(string text) + => text.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + public PDChatPage() { // The new dock mode is read from the service when re-rendering diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs new file mode 100644 index 000000000..c99323b02 --- /dev/null +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs @@ -0,0 +1,250 @@ +using AwesomeAssertions; +using Bunit; +using Microsoft.AspNetCore.Components.Web; +using PanoramicData.Blazor.Interfaces; + +namespace PanoramicData.Blazor.Test; + +/// +/// The wake phrase: with wake phrases set, a microphone that hears nothing for the idle timeout goes dormant, types and +/// sends nothing, and listens again only once a wake phrase is heard. +/// +public partial class PDChatTests +{ + private static readonly TimeSpan _briefIdle = TimeSpan.FromMilliseconds(50); + + /// Verifies that wake phrases are off and the idle timeout is ten seconds by default. + [Fact] + public void Wake_phrases_are_off_by_default() + { + IChatService service = new FakeChatService(); + + service.WakePhrases.Should().BeNull(); + service.VoiceIdleTimeout.Should().Be(TimeSpan.FromSeconds(10)); + } + + /// Verifies that with wake phrases set, listening without a word goes dormant and the microphone stays on. + [Fact] + public async Task With_wake_phrases_an_idle_microphone_goes_dormant() + { + var module = SetUpVoiceModule(); + var service = WakeService(); + ((IChatService)service).VoiceIdleTimeout = _briefIdle; + + var component = await RenderListeningAsync(service); + + component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience); + module.Invocations["stop"].Should().BeEmpty(); + component.Find(".pdchat-voice-toggle").GetAttribute("aria-pressed").Should().Be("true"); + } + + /// Verifies that without wake phrases the microphone never goes dormant. + [Fact] + public async Task Without_wake_phrases_the_microphone_never_goes_dormant() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMinutes(10)); + ((IChatService)service).VoiceIdleTimeout = _briefIdle; + + var component = await RenderListeningAsync(service); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + } + + /// Verifies that the status line and the microphone control say which phrase is awaited. + [Fact] + public async Task A_dormant_microphone_says_what_it_is_waiting_for() + { + SetUpVoiceModule(); + var component = await RenderDormantAsync(WakeService()); + + component.Find(".pdchat-voice-status").TextContent.Trim().Should().Be("Waiting for \"Hey DumbBot\"…"); + component.Find(".pdchat-voice-status").ClassList.Should().Contain("pdchat-voice-dormant"); + var toggle = component.Find(".pdchat-voice-toggle"); + toggle.GetAttribute("title").Should().Be("Voice: waiting for \"Hey DumbBot\". Press to stop listening"); + toggle.GetAttribute("aria-label").Should().Be(toggle.GetAttribute("title")); + } + + /// Verifies that while dormant, words are not typed and a pause never sends. + [Fact] + public async Task Dormant_words_are_not_typed_and_a_dormant_turn_never_sends() + { + SetUpVoiceModule(); + var service = WakeService(TimeSpan.Zero); + var component = await RenderDormantAsync(service); + + await SayAsync(component, "What", "time", "is", "it?"); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty(); + service.Sent.Should().BeEmpty(); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant); + } + + /// Verifies that the wake phrase, in any case and with punctuation, wakes it and only the words after it are sent. + [Fact] + public async Task The_wake_phrase_wakes_it_and_only_the_words_after_it_are_sent() + { + SetUpVoiceModule(); + var service = WakeService(TimeSpan.Zero, readAloud: false); + var component = await RenderDormantAsync(service); + + await SayAsync(component, "Ignore", "this.", "HEY", "dumbbot,", "is", "it", "done?"); + + component.WaitForAssertion(() => service.Sent.Should().ContainSingle().Which.Message.Should().Be("is it done?"), Patience); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + } + + /// Verifies that a phrase of several words wakes it whether heard word by word or in one piece. + [Theory] + [InlineData("OK", "Dumb", "Bot.", "Is", "it", "done?")] + [InlineData("OK Dumb Bot. Is", "it", "done?")] + public async Task A_phrase_of_several_words_wakes_it(params string[] words) + { + SetUpVoiceModule(); + var service = WakeService(); + ((IChatService)service).WakePhrases = ["Hey DumbBot", "OK Dumb Bot"]; + var component = await RenderDormantAsync(service); + + foreach (var word in words) + { + await component.InvokeAsync(() => component.Instance.OnVoiceWord(word)); + } + + component.Find("textarea").GetAttribute("value").Should().Be("Is it done?"); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + } + + /// Verifies that the words of a phrase heard apart, or only in part, do not wake it. + [Theory] + [InlineData("Hey", "there", "DumbBot")] + [InlineData("Hey")] + [InlineData("DumbBot")] + public async Task A_partial_phrase_does_not_wake_it(params string[] words) + { + SetUpVoiceModule(); + var component = await RenderDormantAsync(WakeService()); + + await SayAsync(component, words); + + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant); + component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty(); + } + + /// Verifies that a wake phrase followed by a pause types nothing, so the phrase itself is never sent. + [Fact] + public async Task The_wake_phrase_alone_is_never_typed() + { + SetUpVoiceModule(); + var service = WakeService(TimeSpan.Zero); + var component = await RenderDormantAsync(service); + + await SayAsync(component, "Hey", "DumbBot"); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty(); + service.Sent.Should().BeEmpty(); + } + + /// Verifies that a host reporting only pauses can wake it, and the words after the phrase are typed. + [Fact] + public async Task A_host_that_reports_only_pauses_can_wake_it() + { + SetUpVoiceModule(); + var component = await RenderDormantAsync(WakeService()); + + await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Nothing to see.")); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant); + + await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Hey DumbBot, is it done?")); + + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + component.Find("textarea").GetAttribute("value").Should().Be("is it done?"); + } + + /// Verifies that each word restarts the idle timer, so a steady speaker never goes dormant. + [Fact] + public async Task Each_word_restarts_the_idle_timer() + { + SetUpVoiceModule(); + var service = WakeService(); + ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMilliseconds(400); + var component = await RenderListeningAsync(service); + + for (var word = 0; word < 6; word++) + { + await Task.Delay(100, Xunit.TestContext.Current.CancellationToken); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("more")); + } + + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience); + } + + /// Verifies that it never goes dormant while the user is editing the text box, and does once they stop. + [Fact] + public async Task It_does_not_go_dormant_while_the_text_box_has_focus() + { + SetUpVoiceModule(); + var service = WakeService(); + ((IChatService)service).VoiceIdleTimeout = _briefIdle; + var component = RenderChat(service); + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + + await component.Find("textarea").BlurAsync(new FocusEventArgs()); + + component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience); + } + + /// Verifies that turning Voice off cancels the idle timer, so it stays off rather than going dormant. + [Fact] + public async Task Turning_Voice_off_cancels_the_idle_timer() + { + SetUpVoiceModule(); + var service = WakeService(); + ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMilliseconds(150); + var component = await RenderListeningAsync(service); + + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Off); + component.FindAll(".pdchat-voice-status").Should().BeEmpty(); + } + + /// Verifies that pressing the microphone while dormant turns Voice off. + [Fact] + public async Task Pressing_the_microphone_while_dormant_turns_Voice_off() + { + var module = SetUpVoiceModule(); + var component = await RenderDormantAsync(WakeService()); + + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + + module.Invocations["stop"].Should().ContainSingle(); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Off); + } + + private static FakeChatService WakeService(TimeSpan? autoSendDelay = null, bool readAloud = true) + { + var service = VoiceService(autoSendDelay ?? TimeSpan.FromMinutes(10), readAloud); + ((IChatService)service).WakePhrases = ["Hey DumbBot"]; + return service; + } + + // Once dormant, the idle timeout is lengthened so a woken microphone does not doze off again mid-test. + private async Task> RenderDormantAsync(FakeChatService service) + { + ((IChatService)service).VoiceIdleTimeout = _briefIdle; + var component = await RenderListeningAsync(service); + component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience); + ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMinutes(10); + return component; + } +} diff --git a/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs b/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs new file mode 100644 index 000000000..4d2e2ef0c --- /dev/null +++ b/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs @@ -0,0 +1,73 @@ +using AwesomeAssertions; +using PanoramicData.Blazor.Interfaces; +using PanoramicData.Blazor.Models; +using PanoramicData.Blazor.Services; + +namespace PanoramicData.Blazor.Test; + +/// +/// Tests that the voice, agent and model settings of can be changed, so the demo page can +/// show every option, and that each change is announced. +/// +public class DumbChatServiceVoiceTests +{ + /// Verifies the defaults the demo starts with, read through the interface so that the members are seen to bind. + [Fact] + public void The_demo_service_starts_with_the_documented_defaults() + { + using var service = new DumbChatService(); + IChatService chat = service; + + chat.VoiceEndpoints.Should().Be(PDChatVoiceEndpoints.Simulated); + chat.Agents.Should().HaveCount(2); + chat.Models.Should().HaveCount(2); + chat.WakePhrases.Should().BeNull(); + chat.VoiceIdleTimeout.Should().Be(TimeSpan.FromSeconds(10)); + chat.VoiceAutoSendDelay.Should().Be(TimeSpan.FromMilliseconds(1000)); + chat.IsReadAloudEnabled.Should().BeFalse(); + } + + /// Verifies that each setting, changed through the interface where it is settable there, announces the change once. + [Theory] + [MemberData(nameof(Settings))] + public void Changing_a_voice_setting_announces_it(string setting) + { + using var service = new DumbChatService(); + var announcements = 0; + service.OnConfigurationChanged += () => announcements++; + + _changes[setting](service); + _changes[setting](service); + + announcements.Should().Be(1, $"{setting} changed once and was then set to the same value"); + } + + /// Verifies that a wake phrase set through the interface reaches the demo service rather than the default store. + [Fact] + public void Wake_phrases_set_through_the_interface_are_the_services_own() + { + using var service = new DumbChatService(); + string[] phrases = ["Hey DumbBot"]; + + ((IChatService)service).WakePhrases = phrases; + + service.WakePhrases.Should().BeSameAs(phrases); + } + + public static TheoryData Settings() => [.. _changes.Keys]; + + private static readonly string[] _phrases = ["Hey DumbBot"]; + + private static readonly Dictionary> _changes = new() + { + [nameof(DumbChatService.VoiceEndpoints)] = service => service.VoiceEndpoints = null, + [nameof(DumbChatService.Agents)] = service => service.Agents = null, + [nameof(DumbChatService.Models)] = service => service.Models = null, + [nameof(IChatService.WakePhrases)] = service => ((IChatService)service).WakePhrases = _phrases, + [nameof(IChatService.VoiceIdleTimeout)] = service => ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromSeconds(3), + [nameof(IChatService.VoiceAutoSendDelay)] = service => ((IChatService)service).VoiceAutoSendDelay = TimeSpan.FromMilliseconds(250), + [nameof(IChatService.IsReadAloudEnabled)] = service => ((IChatService)service).IsReadAloudEnabled = true, + [nameof(IChatService.SelectedAgentId)] = service => ((IChatService)service).SelectedAgentId = "pedant", + [nameof(IChatService.SelectedModelId)] = service => ((IChatService)service).SelectedModelId = "dumb-max", + }; +} \ No newline at end of file diff --git a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs index e2544d62f..b9a402e4b 100644 --- a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs +++ b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs @@ -48,6 +48,10 @@ internal sealed class ChatServiceDefaultState public TimeSpan VoiceAutoSendDelay { get; set; } = TimeSpan.FromMilliseconds(1000); + public IReadOnlyList? WakePhrases { get; set; } + + public TimeSpan VoiceIdleTimeout { get; set; } = TimeSpan.FromSeconds(10); + public string? SelectedAgentId { get; set; } public string? SelectedModelId { get; set; } diff --git a/PanoramicData.Blazor/Interfaces/IChatService.cs b/PanoramicData.Blazor/Interfaces/IChatService.cs index 80fe70c0e..a20eca07e 100644 --- a/PanoramicData.Blazor/Interfaces/IChatService.cs +++ b/PanoramicData.Blazor/Interfaces/IChatService.cs @@ -104,6 +104,32 @@ TimeSpan VoiceAutoSendDelay set => ChatServiceDefaultState.For(this).VoiceAutoSendDelay = value; } + /// + /// Gets or sets the phrases that wake a dormant microphone, such as "Hey Merlin", or null or empty (the default) + /// for a microphone that never goes dormant. Matching ignores case and punctuation. + /// + /// + /// When set, Voice Mode goes dormant after with no words, and then types and sends + /// nothing until one of these phrases is heard; the words after it are typed as usual. The phrase itself is never + /// typed. While dormant the microphone keeps streaming to the speech service, which must hear the audio to + /// recognise the phrase, so a host should make clear to its users that a dormant microphone is still listening. + /// + IReadOnlyList? WakePhrases + { + get => ChatServiceDefaultState.For(this).WakePhrases; + set => ChatServiceDefaultState.For(this).WakePhrases = value; + } + + /// + /// Gets or sets how long Voice Mode may listen without hearing a word before the microphone goes dormant to wait + /// for one of the . Applies only when they are set. Defaults to ten seconds. + /// + TimeSpan VoiceIdleTimeout + { + get => ChatServiceDefaultState.For(this).VoiceIdleTimeout; + set => ChatServiceDefaultState.For(this).VoiceIdleTimeout = value; + } + /// /// Gets the agents the user may choose to talk to, or null (the default) when the host offers no choice. /// A picker is shown in the input area only when there are two or more. diff --git a/PanoramicData.Blazor/PDChat.Voice.cs b/PanoramicData.Blazor/PDChat.Voice.cs index 3fbb10146..233be3351 100644 --- a/PanoramicData.Blazor/PDChat.Voice.cs +++ b/PanoramicData.Blazor/PDChat.Voice.cs @@ -19,6 +19,12 @@ public enum PDChatVoiceState /// The answer is being spoken. Speaking, + + /// + /// Nothing has been said for , so nothing is typed or sent until one of + /// the is heard. The microphone is still on. + /// + Dormant, } /// @@ -56,9 +62,12 @@ public partial class PDChat private bool IsReadAloudOn => IsVoiceModeOffered && ChatService.IsReadAloudEnabled; - private string VoiceButtonTitle => IsVoiceModeOn - ? "Voice: stop listening" - : "Voice: speak instead of typing"; + private string VoiceButtonTitle => VoiceState switch + { + PDChatVoiceState.Off => "Voice: speak instead of typing", + PDChatVoiceState.Dormant => $"Voice: waiting for \"{FirstWakePhrase}\". Press to stop listening", + _ => "Voice: stop listening", + }; private string ReadAloudButtonTitle => IsReadAloudOn ? "Read answers aloud: on" @@ -74,6 +83,7 @@ public partial class PDChat : "Listening. Ask your question, then pause.", PDChatVoiceState.Thinking => "Thinking…", PDChatVoiceState.Speaking => "Reading the answer aloud.", + PDChatVoiceState.Dormant => $"Waiting for \"{FirstWakePhrase}\"…", _ => string.Empty, }; @@ -97,6 +107,7 @@ private async Task ToggleVoiceModeAsync() var module = await GetVoiceModuleAsync(endpoints); await module.InvokeVoidAsync("start", endpoints.ListenUrl, _voiceReference); VoiceState = PDChatVoiceState.Listening; + RestartIdleTimer(); } catch (JSException) { @@ -137,12 +148,15 @@ private async Task ToggleReadAloudAsync() private async Task ResumeListeningAsync() { VoiceState = PDChatVoiceState.Listening; + RestartIdleTimer(); await (_voiceModule?.InvokeVoidAsync("pause", false) ?? ValueTask.CompletedTask); } private async Task StopVoiceModeAsync() { CancelAutoSend(); + CancelIdleTimer(); + _wakeBuffer.Clear(); VoiceState = PDChatVoiceState.Off; _isAwaitingSpokenAnswer = false; _hasDictatedSinceTurn = false; @@ -157,14 +171,26 @@ private async Task StopVoiceModeAsync() [JSInvokable] public Task OnVoiceWord(string text) => InvokeAsync(async () => { + if (VoiceState == PDChatVoiceState.Dormant) + { + _hasDictatedSinceTurn = true; + text = HearWhileDormant(text); + } + if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input) { return; } + RestartIdleTimer(); + _hasDictatedSinceTurn = true; + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + // The speaker has carried on, so the pause that scheduled a send is over. CancelAutoSend(); - _hasDictatedSinceTurn = true; await input.AppendAsync(text); }); @@ -176,6 +202,15 @@ public Task OnVoiceWord(string text) => InvokeAsync(async () => [JSInvokable] public Task OnVoiceTurn(string text) => InvokeAsync(async () => { + var hasHeardWords = _hasDictatedSinceTurn; + _hasDictatedSinceTurn = false; + + // A host that reports only pauses can still wake a dormant microphone, from the turn's text. + if (VoiceState == PDChatVoiceState.Dormant && !hasHeardWords) + { + text = HearWhileDormant(text); + } + if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input) { return; @@ -184,12 +219,11 @@ public Task OnVoiceTurn(string text) => InvokeAsync(async () => CancelAutoSend(); // A host that reports no words still has its pause recorded. - if (!_hasDictatedSinceTurn) + if (!hasHeardWords) { await input.AppendAsync(text); } - _hasDictatedSinceTurn = false; if (!input.IsFocused && !string.IsNullOrWhiteSpace(input.Text)) { ScheduleAutoSend(); @@ -212,6 +246,7 @@ public Task OnVoiceError(string text) public Task OnVoiceClosed() { CancelAutoSend(); + CancelIdleTimer(); VoiceState = PDChatVoiceState.Off; _isAwaitingSpokenAnswer = false; VoiceError ??= "Voice Mode stopped: the connection closed."; @@ -344,6 +379,7 @@ private static string ToSpeakableText(ChatMessage message) private async ValueTask DisposeVoiceAsync() { CancelAutoSend(); + CancelIdleTimer(); if (_voiceModule is not null) { try diff --git a/PanoramicData.Blazor/PDChat.WakePhrase.cs b/PanoramicData.Blazor/PDChat.WakePhrase.cs new file mode 100644 index 000000000..f984856d0 --- /dev/null +++ b/PanoramicData.Blazor/PDChat.WakePhrase.cs @@ -0,0 +1,136 @@ +namespace PanoramicData.Blazor; + +/// +/// PDChat: the wake phrase. With set, Voice Mode goes +/// after without a word, and listens +/// again only once one of the phrases has been heard. +/// +public partial class PDChat +{ + private readonly List _wakeBuffer = []; + private CancellationTokenSource? _idleCancellation; + + private string FirstWakePhrase => ChatService.WakePhrases?.FirstOrDefault(phrase => !string.IsNullOrWhiteSpace(phrase))?.Trim() + ?? string.Empty; + + private List WakePhraseWords() => [.. (ChatService.WakePhrases ?? []) + .Select(phrase => ToWakeWords(phrase ?? string.Empty)) + .Where(words => words.Length > 0)]; + + private static string[] ToWakeWords(string text) => [.. text + .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries) + .Select(NormaliseWakeWord) + .Where(word => word.Length > 0)]; + + private static string NormaliseWakeWord(string word) + => new([.. word.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant)]); + + /// + /// Listens for a wake phrase in what was heard while dormant. Returns the words after the phrase, once it wakes, + /// or nothing while it stays dormant. + /// + private string HearWhileDormant(string text) + { + var phrases = WakePhraseWords(); + if (phrases.Count == 0) + { + // The host has removed its wake phrases, so there is nothing to wait for. + Wake(); + return text; + } + + var longest = phrases.Max(phrase => phrase.Length); + var words = text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + for (var index = 0; index < words.Length; index++) + { + var word = NormaliseWakeWord(words[index]); + if (word.Length == 0) + { + continue; + } + + _wakeBuffer.Add(word); + if (_wakeBuffer.Count > longest) + { + _wakeBuffer.RemoveAt(0); + } + + if (phrases.Any(EndsWithWakePhrase)) + { + Wake(); + return string.Join(' ', words[(index + 1)..]); + } + } + + return string.Empty; + } + + private bool EndsWithWakePhrase(string[] phrase) + => _wakeBuffer.Count >= phrase.Length + && _wakeBuffer.Skip(_wakeBuffer.Count - phrase.Length).SequenceEqual(phrase, StringComparer.Ordinal); + + private void Wake() + { + _wakeBuffer.Clear(); + VoiceState = PDChatVoiceState.Listening; + RestartIdleTimer(); + StateHasChanged(); + } + + private void RestartIdleTimer() + { + CancelIdleTimer(); + if (VoiceState != PDChatVoiceState.Listening || WakePhraseWords().Count == 0) + { + return; + } + + var cancellation = new CancellationTokenSource(); + _idleCancellation = cancellation; + _ = GoDormantWhenIdleAsync(cancellation.Token); + } + + private void CancelIdleTimer() + { + if (_idleCancellation is null) + { + return; + } + + _idleCancellation.Cancel(); + _idleCancellation.Dispose(); + _idleCancellation = null; + } + + private async Task GoDormantWhenIdleAsync(CancellationToken cancellationToken) + { + try + { + var timeout = ChatService.VoiceIdleTimeout; + await Task.Delay(timeout < TimeSpan.Zero ? TimeSpan.Zero : timeout, cancellationToken); + await InvokeAsync(() => + { + if (cancellationToken.IsCancellationRequested || VoiceState != PDChatVoiceState.Listening) + { + return; + } + + // The user is editing, or dictation is about to be sent: wait for another quiet spell. + if (VoiceInput?.IsFocused == true || _autoSendCancellation is not null) + { + RestartIdleTimer(); + return; + } + + CancelIdleTimer(); + _wakeBuffer.Clear(); + VoiceState = PDChatVoiceState.Dormant; + StateHasChanged(); + }); + } + catch (Exception ex) when (ex is JSDisconnectedException or OperationCanceledException or ObjectDisposedException) + { + // Cancelled by a word or by Voice going off, or the circuit is going away. + } + } +} diff --git a/PanoramicData.Blazor/PDChat.razor.css b/PanoramicData.Blazor/PDChat.razor.css index 187b0f43b..0b4531ebd 100644 --- a/PanoramicData.Blazor/PDChat.razor.css +++ b/PanoramicData.Blazor/PDChat.razor.css @@ -624,6 +624,11 @@ body:has(.pdchat-container.dock-fullscreen) { color: #dc3545; } +.pdchat-voice-status.pdchat-voice-dormant::before { + content: "○ "; + color: var(--pd-chat-muted-color, #6c757d); +} + .pdchat-voice-status.pdchat-voice-error { background: var(--pd-chat-warning-bg, #fff3cd); } diff --git a/PanoramicData.Blazor/Services/DumbChatService.Voice.cs b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs index 90d34e787..e0912a864 100644 --- a/PanoramicData.Blazor/Services/DumbChatService.Voice.cs +++ b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs @@ -1,25 +1,100 @@ +using System; + namespace PanoramicData.Blazor.Services; /// /// The voice controls and the agent and model pickers of , so a demo shows the whole input -/// toolbar without a speech service. +/// toolbar without a speech service. Each setting announces a change through . /// public partial class DumbChatService { - /// Gets or sets the voice endpoints; simulated by default, and null hides the voice controls. - public PDChatVoiceEndpoints? VoiceEndpoints { get; set; } = PDChatVoiceEndpoints.Simulated; + /// The wake phrases the demo suggests; its simulated microphone says the first of them every other time. + public static IReadOnlyList DemoWakePhrases { get; } = ["Hey DumbBot", "OK DumbBot"]; - /// Gets the agents offered in the input toolbar. - public IReadOnlyList? Agents { get; } = + /// The agents offered by default. + public static IReadOnlyList DemoAgents { get; } = [ new("dumbbot", "DumbBot", "Answers by keyword, and not very well."), new("pedant", "Pedant", "Exactly as helpful as DumbBot, but would like to correct your grammar first."), ]; - /// Gets the models offered in the input toolbar. The demo answers the same whichever is chosen. - public IReadOnlyList? Models { get; } = + /// The models offered by default. The demo answers the same whichever is chosen. + public static IReadOnlyList DemoModels { get; } = [ new("dumb-mini", "Dumb Mini", "Small and quick."), new("dumb-max", "Dumb Max", "Large and thorough. Equally dumb."), ]; + + private PDChatVoiceEndpoints? _voiceEndpoints = PDChatVoiceEndpoints.Simulated; + private IReadOnlyList? _agents = DemoAgents; + private IReadOnlyList? _models = DemoModels; + private IReadOnlyList? _wakePhrases; + private TimeSpan _voiceIdleTimeout = TimeSpan.FromSeconds(10); + private TimeSpan _voiceAutoSendDelay = TimeSpan.FromMilliseconds(1000); + private bool _isReadAloudEnabled; + private string? _selectedAgentId; + private string? _selectedModelId; + + /// Gets or sets the voice endpoints; simulated by default, and null hides the voice controls. + public PDChatVoiceEndpoints? VoiceEndpoints + { + get => _voiceEndpoints; + set => SetConfiguration(ref _voiceEndpoints, value); + } + + /// Gets or sets the agents offered in the input toolbar; by default. + public IReadOnlyList? Agents + { + get => _agents; + set => SetConfiguration(ref _agents, value); + } + + /// Gets or sets the models offered in the input toolbar; by default. + public IReadOnlyList? Models + { + get => _models; + set => SetConfiguration(ref _models, value); + } + + /// + public IReadOnlyList? WakePhrases + { + get => _wakePhrases; + set => SetConfiguration(ref _wakePhrases, value); + } + + /// + public TimeSpan VoiceIdleTimeout + { + get => _voiceIdleTimeout; + set => SetConfiguration(ref _voiceIdleTimeout, value); + } + + /// + public TimeSpan VoiceAutoSendDelay + { + get => _voiceAutoSendDelay; + set => SetConfiguration(ref _voiceAutoSendDelay, value); + } + + /// + public bool IsReadAloudEnabled + { + get => _isReadAloudEnabled; + set => SetConfiguration(ref _isReadAloudEnabled, value); + } + + /// + public string? SelectedAgentId + { + get => _selectedAgentId; + set => SetConfiguration(ref _selectedAgentId, value); + } + + /// + public string? SelectedModelId + { + get => _selectedModelId; + set => SetConfiguration(ref _selectedModelId, value); + } } diff --git a/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js index 5557b86df..75508ad65 100644 --- a/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js +++ b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js @@ -10,21 +10,29 @@ const PHRASES = [ "Testing, testing, one two three. Is this thing on?", ]; +// Every other utterance starts with the demo's wake phrase, so a dormant microphone can be seen to wake. +const WAKE_PHRASE = "Hey DumbBot,"; + const LOUD = 0.02; const WORD_MS = 220; const PAUSE_MS = 700; let listening = null; -let phraseIndex = 0; +let utteranceIndex = 0; let wordIndex = 0; +function utteranceWords(index) { + const phrase = PHRASES[index % PHRASES.length]; + return (index % 2 === 0 ? `${WAKE_PHRASE} ${phrase}` : phrase).split(" "); +} + function nextWord() { - const words = PHRASES[phraseIndex].split(" "); + const words = utteranceWords(utteranceIndex); const word = words[wordIndex]; wordIndex++; if (wordIndex >= words.length) { wordIndex = 0; - phraseIndex = (phraseIndex + 1) % PHRASES.length; + utteranceIndex = (utteranceIndex + 1) % (PHRASES.length * 2); } return word; } From 3d13d991c06bb6c1a900cf99f8dd911699c2ead6 Mon Sep 17 00:00:00 2001 From: David Bond Date: Tue, 6 Oct 2026 01:30:51 +0100 Subject: [PATCH 2/2] MS-26473 PDChat wake phrase tests: wait for the dormant render; document test data The dormant helper waited for VoiceState only, so on a slower runner the markup could still show the listening status when the test read it. It now waits for the rendered dormant status. Adds the XML comment CI requires on the public TheoryData member. Co-Authored-By: Claude Opus 5.5 --- .../Components/PDChatTests.WakePhrase.cs | 4 +++- PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs index c99323b02..1a0bc0a39 100644 --- a/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs @@ -243,7 +243,9 @@ private async Task> RenderDormantAsync(FakeChatServic { ((IChatService)service).VoiceIdleTimeout = _briefIdle; var component = await RenderListeningAsync(service); - component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience); + + // Wait for the render, not just the state, so tests reading the markup see the dormant status. + component.WaitForAssertion(() => component.Find(".pdchat-voice-status").ClassList.Should().Contain("pdchat-voice-dormant"), Patience); ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMinutes(10); return component; } diff --git a/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs b/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs index 4d2e2ef0c..c9f545517 100644 --- a/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs +++ b/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs @@ -54,6 +54,7 @@ public void Wake_phrases_set_through_the_interface_are_the_services_own() service.WakePhrases.Should().BeSameAs(phrases); } + /// The settable voice and assistant settings, by name. public static TheoryData Settings() => [.. _changes.Keys]; private static readonly string[] _phrases = ["Hey DumbBot"];