+ A thin toolbar sits above the text box. It holds an agent picker and a model picker (each shown only when
+ the service lists two or more), the notification sounds bell (IsMuted), and, when the service
+ supplies VoiceEndpoints, a read-aloud speaker and a microphone. Dictated words are typed into
+ the text box and sent after a pause unless the user is editing. All of these are configured on
+ IChatService and have defaults, so an existing service needs no change.
+
+
+
IChatService member
Type
Default
Description
+
+
Agents / SelectedAgentId
IReadOnlyList<PDChatAgentOption>? / string?
null
Who the user is talking to.
+
Models / SelectedModelId
IReadOnlyList<PDChatModelOption>? / string?
null
Which model answers.
+
VoiceEndpoints
PDChatVoiceEndpoints?
null
Where speech is sent and answers fetched; null hides the voice controls. PDChatVoiceEndpoints.Simulated needs no speech service.
+
IsReadAloudEnabled
bool
false
Reads the answer to each sent message aloud. Independent of the microphone and of notification sounds.
+
VoiceAutoSendDelay
TimeSpan
1 second
How long after a pause dictation is sent, when the user is not editing.
+
WakePhrases
IReadOnlyList<string>?
null (off)
Phrases such as "Hey DumbBot" that wake a dormant microphone. Case and punctuation are ignored, and the phrase itself is never typed.
+
VoiceIdleTimeout
TimeSpan
10 seconds
How long the microphone may hear nothing before it goes dormant. Applies only with wake phrases.
+
+
+
+ With wake phrases set, Voice Mode goes Dormant after the idle timeout (never while the user is
+ editing or a send is pending). While dormant nothing is typed or sent until a wake phrase is heard; the words
+ after it are typed as usual. The microphone keeps streaming to the speech service while dormant, since it
+ must hear the phrase, so tell your users that a dormant microphone is still listening. Try it with the
+ Voice and Assistant settings on this page: the simulated microphone starts every other
+ utterance with "Hey DumbBot".
+
Uncheck "Enable Message Input" above to see this. Leave blank for pure toast-only mode (the input simply disappears with no message).
+
+
+
Voice and Assistant
+
The input toolbar above the text box. The simulated microphone makes up what it hears, and starts every other utterance with "Hey DumbBot".
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
With Wake Phrase on, the microphone goes dormant after the idle time and types nothing until it hears one of these. It is still streaming audio while dormant.
diff --git a/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs b/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs
index e7fae61b1..cc44b2603 100644
--- a/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs
+++ b/PanoramicData.Blazor.Demo/Pages/PDChatPage.razor.cs
@@ -1,4 +1,6 @@
-namespace PanoramicData.Blazor.Demo.Pages;
+using PanoramicData.Blazor.Services;
+
+namespace PanoramicData.Blazor.Demo.Pages;
public partial class PDChatPage : IDisposable
{
@@ -23,6 +25,63 @@ private PDChatDockMode CurrentDockMode
private readonly Action _onDockModeChanged;
+ private string _wakePhraseText = string.Join(", ", DumbChatService.DemoWakePhrases);
+
+ // Voice endpoints, agents and models are read-only on IChatService, so the demo sets them on its own service.
+ private DumbChatService? DemoService => ChatService as DumbChatService;
+
+ private bool IsVoiceOffered
+ {
+ get => ChatService.VoiceEndpoints is not null;
+ set => DemoService?.VoiceEndpoints = value ? PDChatVoiceEndpoints.Simulated : null;
+ }
+
+ private bool IsAgentPickerShown
+ {
+ get => ChatService.Agents is { Count: > 1 };
+ set => DemoService?.Agents = value ? DumbChatService.DemoAgents : null;
+ }
+
+ private bool IsModelPickerShown
+ {
+ get => ChatService.Models is { Count: > 1 };
+ set => DemoService?.Models = value ? DumbChatService.DemoModels : null;
+ }
+
+ private int AutoSendDelayMs
+ {
+ get => (int)ChatService.VoiceAutoSendDelay.TotalMilliseconds;
+ set => ChatService.VoiceAutoSendDelay = TimeSpan.FromMilliseconds(Math.Max(0, value));
+ }
+
+ private double IdleTimeoutSeconds
+ {
+ get => ChatService.VoiceIdleTimeout.TotalSeconds;
+ set => ChatService.VoiceIdleTimeout = TimeSpan.FromSeconds(Math.Max(1, value));
+ }
+
+ private bool IsWakePhraseOn
+ {
+ get => ChatService.WakePhrases is { Count: > 0 };
+ set => ChatService.WakePhrases = value ? ParseWakePhrases(_wakePhraseText) : null;
+ }
+
+ private string WakePhraseText
+ {
+ get => _wakePhraseText;
+ set
+ {
+ _wakePhraseText = value;
+ if (IsWakePhraseOn)
+ {
+ ChatService.WakePhrases = ParseWakePhrases(value);
+ }
+ }
+ }
+
+ private static string[] ParseWakePhrases(string text)
+ => text.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
+
public PDChatPage()
{
// The new dock mode is read from the service when re-rendering
diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs
new file mode 100644
index 000000000..1a0bc0a39
--- /dev/null
+++ b/PanoramicData.Blazor.Test/Components/PDChatTests.WakePhrase.cs
@@ -0,0 +1,252 @@
+using AwesomeAssertions;
+using Bunit;
+using Microsoft.AspNetCore.Components.Web;
+using PanoramicData.Blazor.Interfaces;
+
+namespace PanoramicData.Blazor.Test;
+
+///
+/// The wake phrase: with wake phrases set, a microphone that hears nothing for the idle timeout goes dormant, types and
+/// sends nothing, and listens again only once a wake phrase is heard.
+///
+public partial class PDChatTests
+{
+ private static readonly TimeSpan _briefIdle = TimeSpan.FromMilliseconds(50);
+
+ /// Verifies that wake phrases are off and the idle timeout is ten seconds by default.
+ [Fact]
+ public void Wake_phrases_are_off_by_default()
+ {
+ IChatService service = new FakeChatService();
+
+ service.WakePhrases.Should().BeNull();
+ service.VoiceIdleTimeout.Should().Be(TimeSpan.FromSeconds(10));
+ }
+
+ /// Verifies that with wake phrases set, listening without a word goes dormant and the microphone stays on.
+ [Fact]
+ public async Task With_wake_phrases_an_idle_microphone_goes_dormant()
+ {
+ var module = SetUpVoiceModule();
+ var service = WakeService();
+ ((IChatService)service).VoiceIdleTimeout = _briefIdle;
+
+ var component = await RenderListeningAsync(service);
+
+ component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience);
+ module.Invocations["stop"].Should().BeEmpty();
+ component.Find(".pdchat-voice-toggle").GetAttribute("aria-pressed").Should().Be("true");
+ }
+
+ /// Verifies that without wake phrases the microphone never goes dormant.
+ [Fact]
+ public async Task Without_wake_phrases_the_microphone_never_goes_dormant()
+ {
+ SetUpVoiceModule();
+ var service = VoiceService(TimeSpan.FromMinutes(10));
+ ((IChatService)service).VoiceIdleTimeout = _briefIdle;
+
+ var component = await RenderListeningAsync(service);
+ await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken);
+
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+ }
+
+ /// Verifies that the status line and the microphone control say which phrase is awaited.
+ [Fact]
+ public async Task A_dormant_microphone_says_what_it_is_waiting_for()
+ {
+ SetUpVoiceModule();
+ var component = await RenderDormantAsync(WakeService());
+
+ component.Find(".pdchat-voice-status").TextContent.Trim().Should().Be("Waiting for \"Hey DumbBot\"…");
+ component.Find(".pdchat-voice-status").ClassList.Should().Contain("pdchat-voice-dormant");
+ var toggle = component.Find(".pdchat-voice-toggle");
+ toggle.GetAttribute("title").Should().Be("Voice: waiting for \"Hey DumbBot\". Press to stop listening");
+ toggle.GetAttribute("aria-label").Should().Be(toggle.GetAttribute("title"));
+ }
+
+ /// Verifies that while dormant, words are not typed and a pause never sends.
+ [Fact]
+ public async Task Dormant_words_are_not_typed_and_a_dormant_turn_never_sends()
+ {
+ SetUpVoiceModule();
+ var service = WakeService(TimeSpan.Zero);
+ var component = await RenderDormantAsync(service);
+
+ await SayAsync(component, "What", "time", "is", "it?");
+ await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken);
+
+ component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty();
+ service.Sent.Should().BeEmpty();
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant);
+ }
+
+ /// Verifies that the wake phrase, in any case and with punctuation, wakes it and only the words after it are sent.
+ [Fact]
+ public async Task The_wake_phrase_wakes_it_and_only_the_words_after_it_are_sent()
+ {
+ SetUpVoiceModule();
+ var service = WakeService(TimeSpan.Zero, readAloud: false);
+ var component = await RenderDormantAsync(service);
+
+ await SayAsync(component, "Ignore", "this.", "HEY", "dumbbot,", "is", "it", "done?");
+
+ component.WaitForAssertion(() => service.Sent.Should().ContainSingle().Which.Message.Should().Be("is it done?"), Patience);
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+ }
+
+ /// Verifies that a phrase of several words wakes it whether heard word by word or in one piece.
+ [Theory]
+ [InlineData("OK", "Dumb", "Bot.", "Is", "it", "done?")]
+ [InlineData("OK Dumb Bot. Is", "it", "done?")]
+ public async Task A_phrase_of_several_words_wakes_it(params string[] words)
+ {
+ SetUpVoiceModule();
+ var service = WakeService();
+ ((IChatService)service).WakePhrases = ["Hey DumbBot", "OK Dumb Bot"];
+ var component = await RenderDormantAsync(service);
+
+ foreach (var word in words)
+ {
+ await component.InvokeAsync(() => component.Instance.OnVoiceWord(word));
+ }
+
+ component.Find("textarea").GetAttribute("value").Should().Be("Is it done?");
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+ }
+
+ /// Verifies that the words of a phrase heard apart, or only in part, do not wake it.
+ [Theory]
+ [InlineData("Hey", "there", "DumbBot")]
+ [InlineData("Hey")]
+ [InlineData("DumbBot")]
+ public async Task A_partial_phrase_does_not_wake_it(params string[] words)
+ {
+ SetUpVoiceModule();
+ var component = await RenderDormantAsync(WakeService());
+
+ await SayAsync(component, words);
+
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant);
+ component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty();
+ }
+
+ /// Verifies that a wake phrase followed by a pause types nothing, so the phrase itself is never sent.
+ [Fact]
+ public async Task The_wake_phrase_alone_is_never_typed()
+ {
+ SetUpVoiceModule();
+ var service = WakeService(TimeSpan.Zero);
+ var component = await RenderDormantAsync(service);
+
+ await SayAsync(component, "Hey", "DumbBot");
+ await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken);
+
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+ component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty();
+ service.Sent.Should().BeEmpty();
+ }
+
+ /// Verifies that a host reporting only pauses can wake it, and the words after the phrase are typed.
+ [Fact]
+ public async Task A_host_that_reports_only_pauses_can_wake_it()
+ {
+ SetUpVoiceModule();
+ var component = await RenderDormantAsync(WakeService());
+
+ await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Nothing to see."));
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant);
+
+ await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Hey DumbBot, is it done?"));
+
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+ component.Find("textarea").GetAttribute("value").Should().Be("is it done?");
+ }
+
+ /// Verifies that each word restarts the idle timer, so a steady speaker never goes dormant.
+ [Fact]
+ public async Task Each_word_restarts_the_idle_timer()
+ {
+ SetUpVoiceModule();
+ var service = WakeService();
+ ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMilliseconds(400);
+ var component = await RenderListeningAsync(service);
+
+ for (var word = 0; word < 6; word++)
+ {
+ await Task.Delay(100, Xunit.TestContext.Current.CancellationToken);
+ await component.InvokeAsync(() => component.Instance.OnVoiceWord("more"));
+ }
+
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+ component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience);
+ }
+
+ /// Verifies that it never goes dormant while the user is editing the text box, and does once they stop.
+ [Fact]
+ public async Task It_does_not_go_dormant_while_the_text_box_has_focus()
+ {
+ SetUpVoiceModule();
+ var service = WakeService();
+ ((IChatService)service).VoiceIdleTimeout = _briefIdle;
+ var component = RenderChat(service);
+ await component.Find("textarea").FocusAsync(new FocusEventArgs());
+ await component.Find(".pdchat-voice-toggle").ClickAsync(new());
+
+ await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken);
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening);
+
+ await component.Find("textarea").BlurAsync(new FocusEventArgs());
+
+ component.WaitForAssertion(() => component.Instance.VoiceState.Should().Be(PDChatVoiceState.Dormant), Patience);
+ }
+
+ /// Verifies that turning Voice off cancels the idle timer, so it stays off rather than going dormant.
+ [Fact]
+ public async Task Turning_Voice_off_cancels_the_idle_timer()
+ {
+ SetUpVoiceModule();
+ var service = WakeService();
+ ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMilliseconds(150);
+ var component = await RenderListeningAsync(service);
+
+ await component.Find(".pdchat-voice-toggle").ClickAsync(new());
+ await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken);
+
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Off);
+ component.FindAll(".pdchat-voice-status").Should().BeEmpty();
+ }
+
+ /// Verifies that pressing the microphone while dormant turns Voice off.
+ [Fact]
+ public async Task Pressing_the_microphone_while_dormant_turns_Voice_off()
+ {
+ var module = SetUpVoiceModule();
+ var component = await RenderDormantAsync(WakeService());
+
+ await component.Find(".pdchat-voice-toggle").ClickAsync(new());
+
+ module.Invocations["stop"].Should().ContainSingle();
+ component.Instance.VoiceState.Should().Be(PDChatVoiceState.Off);
+ }
+
+ private static FakeChatService WakeService(TimeSpan? autoSendDelay = null, bool readAloud = true)
+ {
+ var service = VoiceService(autoSendDelay ?? TimeSpan.FromMinutes(10), readAloud);
+ ((IChatService)service).WakePhrases = ["Hey DumbBot"];
+ return service;
+ }
+
+ // Once dormant, the idle timeout is lengthened so a woken microphone does not doze off again mid-test.
+ private async Task> RenderDormantAsync(FakeChatService service)
+ {
+ ((IChatService)service).VoiceIdleTimeout = _briefIdle;
+ var component = await RenderListeningAsync(service);
+
+ // Wait for the render, not just the state, so tests reading the markup see the dormant status.
+ component.WaitForAssertion(() => component.Find(".pdchat-voice-status").ClassList.Should().Contain("pdchat-voice-dormant"), Patience);
+ ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromMinutes(10);
+ return component;
+ }
+}
diff --git a/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs b/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs
new file mode 100644
index 000000000..c9f545517
--- /dev/null
+++ b/PanoramicData.Blazor.Test/DumbChatServiceVoiceTests.cs
@@ -0,0 +1,74 @@
+using AwesomeAssertions;
+using PanoramicData.Blazor.Interfaces;
+using PanoramicData.Blazor.Models;
+using PanoramicData.Blazor.Services;
+
+namespace PanoramicData.Blazor.Test;
+
+///
+/// Tests that the voice, agent and model settings of can be changed, so the demo page can
+/// show every option, and that each change is announced.
+///
+public class DumbChatServiceVoiceTests
+{
+ /// Verifies the defaults the demo starts with, read through the interface so that the members are seen to bind.
+ [Fact]
+ public void The_demo_service_starts_with_the_documented_defaults()
+ {
+ using var service = new DumbChatService();
+ IChatService chat = service;
+
+ chat.VoiceEndpoints.Should().Be(PDChatVoiceEndpoints.Simulated);
+ chat.Agents.Should().HaveCount(2);
+ chat.Models.Should().HaveCount(2);
+ chat.WakePhrases.Should().BeNull();
+ chat.VoiceIdleTimeout.Should().Be(TimeSpan.FromSeconds(10));
+ chat.VoiceAutoSendDelay.Should().Be(TimeSpan.FromMilliseconds(1000));
+ chat.IsReadAloudEnabled.Should().BeFalse();
+ }
+
+ /// Verifies that each setting, changed through the interface where it is settable there, announces the change once.
+ [Theory]
+ [MemberData(nameof(Settings))]
+ public void Changing_a_voice_setting_announces_it(string setting)
+ {
+ using var service = new DumbChatService();
+ var announcements = 0;
+ service.OnConfigurationChanged += () => announcements++;
+
+ _changes[setting](service);
+ _changes[setting](service);
+
+ announcements.Should().Be(1, $"{setting} changed once and was then set to the same value");
+ }
+
+ /// Verifies that a wake phrase set through the interface reaches the demo service rather than the default store.
+ [Fact]
+ public void Wake_phrases_set_through_the_interface_are_the_services_own()
+ {
+ using var service = new DumbChatService();
+ string[] phrases = ["Hey DumbBot"];
+
+ ((IChatService)service).WakePhrases = phrases;
+
+ service.WakePhrases.Should().BeSameAs(phrases);
+ }
+
+ /// The settable voice and assistant settings, by name.
+ public static TheoryData Settings() => [.. _changes.Keys];
+
+ private static readonly string[] _phrases = ["Hey DumbBot"];
+
+ private static readonly Dictionary> _changes = new()
+ {
+ [nameof(DumbChatService.VoiceEndpoints)] = service => service.VoiceEndpoints = null,
+ [nameof(DumbChatService.Agents)] = service => service.Agents = null,
+ [nameof(DumbChatService.Models)] = service => service.Models = null,
+ [nameof(IChatService.WakePhrases)] = service => ((IChatService)service).WakePhrases = _phrases,
+ [nameof(IChatService.VoiceIdleTimeout)] = service => ((IChatService)service).VoiceIdleTimeout = TimeSpan.FromSeconds(3),
+ [nameof(IChatService.VoiceAutoSendDelay)] = service => ((IChatService)service).VoiceAutoSendDelay = TimeSpan.FromMilliseconds(250),
+ [nameof(IChatService.IsReadAloudEnabled)] = service => ((IChatService)service).IsReadAloudEnabled = true,
+ [nameof(IChatService.SelectedAgentId)] = service => ((IChatService)service).SelectedAgentId = "pedant",
+ [nameof(IChatService.SelectedModelId)] = service => ((IChatService)service).SelectedModelId = "dumb-max",
+ };
+}
\ No newline at end of file
diff --git a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs
index e2544d62f..b9a402e4b 100644
--- a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs
+++ b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs
@@ -48,6 +48,10 @@ internal sealed class ChatServiceDefaultState
public TimeSpan VoiceAutoSendDelay { get; set; } = TimeSpan.FromMilliseconds(1000);
+ public IReadOnlyList? WakePhrases { get; set; }
+
+ public TimeSpan VoiceIdleTimeout { get; set; } = TimeSpan.FromSeconds(10);
+
public string? SelectedAgentId { get; set; }
public string? SelectedModelId { get; set; }
diff --git a/PanoramicData.Blazor/Interfaces/IChatService.cs b/PanoramicData.Blazor/Interfaces/IChatService.cs
index 80fe70c0e..a20eca07e 100644
--- a/PanoramicData.Blazor/Interfaces/IChatService.cs
+++ b/PanoramicData.Blazor/Interfaces/IChatService.cs
@@ -104,6 +104,32 @@ TimeSpan VoiceAutoSendDelay
set => ChatServiceDefaultState.For(this).VoiceAutoSendDelay = value;
}
+ ///
+ /// Gets or sets the phrases that wake a dormant microphone, such as "Hey Merlin", or null or empty (the default)
+ /// for a microphone that never goes dormant. Matching ignores case and punctuation.
+ ///
+ ///
+ /// When set, Voice Mode goes dormant after with no words, and then types and sends
+ /// nothing until one of these phrases is heard; the words after it are typed as usual. The phrase itself is never
+ /// typed. While dormant the microphone keeps streaming to the speech service, which must hear the audio to
+ /// recognise the phrase, so a host should make clear to its users that a dormant microphone is still listening.
+ ///
+ IReadOnlyList? WakePhrases
+ {
+ get => ChatServiceDefaultState.For(this).WakePhrases;
+ set => ChatServiceDefaultState.For(this).WakePhrases = value;
+ }
+
+ ///
+ /// Gets or sets how long Voice Mode may listen without hearing a word before the microphone goes dormant to wait
+ /// for one of the . Applies only when they are set. Defaults to ten seconds.
+ ///
+ TimeSpan VoiceIdleTimeout
+ {
+ get => ChatServiceDefaultState.For(this).VoiceIdleTimeout;
+ set => ChatServiceDefaultState.For(this).VoiceIdleTimeout = value;
+ }
+
///
/// Gets the agents the user may choose to talk to, or null (the default) when the host offers no choice.
/// A picker is shown in the input area only when there are two or more.
diff --git a/PanoramicData.Blazor/PDChat.Voice.cs b/PanoramicData.Blazor/PDChat.Voice.cs
index 3fbb10146..233be3351 100644
--- a/PanoramicData.Blazor/PDChat.Voice.cs
+++ b/PanoramicData.Blazor/PDChat.Voice.cs
@@ -19,6 +19,12 @@ public enum PDChatVoiceState
/// The answer is being spoken.
Speaking,
+
+ ///
+ /// Nothing has been said for , so nothing is typed or sent until one of
+ /// the is heard. The microphone is still on.
+ ///
+ Dormant,
}
///
@@ -56,9 +62,12 @@ public partial class PDChat
private bool IsReadAloudOn => IsVoiceModeOffered && ChatService.IsReadAloudEnabled;
- private string VoiceButtonTitle => IsVoiceModeOn
- ? "Voice: stop listening"
- : "Voice: speak instead of typing";
+ private string VoiceButtonTitle => VoiceState switch
+ {
+ PDChatVoiceState.Off => "Voice: speak instead of typing",
+ PDChatVoiceState.Dormant => $"Voice: waiting for \"{FirstWakePhrase}\". Press to stop listening",
+ _ => "Voice: stop listening",
+ };
private string ReadAloudButtonTitle => IsReadAloudOn
? "Read answers aloud: on"
@@ -74,6 +83,7 @@ public partial class PDChat
: "Listening. Ask your question, then pause.",
PDChatVoiceState.Thinking => "Thinking…",
PDChatVoiceState.Speaking => "Reading the answer aloud.",
+ PDChatVoiceState.Dormant => $"Waiting for \"{FirstWakePhrase}\"…",
_ => string.Empty,
};
@@ -97,6 +107,7 @@ private async Task ToggleVoiceModeAsync()
var module = await GetVoiceModuleAsync(endpoints);
await module.InvokeVoidAsync("start", endpoints.ListenUrl, _voiceReference);
VoiceState = PDChatVoiceState.Listening;
+ RestartIdleTimer();
}
catch (JSException)
{
@@ -137,12 +148,15 @@ private async Task ToggleReadAloudAsync()
private async Task ResumeListeningAsync()
{
VoiceState = PDChatVoiceState.Listening;
+ RestartIdleTimer();
await (_voiceModule?.InvokeVoidAsync("pause", false) ?? ValueTask.CompletedTask);
}
private async Task StopVoiceModeAsync()
{
CancelAutoSend();
+ CancelIdleTimer();
+ _wakeBuffer.Clear();
VoiceState = PDChatVoiceState.Off;
_isAwaitingSpokenAnswer = false;
_hasDictatedSinceTurn = false;
@@ -157,14 +171,26 @@ private async Task StopVoiceModeAsync()
[JSInvokable]
public Task OnVoiceWord(string text) => InvokeAsync(async () =>
{
+ if (VoiceState == PDChatVoiceState.Dormant)
+ {
+ _hasDictatedSinceTurn = true;
+ text = HearWhileDormant(text);
+ }
+
if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input)
{
return;
}
+ RestartIdleTimer();
+ _hasDictatedSinceTurn = true;
+ if (string.IsNullOrWhiteSpace(text))
+ {
+ return;
+ }
+
// The speaker has carried on, so the pause that scheduled a send is over.
CancelAutoSend();
- _hasDictatedSinceTurn = true;
await input.AppendAsync(text);
});
@@ -176,6 +202,15 @@ public Task OnVoiceWord(string text) => InvokeAsync(async () =>
[JSInvokable]
public Task OnVoiceTurn(string text) => InvokeAsync(async () =>
{
+ var hasHeardWords = _hasDictatedSinceTurn;
+ _hasDictatedSinceTurn = false;
+
+ // A host that reports only pauses can still wake a dormant microphone, from the turn's text.
+ if (VoiceState == PDChatVoiceState.Dormant && !hasHeardWords)
+ {
+ text = HearWhileDormant(text);
+ }
+
if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input)
{
return;
@@ -184,12 +219,11 @@ public Task OnVoiceTurn(string text) => InvokeAsync(async () =>
CancelAutoSend();
// A host that reports no words still has its pause recorded.
- if (!_hasDictatedSinceTurn)
+ if (!hasHeardWords)
{
await input.AppendAsync(text);
}
- _hasDictatedSinceTurn = false;
if (!input.IsFocused && !string.IsNullOrWhiteSpace(input.Text))
{
ScheduleAutoSend();
@@ -212,6 +246,7 @@ public Task OnVoiceError(string text)
public Task OnVoiceClosed()
{
CancelAutoSend();
+ CancelIdleTimer();
VoiceState = PDChatVoiceState.Off;
_isAwaitingSpokenAnswer = false;
VoiceError ??= "Voice Mode stopped: the connection closed.";
@@ -344,6 +379,7 @@ private static string ToSpeakableText(ChatMessage message)
private async ValueTask DisposeVoiceAsync()
{
CancelAutoSend();
+ CancelIdleTimer();
if (_voiceModule is not null)
{
try
diff --git a/PanoramicData.Blazor/PDChat.WakePhrase.cs b/PanoramicData.Blazor/PDChat.WakePhrase.cs
new file mode 100644
index 000000000..f984856d0
--- /dev/null
+++ b/PanoramicData.Blazor/PDChat.WakePhrase.cs
@@ -0,0 +1,136 @@
+namespace PanoramicData.Blazor;
+
+///
+/// PDChat: the wake phrase. With set, Voice Mode goes
+/// after without a word, and listens
+/// again only once one of the phrases has been heard.
+///
+public partial class PDChat
+{
+ private readonly List _wakeBuffer = [];
+ private CancellationTokenSource? _idleCancellation;
+
+ private string FirstWakePhrase => ChatService.WakePhrases?.FirstOrDefault(phrase => !string.IsNullOrWhiteSpace(phrase))?.Trim()
+ ?? string.Empty;
+
+ private List WakePhraseWords() => [.. (ChatService.WakePhrases ?? [])
+ .Select(phrase => ToWakeWords(phrase ?? string.Empty))
+ .Where(words => words.Length > 0)];
+
+ private static string[] ToWakeWords(string text) => [.. text
+ .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)
+ .Select(NormaliseWakeWord)
+ .Where(word => word.Length > 0)];
+
+ private static string NormaliseWakeWord(string word)
+ => new([.. word.Where(char.IsLetterOrDigit).Select(char.ToLowerInvariant)]);
+
+ ///
+ /// Listens for a wake phrase in what was heard while dormant. Returns the words after the phrase, once it wakes,
+ /// or nothing while it stays dormant.
+ ///
+ private string HearWhileDormant(string text)
+ {
+ var phrases = WakePhraseWords();
+ if (phrases.Count == 0)
+ {
+ // The host has removed its wake phrases, so there is nothing to wait for.
+ Wake();
+ return text;
+ }
+
+ var longest = phrases.Max(phrase => phrase.Length);
+ var words = text.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
+ for (var index = 0; index < words.Length; index++)
+ {
+ var word = NormaliseWakeWord(words[index]);
+ if (word.Length == 0)
+ {
+ continue;
+ }
+
+ _wakeBuffer.Add(word);
+ if (_wakeBuffer.Count > longest)
+ {
+ _wakeBuffer.RemoveAt(0);
+ }
+
+ if (phrases.Any(EndsWithWakePhrase))
+ {
+ Wake();
+ return string.Join(' ', words[(index + 1)..]);
+ }
+ }
+
+ return string.Empty;
+ }
+
+ private bool EndsWithWakePhrase(string[] phrase)
+ => _wakeBuffer.Count >= phrase.Length
+ && _wakeBuffer.Skip(_wakeBuffer.Count - phrase.Length).SequenceEqual(phrase, StringComparer.Ordinal);
+
+ private void Wake()
+ {
+ _wakeBuffer.Clear();
+ VoiceState = PDChatVoiceState.Listening;
+ RestartIdleTimer();
+ StateHasChanged();
+ }
+
+ private void RestartIdleTimer()
+ {
+ CancelIdleTimer();
+ if (VoiceState != PDChatVoiceState.Listening || WakePhraseWords().Count == 0)
+ {
+ return;
+ }
+
+ var cancellation = new CancellationTokenSource();
+ _idleCancellation = cancellation;
+ _ = GoDormantWhenIdleAsync(cancellation.Token);
+ }
+
+ private void CancelIdleTimer()
+ {
+ if (_idleCancellation is null)
+ {
+ return;
+ }
+
+ _idleCancellation.Cancel();
+ _idleCancellation.Dispose();
+ _idleCancellation = null;
+ }
+
+ private async Task GoDormantWhenIdleAsync(CancellationToken cancellationToken)
+ {
+ try
+ {
+ var timeout = ChatService.VoiceIdleTimeout;
+ await Task.Delay(timeout < TimeSpan.Zero ? TimeSpan.Zero : timeout, cancellationToken);
+ await InvokeAsync(() =>
+ {
+ if (cancellationToken.IsCancellationRequested || VoiceState != PDChatVoiceState.Listening)
+ {
+ return;
+ }
+
+ // The user is editing, or dictation is about to be sent: wait for another quiet spell.
+ if (VoiceInput?.IsFocused == true || _autoSendCancellation is not null)
+ {
+ RestartIdleTimer();
+ return;
+ }
+
+ CancelIdleTimer();
+ _wakeBuffer.Clear();
+ VoiceState = PDChatVoiceState.Dormant;
+ StateHasChanged();
+ });
+ }
+ catch (Exception ex) when (ex is JSDisconnectedException or OperationCanceledException or ObjectDisposedException)
+ {
+ // Cancelled by a word or by Voice going off, or the circuit is going away.
+ }
+ }
+}
diff --git a/PanoramicData.Blazor/PDChat.razor.css b/PanoramicData.Blazor/PDChat.razor.css
index 187b0f43b..0b4531ebd 100644
--- a/PanoramicData.Blazor/PDChat.razor.css
+++ b/PanoramicData.Blazor/PDChat.razor.css
@@ -624,6 +624,11 @@ body:has(.pdchat-container.dock-fullscreen) {
color: #dc3545;
}
+.pdchat-voice-status.pdchat-voice-dormant::before {
+ content: "○ ";
+ color: var(--pd-chat-muted-color, #6c757d);
+}
+
.pdchat-voice-status.pdchat-voice-error {
background: var(--pd-chat-warning-bg, #fff3cd);
}
diff --git a/PanoramicData.Blazor/Services/DumbChatService.Voice.cs b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs
index 90d34e787..e0912a864 100644
--- a/PanoramicData.Blazor/Services/DumbChatService.Voice.cs
+++ b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs
@@ -1,25 +1,100 @@
+using System;
+
namespace PanoramicData.Blazor.Services;
///
/// The voice controls and the agent and model pickers of , so a demo shows the whole input
-/// toolbar without a speech service.
+/// toolbar without a speech service. Each setting announces a change through .
///
public partial class DumbChatService
{
- /// Gets or sets the voice endpoints; simulated by default, and null hides the voice controls.
- public PDChatVoiceEndpoints? VoiceEndpoints { get; set; } = PDChatVoiceEndpoints.Simulated;
+ /// The wake phrases the demo suggests; its simulated microphone says the first of them every other time.
+ public static IReadOnlyList DemoWakePhrases { get; } = ["Hey DumbBot", "OK DumbBot"];
- /// Gets the agents offered in the input toolbar.
- public IReadOnlyList? Agents { get; } =
+ /// The agents offered by default.
+ public static IReadOnlyList DemoAgents { get; } =
[
new("dumbbot", "DumbBot", "Answers by keyword, and not very well."),
new("pedant", "Pedant", "Exactly as helpful as DumbBot, but would like to correct your grammar first."),
];
- /// Gets the models offered in the input toolbar. The demo answers the same whichever is chosen.
- public IReadOnlyList? Models { get; } =
+ /// The models offered by default. The demo answers the same whichever is chosen.
+ public static IReadOnlyList DemoModels { get; } =
[
new("dumb-mini", "Dumb Mini", "Small and quick."),
new("dumb-max", "Dumb Max", "Large and thorough. Equally dumb."),
];
+
+ private PDChatVoiceEndpoints? _voiceEndpoints = PDChatVoiceEndpoints.Simulated;
+ private IReadOnlyList? _agents = DemoAgents;
+ private IReadOnlyList? _models = DemoModels;
+ private IReadOnlyList? _wakePhrases;
+ private TimeSpan _voiceIdleTimeout = TimeSpan.FromSeconds(10);
+ private TimeSpan _voiceAutoSendDelay = TimeSpan.FromMilliseconds(1000);
+ private bool _isReadAloudEnabled;
+ private string? _selectedAgentId;
+ private string? _selectedModelId;
+
+ /// Gets or sets the voice endpoints; simulated by default, and null hides the voice controls.
+ public PDChatVoiceEndpoints? VoiceEndpoints
+ {
+ get => _voiceEndpoints;
+ set => SetConfiguration(ref _voiceEndpoints, value);
+ }
+
+ /// Gets or sets the agents offered in the input toolbar; by default.
+ public IReadOnlyList? Agents
+ {
+ get => _agents;
+ set => SetConfiguration(ref _agents, value);
+ }
+
+ /// Gets or sets the models offered in the input toolbar; by default.
+ public IReadOnlyList? Models
+ {
+ get => _models;
+ set => SetConfiguration(ref _models, value);
+ }
+
+ ///
+ public IReadOnlyList? WakePhrases
+ {
+ get => _wakePhrases;
+ set => SetConfiguration(ref _wakePhrases, value);
+ }
+
+ ///
+ public TimeSpan VoiceIdleTimeout
+ {
+ get => _voiceIdleTimeout;
+ set => SetConfiguration(ref _voiceIdleTimeout, value);
+ }
+
+ ///
+ public TimeSpan VoiceAutoSendDelay
+ {
+ get => _voiceAutoSendDelay;
+ set => SetConfiguration(ref _voiceAutoSendDelay, value);
+ }
+
+ ///
+ public bool IsReadAloudEnabled
+ {
+ get => _isReadAloudEnabled;
+ set => SetConfiguration(ref _isReadAloudEnabled, value);
+ }
+
+ ///
+ public string? SelectedAgentId
+ {
+ get => _selectedAgentId;
+ set => SetConfiguration(ref _selectedAgentId, value);
+ }
+
+ ///
+ public string? SelectedModelId
+ {
+ get => _selectedModelId;
+ set => SetConfiguration(ref _selectedModelId, value);
+ }
}
diff --git a/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js
index 5557b86df..75508ad65 100644
--- a/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js
+++ b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js
@@ -10,21 +10,29 @@ const PHRASES = [
"Testing, testing, one two three. Is this thing on?",
];
+// Every other utterance starts with the demo's wake phrase, so a dormant microphone can be seen to wake.
+const WAKE_PHRASE = "Hey DumbBot,";
+
const LOUD = 0.02;
const WORD_MS = 220;
const PAUSE_MS = 700;
let listening = null;
-let phraseIndex = 0;
+let utteranceIndex = 0;
let wordIndex = 0;
+function utteranceWords(index) {
+ const phrase = PHRASES[index % PHRASES.length];
+ return (index % 2 === 0 ? `${WAKE_PHRASE} ${phrase}` : phrase).split(" ");
+}
+
function nextWord() {
- const words = PHRASES[phraseIndex].split(" ");
+ const words = utteranceWords(utteranceIndex);
const word = words[wordIndex];
wordIndex++;
if (wordIndex >= words.length) {
wordIndex = 0;
- phraseIndex = (phraseIndex + 1) % PHRASES.length;
+ utteranceIndex = (utteranceIndex + 1) % (PHRASES.length * 2);
}
return word;
}