From 7a1261908720e5ff2456d1b5bd6bd04213512a1b Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 5 Oct 2026 22:46:26 +0100 Subject: [PATCH 1/2] MS-26473 PDChat: Voice in the input area, dictation with auto-send, agent picker - Voice control moves from the header to the input row beside Send. - Recognised words are dictated into the text box; a pause sends after IChatService.VoiceAutoSendDelay (default 1000 ms) unless the box has focus. - New IChatInput abstraction, implemented by PDMessages (Text, IsFocused, AppendAsync, SendAsync), plus PDMessages parameters InputAccessories, InputFocusChanged and IsInputAutoFocused. - Optional agent picker: IChatService.Agents and SelectedAgentId, shown with two or more PDChatAgentOption entries. - The microphone is still requested only when Voice is first turned on. All public API changes are additive; new IChatService members are default interface members. Co-Authored-By: Claude Opus 5.5 --- .../Components/PDChatTests.Agents.cs | 90 +++++++ .../Components/PDChatTests.Fakes.cs | 1 + .../Components/PDChatTests.Voice.cs | 235 ++++++++++++++++-- .../Components/PDMessagesTests.ChatInput.cs | 108 ++++++++ .../Interfaces/ChatServiceDefaultState.cs | 4 + PanoramicData.Blazor/Interfaces/IChatInput.cs | 24 ++ .../Interfaces/IChatService.cs | 27 ++ .../Models/PDChatAgentOption.cs | 11 + PanoramicData.Blazor/PDChat.Agents.cs | 25 ++ PanoramicData.Blazor/PDChat.Voice.cs | 153 ++++++++++-- PanoramicData.Blazor/PDChat.razor | 51 +++- PanoramicData.Blazor/PDChat.razor.cs | 3 + PanoramicData.Blazor/PDChat.razor.css | 26 +- PanoramicData.Blazor/PDMessages.Interop.cs | 5 +- PanoramicData.Blazor/PDMessages.razor | 6 + PanoramicData.Blazor/PDMessages.razor.cs | 51 +++- PanoramicData.Blazor/PDMessages.razor.css | 11 + 17 files changed, 762 insertions(+), 69 deletions(-) create mode 100644 PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs create mode 100644 PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs create mode 100644 PanoramicData.Blazor/Interfaces/IChatInput.cs create mode 100644 PanoramicData.Blazor/Models/PDChatAgentOption.cs create mode 100644 PanoramicData.Blazor/PDChat.Agents.cs diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs new file mode 100644 index 000000000..095de2328 --- /dev/null +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Agents.cs @@ -0,0 +1,90 @@ +using AwesomeAssertions; +using Bunit; +using Microsoft.AspNetCore.Components; +using PanoramicData.Blazor.Interfaces; +using PanoramicData.Blazor.Models; + +namespace PanoramicData.Blazor.Test; + +/// +/// The agent picker: offered in the input area only when the service lists two or more agents, and choosing one sets +/// . +/// +public partial class PDChatTests +{ + private static readonly PDChatAgentOption _merlin = new("merlin", "Merlin", "Answers about the product", "/merlin.png"); + private static readonly PDChatAgentOption _alice = new("alice", "Alice", "Development help"); + + /// Verifies that no picker is shown with no agents, or with only one to choose. + [Theory] + [InlineData(0)] + [InlineData(1)] + public void The_agent_picker_is_hidden_with_fewer_than_two_agents(int count) + { + var component = RenderChat(new FakeChatService { Agents = [.. new[] { _merlin, _alice }.Take(count)] }); + + component.FindAll(".pdchat-agent-picker").Should().BeEmpty(); + component.FindAll(".chat-input-accessories").Should().BeEmpty("there is nothing else to show there"); + } + + /// Verifies that no picker is shown when the service does not list agents at all. + [Fact] + public void The_agent_picker_is_hidden_without_agents() + => RenderChat(new FakeChatService()).FindAll(".pdchat-agent-picker").Should().BeEmpty(); + + /// Verifies that two or more agents are offered in the input area, the first selected by default. + [Fact] + public void The_agent_picker_is_shown_in_the_input_area_with_two_agents() + { + var component = RenderChat(new FakeChatService { Agents = [_merlin, _alice] }); + + var select = component.Find(".chat-input-container .chat-input-accessories .pdchat-agent-picker select"); + select.GetAttribute("title").Should().Be("Who you are talking to"); + var options = select.QuerySelectorAll("option"); + options.Select(option => option.TextContent).Should().Equal("Merlin", "Alice"); + options.Select(option => option.GetAttribute("value")).Should().Equal("merlin", "alice"); + options[0].GetAttribute("title").Should().Be("Answers about the product"); + options[0].HasAttribute("selected").Should().BeTrue(); + component.Find(".pdchat-agent-icon").GetAttribute("src").Should().Be("/merlin.png"); + } + + /// Verifies that choosing an agent sets the service's selected agent, and the picker follows it. + [Fact] + public async Task Choosing_an_agent_sets_the_selected_agent() + { + var service = new FakeChatService { Agents = [_merlin, _alice] }; + var component = RenderChat(service); + ((IChatService)service).SelectedAgentId.Should().BeNull(); + + await component.Find(".pdchat-agent-picker select").ChangeAsync(new ChangeEventArgs { Value = "alice" }); + + ((IChatService)service).SelectedAgentId.Should().Be("alice"); + component.WaitForAssertion(() => component.Find("option[value='alice']").HasAttribute("selected").Should().BeTrue(), Patience); + component.FindAll(".pdchat-agent-icon").Should().BeEmpty("Alice has no icon"); + } + + /// Verifies that a value that is not one of the agents is ignored. + [Fact] + public async Task An_unknown_agent_is_not_selected() + { + var service = new FakeChatService { Agents = [_merlin, _alice] }; + var component = RenderChat(service); + + await component.Find(".pdchat-agent-picker select").ChangeAsync(new ChangeEventArgs { Value = "mallory" }); + + ((IChatService)service).SelectedAgentId.Should().BeNull(); + } + + /// Verifies that the picker and the Voice control share the input area. + [Fact] + public void The_agent_picker_sits_beside_the_Voice_control() + { + var service = VoiceService(); + service.Agents = [_merlin, _alice]; + + var accessories = RenderChat(service).Find(".chat-input-accessories"); + + accessories.QuerySelector(".pdchat-agent-picker").Should().NotBeNull(); + accessories.QuerySelector(".pdchat-voice-toggle").Should().NotBeNull(); + } +} diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs index 34a56d816..319da5a7a 100644 --- a/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Fakes.cs @@ -55,6 +55,7 @@ private sealed partial class FakeChatService : IChatService public int ToastMaxVisible { get; set; } = 3; public PDChatButtonPosition ToastAnchor { get; set; } = PDChatButtonPosition.BottomRight; public PDChatVoiceEndpoints? VoiceEndpoints { get; set; } + public IReadOnlyList? Agents { get; set; } public IReadOnlyList Messages => Store; public bool SupportsConversations { get; init; } diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs index 45d527daa..dd6794004 100644 --- a/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs @@ -1,5 +1,6 @@ using AwesomeAssertions; using Bunit; +using Microsoft.AspNetCore.Components.Web; using Microsoft.JSInterop; using PanoramicData.Blazor.Interfaces; using PanoramicData.Blazor.Models; @@ -7,14 +8,17 @@ namespace PanoramicData.Blazor.Test; /// -/// Voice Mode: offered only when the service supplies endpoints, off by default, and when on, a spoken question is -/// sent like a typed one and the first finished reply is read aloud. +/// Voice Mode: offered in the input area only when the service supplies endpoints, off by default. When on, words are +/// dictated into the text box, sent after a pause unless the user is editing, and the first finished reply is read aloud. /// public partial class PDChatTests { private const string VoiceModulePath = "./_content/PanoramicData.Blazor/js/pdchat-voice.js"; private static readonly PDChatVoiceEndpoints _voiceEndpoints = new("/voice/listen", "/voice/speak"); + /// Long enough that a send which should not happen would have. + private static readonly TimeSpan _settle = TimeSpan.FromMilliseconds(400); + /// Verifies that a service without voice endpoints offers no Voice Mode at all. [Fact] public void Without_voice_endpoints_there_is_no_Voice_Mode_control() @@ -24,21 +28,57 @@ public void Without_voice_endpoints_there_is_no_Voice_Mode_control() [Fact] public void Voice_Mode_is_offered_but_off_by_default() { - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); + var component = RenderChat(VoiceService()); var toggle = component.Find(".pdchat-voice-toggle"); toggle.TextContent.Should().Contain("Voice"); - toggle.GetAttribute("title").Should().Be("Turn Voice Mode on: ask out loud and hear the answer"); + toggle.GetAttribute("title").Should().Be("Voice: speak instead of typing"); toggle.GetAttribute("aria-pressed").Should().Be("false"); component.FindAll(".pdchat-voice-status").Should().BeEmpty(); component.Instance.VoiceState.Should().Be(PDChatVoiceState.Off); } + /// Verifies that the Voice control sits in the input area beside Send, not in the header. + [Fact] + public void The_Voice_control_is_in_the_input_area() + { + var component = RenderChat(VoiceService()); + + component.FindAll(".pdchat-header .pdchat-voice-toggle").Should().BeEmpty(); + component.FindAll(".chat-input-container .chat-input-accessories .pdchat-voice-toggle").Should().ContainSingle(); + } + + /// + /// Verifies that the microphone is not asked for until Voice is first turned on: rendering the chat, its input area + /// and the agent picker loads no voice module and starts nothing. + /// + [Fact] + public async Task The_microphone_is_not_requested_until_Voice_is_turned_on() + { + var module = SetUpVoiceModule(); + var service = VoiceService(); + service.Agents = [new("merlin", "Merlin"), new("alice", "Alice")]; + var component = RenderChat(service); + component.Find(".pdchat-voice-toggle"); + await component.Find(".pdchat-agent-picker select").ChangeAsync(new Microsoft.AspNetCore.Components.ChangeEventArgs { Value = "alice" }); + + VoiceModuleImports().Should().BeEmpty(); + module.Invocations.Should().BeEmpty(); + + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + + VoiceModuleImports().Should().ContainSingle(); + module.Invocations["start"].Should().ContainSingle(); + } + + private IEnumerable VoiceModuleImports() + => JSInterop.Invocations["import"].Where(call => VoiceModulePath.Equals(call.Arguments[0])); + /// Verifies that a chat whose input is not permitted offers no Voice Mode either. [Fact] public void Voice_Mode_is_not_offered_where_typing_is_not_permitted() { - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; + var service = VoiceService(); ((IChatService)service).IsInputPermitted = false; RenderChat(service).FindAll(".pdchat-voice-toggle").Should().BeEmpty(); @@ -49,43 +89,158 @@ public void Voice_Mode_is_not_offered_where_typing_is_not_permitted() public async Task Turning_Voice_Mode_on_starts_listening() { var module = SetUpVoiceModule(); - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); + var component = RenderChat(VoiceService()); await component.Find(".pdchat-voice-toggle").ClickAsync(new()); module.Invocations["start"].Should().ContainSingle().Which.Arguments[0].Should().Be("/voice/listen"); component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); component.Find(".pdchat-voice-toggle").GetAttribute("aria-pressed").Should().Be("true"); + component.Find(".pdchat-voice-toggle").GetAttribute("title").Should().Be("Voice: stop listening"); component.Find(".pdchat-voice-status").TextContent.Trim().Should().Be("Listening. Ask your question, then pause."); } - /// Verifies that a spoken question is sent as the user, as typed text is, and the microphone pauses. + /// Verifies that the auto-send delay defaults to one second and can be set on a service that does not implement it. + [Fact] + public void The_auto_send_delay_defaults_to_one_second() + { + IChatService service = new FakeChatService(); + + service.VoiceAutoSendDelay.Should().Be(TimeSpan.FromMilliseconds(1000)); + + service.VoiceAutoSendDelay = TimeSpan.FromMilliseconds(250); + service.VoiceAutoSendDelay.Should().Be(TimeSpan.FromMilliseconds(250)); + } + + /// Verifies that recognised words are dictated into the text box, and nothing is sent as they arrive. [Fact] - public async Task A_spoken_question_is_sent_like_a_typed_one() + public async Task Dictated_words_fill_the_text_box() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMinutes(10)); + var component = await RenderListeningAsync(service); + + await component.InvokeAsync(() => component.Instance.OnVoiceWord("Is")); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("MS-26473")); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("done?")); + + component.Find("textarea").GetAttribute("value").Should().Be("Is MS-26473 done?"); + service.Sent.Should().BeEmpty(); + } + + /// Verifies that the pause does not repeat words already dictated, and a host that reports only pauses still fills the box. + [Fact] + public async Task A_pause_adds_only_what_was_not_dictated() + { + SetUpVoiceModule(); + var component = await RenderListeningAsync(VoiceService(TimeSpan.FromMinutes(10))); + + await SayAsync(component, "Is", "it", "done?"); + await component.InvokeAsync(() => component.Instance.OnVoiceTurn("And the other one?")); + + component.Find("textarea").GetAttribute("value").Should().Be("Is it done? And the other one?"); + } + + /// Verifies that after a pause, with the text box not focused, the text is sent as the user once the delay has passed. + [Fact] + public async Task Dictation_is_sent_after_the_delay_when_the_text_box_is_not_focused() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; - var component = RenderChat(service); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + var service = VoiceService(TimeSpan.FromMilliseconds(150)); + var component = await RenderListeningAsync(service); - await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Is MS-26473 done?")); + await SayAsync(component, "Is", "it", "done?"); + service.Sent.Should().BeEmpty("the delay has not passed"); - var sent = service.Sent.Should().ContainSingle().Subject; - sent.Message.Should().Be("Is MS-26473 done?"); + component.WaitForAssertion(() => service.Sent.Should().ContainSingle(), Patience); + var sent = service.Sent[0]; + sent.Message.Should().Be("Is it done?"); sent.Sender.Should().BeSameAs(_user); module.Invocations["pause"].Should().ContainSingle().Which.Arguments[0].Should().Be(true); + component.WaitForAssertion(() => component.Find("textarea").GetAttribute("value").Should().BeNullOrEmpty(), Patience); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Thinking); + } + + /// Verifies that while the user is editing the text box, a pause never sends; the user presses Send. + [Fact] + public async Task Dictation_is_never_auto_sent_while_the_text_box_has_focus() + { + var module = SetUpVoiceModule(); + var service = VoiceService(); + var component = await RenderListeningAsync(service); + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + + await SayAsync(component, "Is", "it", "done?"); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + service.Sent.Should().BeEmpty(); + component.Find(".pdchat-voice-status").TextContent.Trim().Should().Be("Listening. You are editing, so press Send when ready."); + + await component.Find(".chat-input-container > button").ClickAsync(new MouseEventArgs()); + + service.Sent.Should().ContainSingle().Which.Message.Should().Be("Is it done?"); + module.Invocations["pause"].Should().ContainSingle().Which.Arguments[0].Should().Be(true); component.Instance.VoiceState.Should().Be(PDChatVoiceState.Thinking); } + /// Verifies that focusing the text box while a send is pending cancels it. + [Fact] + public async Task Focusing_the_text_box_cancels_a_pending_send() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMilliseconds(200)); + var component = await RenderListeningAsync(service); + + await SayAsync(component, "Is", "it", "done?"); + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + await component.Find("textarea").BlurAsync(new FocusEventArgs()); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + service.Sent.Should().BeEmpty(); + } + + /// Verifies that speaking again before the delay has passed cancels the pending send until the next pause. + [Fact] + public async Task Speaking_again_cancels_a_pending_send() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMilliseconds(200)); + var component = await RenderListeningAsync(service); + + await SayAsync(component, "Is", "it"); + await component.InvokeAsync(() => component.Instance.OnVoiceWord("done?")); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + service.Sent.Should().BeEmpty(); + + await component.InvokeAsync(() => component.Instance.OnVoiceTurn("done?")); + + component.WaitForAssertion(() => service.Sent.Should().ContainSingle().Which.Message.Should().Be("Is it done?"), Patience); + } + + /// Verifies that turning Voice Mode off cancels a pending send. + [Fact] + public async Task Turning_Voice_Mode_off_cancels_a_pending_send() + { + SetUpVoiceModule(); + var service = VoiceService(TimeSpan.FromMilliseconds(200)); + var component = await RenderListeningAsync(service); + + await SayAsync(component, "Is", "it", "done?"); + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + await Task.Delay(_settle, Xunit.TestContext.Current.CancellationToken); + + service.Sent.Should().BeEmpty(); + component.Find("textarea").GetAttribute("value").Should().Be("Is it done?", "dictation stays for the user to send or edit"); + } + /// Verifies that only the finished answer is read aloud, as plain text, and listening then resumes. [Fact] public async Task The_finished_answer_is_spoken_and_listening_resumes() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; - var component = RenderChat(service); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); - await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Is it done?")); + var service = VoiceService(); + var component = await RenderListeningAsync(service); + await AskAsync(component, service, "Is it done?"); var typing = Message("Looking it up", MessageType.Typing); await component.InvokeAsync(() => service.Receive(typing)); @@ -111,10 +266,9 @@ public async Task The_finished_answer_is_spoken_and_listening_resumes() public async Task Only_the_first_reply_to_a_spoken_question_is_spoken() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; - var component = RenderChat(service); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); - await component.InvokeAsync(() => component.Instance.OnVoiceTurn("Is it done?")); + var service = VoiceService(); + var component = await RenderListeningAsync(service); + await AskAsync(component, service, "Is it done?"); await component.InvokeAsync(() => service.Receive(Message("Yes."))); await component.InvokeAsync(() => service.Receive(Message("An unrelated notification"))); @@ -127,7 +281,7 @@ public async Task Only_the_first_reply_to_a_spoken_question_is_spoken() public async Task With_Voice_Mode_off_nothing_is_spoken() { var module = SetUpVoiceModule(); - var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; + var service = VoiceService(); var component = RenderChat(service); await component.InvokeAsync(() => service.Receive(Message("An answer"))); @@ -140,8 +294,7 @@ public async Task With_Voice_Mode_off_nothing_is_spoken() public async Task Turning_Voice_Mode_off_stops_it() { var module = SetUpVoiceModule(); - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); - await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + var component = await RenderListeningAsync(VoiceService()); await component.Find(".pdchat-voice-toggle").ClickAsync(new()); @@ -156,7 +309,7 @@ public async Task A_refused_microphone_is_explained() { var module = SetUpVoiceModule(); _ = module.SetupVoid("start", _ => true).SetException(new JSException("NotAllowedError")); - var component = RenderChat(new FakeChatService { VoiceEndpoints = _voiceEndpoints }); + var component = RenderChat(VoiceService()); await component.Find(".pdchat-voice-toggle").ClickAsync(new()); @@ -164,6 +317,36 @@ public async Task A_refused_microphone_is_explained() component.Find(".pdchat-voice-status").TextContent.Should().Contain("microphone could not be opened"); } + private static FakeChatService VoiceService(TimeSpan? autoSendDelay = null) + { + var service = new FakeChatService { VoiceEndpoints = _voiceEndpoints }; + ((IChatService)service).VoiceAutoSendDelay = autoSendDelay ?? TimeSpan.Zero; + return service; + } + + private async Task> RenderListeningAsync(FakeChatService service) + { + var component = RenderChat(service); + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + return component; + } + + private static async Task SayAsync(IRenderedComponent component, params string[] words) + { + foreach (var word in words) + { + await component.InvokeAsync(() => component.Instance.OnVoiceWord(word)); + } + + await component.InvokeAsync(() => component.Instance.OnVoiceTurn(string.Join(' ', words))); + } + + private static async Task AskAsync(IRenderedComponent component, FakeChatService service, string question) + { + await SayAsync(component, question.Split(' ')); + component.WaitForAssertion(() => service.Sent.Should().ContainSingle(), Patience); + } + private BunitJSModuleInterop SetUpVoiceModule() { var module = JSInterop.SetupModule(VoiceModulePath); diff --git a/PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs b/PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs new file mode 100644 index 000000000..98e180a49 --- /dev/null +++ b/PanoramicData.Blazor.Test/Components/PDMessagesTests.ChatInput.cs @@ -0,0 +1,108 @@ +using AwesomeAssertions; +using Bunit; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Web; +using PanoramicData.Blazor.Interfaces; + +namespace PanoramicData.Blazor.Test; + +/// +/// as an : text appended from elsewhere, focus tracking, a virtual +/// Send, and host controls in the input row. +/// +public partial class PDMessagesTests +{ + private const string FocusIdentifier = "Blazor._internal.domWrapper.focus"; + + /// Verifies that appended text joins what was typed with a space and is pushed to the parent. + [Fact] + public async Task Appended_text_follows_what_was_typed() + { + var events = new List(); + var component = RenderLive(events); + await component.InvokeAsync(() => component.Find("textarea").InputAsync(new ChangeEventArgs { Value = "Is" })); + + await component.InvokeAsync(() => component.Instance.AppendAsync(" it done? ")); + await component.InvokeAsync(() => component.Instance.AppendAsync(" ")); + + component.Instance.Text.Should().Be("Is it done?"); + component.Find("textarea").GetAttribute("value").Should().Be("Is it done?"); + events.Should().Equal("input:Is it done?"); + } + + /// Verifies that the text box reports focus and blur, and loses focus when cleared. + [Fact] + public async Task Focus_is_tracked_and_reported() + { + var reported = new List(); + var component = Render(parameters => parameters + .Add(p => p.IsLive, true) + .Add(p => p.InputFocusChanged, isFocused => reported.Add(isFocused))); + + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + component.Instance.IsFocused.Should().BeTrue(); + + await component.Find("textarea").BlurAsync(new FocusEventArgs()); + component.Instance.IsFocused.Should().BeFalse(); + + await component.Find("textarea").FocusAsync(new FocusEventArgs()); + await component.InvokeAsync(component.Instance.ClearInput); + component.Instance.IsFocused.Should().BeFalse("the text box was replaced"); + + reported.Should().Equal(true, false, true); + } + + /// Verifies that a virtual Send behaves as the button does, and sends nothing when empty. + [Fact] + public async Task SendAsync_is_the_same_as_pressing_send() + { + var events = new List(); + var component = RenderLive(events); + + await component.InvokeAsync(component.Instance.SendAsync); + events.Should().BeEmpty(); + + await component.InvokeAsync(() => component.Instance.AppendAsync("spoken")); + await component.InvokeAsync(component.Instance.SendAsync); + + events.Should().Equal("input:spoken", "input:spoken", "send"); + } + + /// Verifies that host controls are shown between the text box and Send, and not at all when absent. + [Fact] + public void Input_accessories_sit_beside_send() + { + RenderLive([]).FindAll(".chat-input-accessories").Should().BeEmpty(); + + var component = Render(parameters => parameters + .Add(p => p.IsLive, true) + .Add(p => p.InputAccessories, builder => builder.AddMarkupContent(0, ""))); + + var row = component.Find(".chat-input-container"); + row.Children.Select(child => child.LocalName).Should().Equal("textarea", "div", "button"); + row.QuerySelector(".chat-input-accessories .extra").Should().NotBeNull(); + } + + /// Verifies that the text box takes focus when shown, unless that is turned off. + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task The_text_box_takes_focus_only_when_auto_focus_is_on(bool isAutoFocused) + { + var attached = SignalOn("attachEnterHandler"); + var component = Render(parameters => parameters + .Add(p => p.IsLive, true) + .Add(p => p.IsInputAutoFocused, isAutoFocused)); + await attached.WaitForCountAsync(1); + + if (isAutoFocused) + { + component.WaitForAssertion(() => JSInterop.Invocations[FocusIdentifier].Should().ContainSingle(), TimeSpan.FromSeconds(10)); + } + else + { + await Task.Delay(100, Xunit.TestContext.Current.CancellationToken); + JSInterop.Invocations[FocusIdentifier].Should().BeEmpty(); + } + } +} diff --git a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs index 6f5d3506c..70da9c6e5 100644 --- a/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs +++ b/PanoramicData.Blazor/Interfaces/ChatServiceDefaultState.cs @@ -46,6 +46,10 @@ internal sealed class ChatServiceDefaultState public PDChatButtonPosition ToastAnchor { get; set; } = PDChatButtonPosition.BottomRight; + public TimeSpan VoiceAutoSendDelay { get; set; } = TimeSpan.FromMilliseconds(1000); + + public string? SelectedAgentId { get; set; } + /// /// Gets or sets the handlers subscribed to the default . /// They are kept so that subscribing and unsubscribing behave as they would on any event, but a service that diff --git a/PanoramicData.Blazor/Interfaces/IChatInput.cs b/PanoramicData.Blazor/Interfaces/IChatInput.cs new file mode 100644 index 000000000..341914aa7 --- /dev/null +++ b/PanoramicData.Blazor/Interfaces/IChatInput.cs @@ -0,0 +1,24 @@ +namespace PanoramicData.Blazor.Interfaces; + +/// +/// The text box a chat message is composed in, whatever wrote the text: the keyboard, dictation, or both. +/// +/// +/// implements this, and 's Voice +/// control writes through it, so dictated words land where typed ones do and can be edited before sending. +/// +public interface IChatInput +{ + /// Gets the text currently in the box. + string Text { get; } + + /// Gets a value indicating whether the box has focus, meaning the user may be editing it. + bool IsFocused { get; } + + /// Appends text to the end of the box, separated from what is already there by a space. + /// The text to add. + Task AppendAsync(string text); + + /// Sends the box's text exactly as pressing Send would; does nothing when there is nothing to send. + Task SendAsync(); +} diff --git a/PanoramicData.Blazor/Interfaces/IChatService.cs b/PanoramicData.Blazor/Interfaces/IChatService.cs index e108f280a..cbaa41c20 100644 --- a/PanoramicData.Blazor/Interfaces/IChatService.cs +++ b/PanoramicData.Blazor/Interfaces/IChatService.cs @@ -94,6 +94,33 @@ bool IsInputPermitted /// PDChatVoiceEndpoints? VoiceEndpoints => null; + /// + /// Gets or sets how long after the speaker pauses Voice Mode sends the dictated text, when the user is not + /// editing it. Defaults to one second. + /// + TimeSpan VoiceAutoSendDelay + { + get => ChatServiceDefaultState.For(this).VoiceAutoSendDelay; + set => ChatServiceDefaultState.For(this).VoiceAutoSendDelay = value; + } + + /// + /// Gets the agents the user may choose to talk to, or null (the default) when the host offers no choice. + /// A picker is shown in the input area only when there are two or more. + /// + IReadOnlyList? Agents => null; + + /// + /// Gets or sets the of the agent the user is talking to, or null (the default) + /// for the host's default. The chat sets this when the user picks an agent; a host that needs to react implements + /// the setter. + /// + string? SelectedAgentId + { + get => ChatServiceDefaultState.For(this).SelectedAgentId; + set => ChatServiceDefaultState.For(this).SelectedAgentId = value; + } + /// /// Gets or sets whether the chat should auto-restore when new messages arrive. /// diff --git a/PanoramicData.Blazor/Models/PDChatAgentOption.cs b/PanoramicData.Blazor/Models/PDChatAgentOption.cs new file mode 100644 index 000000000..be0a2b7d0 --- /dev/null +++ b/PanoramicData.Blazor/Models/PDChatAgentOption.cs @@ -0,0 +1,11 @@ +namespace PanoramicData.Blazor.Models; + +/// +/// One of the agents a host offers in 's input area, for the user to choose +/// who they are talking to. +/// +/// The identifier the host recognises, stored in . +/// The name shown in the picker. +/// An optional description, shown as the option's tooltip. +/// An optional icon, shown beside the picker while this agent is selected. +public sealed record PDChatAgentOption(string Id, string Name, string? Description = null, string? IconUrl = null); diff --git a/PanoramicData.Blazor/PDChat.Agents.cs b/PanoramicData.Blazor/PDChat.Agents.cs new file mode 100644 index 000000000..bb40aeafb --- /dev/null +++ b/PanoramicData.Blazor/PDChat.Agents.cs @@ -0,0 +1,25 @@ +namespace PanoramicData.Blazor; + +/// +/// PDChat: the agent picker in the input area, offered when the chat service lists two or more +/// . +/// +public partial class PDChat +{ + private bool IsAgentPickerOffered => ChatService.Agents is { Count: > 1 }; + + // Null or unknown means the host's default, which the picker shows as its first agent. + private PDChatAgentOption? SelectedAgent => ChatService.Agents?.FirstOrDefault(agent => agent.Id == ChatService.SelectedAgentId) + ?? (ChatService.Agents is { Count: > 0 } agents ? agents[0] : null); + + private RenderFragment? InputAccessories => IsVoiceModeOffered || IsAgentPickerOffered ? InputAccessoriesContent : null; + + private void OnAgentSelected(ChangeEventArgs args) + { + var id = args.Value?.ToString(); + if (ChatService.Agents?.Any(agent => agent.Id == id) == true) + { + ChatService.SelectedAgentId = id; + } + } +} diff --git a/PanoramicData.Blazor/PDChat.Voice.cs b/PanoramicData.Blazor/PDChat.Voice.cs index b87374d49..ab6d6065e 100644 --- a/PanoramicData.Blazor/PDChat.Voice.cs +++ b/PanoramicData.Blazor/PDChat.Voice.cs @@ -22,12 +22,14 @@ public enum PDChatVoiceState } /// -/// Voice Mode for : the user speaks a question, pauses, and hears the answer. Off by default, and +/// Voice Mode for : the user dictates into the text box, and hears the answer. Off by default, and /// offered only when the chat service supplies . /// /// -/// A spoken question goes through the same path as a typed one, so the question and the written answer are both in -/// the transcript. Only the first finished reply to a spoken question is read aloud. +/// Recognised words are appended to the text box, where they can be edited. When the speaker pauses and the box does +/// not have focus, the text is sent after as if Send had been pressed; +/// while the user is editing, they press Send themselves. The first finished reply to a message sent while Voice Mode +/// is listening is read aloud. /// public partial class PDChat { @@ -36,7 +38,8 @@ public partial class PDChat private IJSObjectReference? _voiceModule; private DotNetObjectReference? _voiceReference; private bool _isAwaitingSpokenAnswer; - private string _voiceHeard = string.Empty; + private bool _hasDictatedSinceTurn; + private CancellationTokenSource? _autoSendCancellation; /// Gets where Voice Mode is in a spoken exchange. public PDChatVoiceState VoiceState { get; private set; } = PDChatVoiceState.Off; @@ -44,22 +47,27 @@ public partial class PDChat /// Gets the last problem Voice Mode reported, shown in place of its status. public string? VoiceError { get; private set; } + // Dictation goes through the same text box as typing, via its IChatInput members. + private PDMessages? VoiceInput => MessagesComponent; + private bool IsVoiceModeOffered => ChatService.VoiceEndpoints is not null && ChatService.IsInputPermitted; private bool IsVoiceModeOn => VoiceState != PDChatVoiceState.Off; private string VoiceButtonTitle => IsVoiceModeOn - ? "Turn Voice Mode off" - : "Turn Voice Mode on: ask out loud and hear the answer"; + ? "Voice: stop listening" + : "Voice: speak instead of typing"; private string VoiceStatusCssClass => $"pdchat-voice-status pdchat-voice-{VoiceState.ToString().ToLowerInvariant()}" + (VoiceError is null ? string.Empty : " pdchat-voice-error"); private string VoiceStatusText => VoiceError ?? VoiceState switch { PDChatVoiceState.Starting => "Opening the microphone…", - PDChatVoiceState.Listening => _voiceHeard.Length > 0 ? _voiceHeard : "Listening. Ask your question, then pause.", + PDChatVoiceState.Listening => VoiceInput?.IsFocused == true + ? "Listening. You are editing, so press Send when ready." + : "Listening. Ask your question, then pause.", PDChatVoiceState.Thinking => "Thinking…", - PDChatVoiceState.Speaking => "Speaking. Turn Voice Mode off to stop.", + PDChatVoiceState.Speaking => "Speaking. Turn Voice off to stop.", _ => string.Empty, }; @@ -94,44 +102,61 @@ private async Task ToggleVoiceModeAsync() private async Task StopVoiceModeAsync() { + CancelAutoSend(); VoiceState = PDChatVoiceState.Off; _isAwaitingSpokenAnswer = false; - _voiceHeard = string.Empty; + _hasDictatedSinceTurn = false; if (_voiceModule is not null) { await _voiceModule.InvokeVoidAsync("stop"); } } - /// Called by the voice module as words are recognised. + /// Called by the voice module as words are recognised; appends them to the text box. /// The word just heard. [JSInvokable] - public Task OnVoiceWord(string text) + public Task OnVoiceWord(string text) => InvokeAsync(async () => { - _voiceHeard = $"{_voiceHeard} {text}".Trim(); - return InvokeAsync(StateHasChanged); - } + if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input) + { + return; + } - /// Called by the voice module when the speaker has finished; sends what they said as a question. - /// The whole question. + // The speaker has carried on, so the pause that scheduled a send is over. + CancelAutoSend(); + _hasDictatedSinceTurn = true; + await input.AppendAsync(text); + }); + + /// + /// Called by the voice module when the speaker pauses. Unless the user is editing the text box, its text is sent + /// after . + /// + /// Everything said since the last pause. [JSInvokable] - public async Task OnVoiceTurn(string text) + public Task OnVoiceTurn(string text) => InvokeAsync(async () => { - if (VoiceState != PDChatVoiceState.Listening || string.IsNullOrWhiteSpace(text)) + if (VoiceState != PDChatVoiceState.Listening || VoiceInput is not { } input) { return; } - _voiceHeard = string.Empty; - _currentInput = text; - VoiceState = PDChatVoiceState.Thinking; - _isAwaitingSpokenAnswer = true; + CancelAutoSend(); - // Half duplex: while Merlin thinks and speaks, the microphone sends nothing, so it never hears itself. - await (_voiceModule?.InvokeVoidAsync("pause", true) ?? ValueTask.CompletedTask); - await SendCurrentMessageAsync(); - await InvokeAsync(StateHasChanged); - } + // A host that reports no words still has its pause recorded. + if (!_hasDictatedSinceTurn) + { + await input.AppendAsync(text); + } + + _hasDictatedSinceTurn = false; + if (!input.IsFocused && !string.IsNullOrWhiteSpace(input.Text)) + { + ScheduleAutoSend(); + } + + StateHasChanged(); + }); /// Called by the voice module when speech recognition reports a problem. /// What went wrong, for the user. @@ -146,12 +171,85 @@ public Task OnVoiceError(string text) [JSInvokable] public Task OnVoiceClosed() { + CancelAutoSend(); VoiceState = PDChatVoiceState.Off; _isAwaitingSpokenAnswer = false; VoiceError ??= "Voice Mode stopped: the connection closed."; return InvokeAsync(StateHasChanged); } + private Task OnInputFocusChangedAsync(bool isFocused) + { + // The user is editing, so whatever was dictated waits for them to press Send. + if (isFocused) + { + CancelAutoSend(); + } + + return Task.CompletedTask; + } + + private void ScheduleAutoSend() + { + var cancellation = new CancellationTokenSource(); + _autoSendCancellation = cancellation; + _ = AutoSendAfterDelayAsync(cancellation.Token); + } + + private void CancelAutoSend() + { + if (_autoSendCancellation is null) + { + return; + } + + _autoSendCancellation.Cancel(); + _autoSendCancellation.Dispose(); + _autoSendCancellation = null; + } + + private async Task AutoSendAfterDelayAsync(CancellationToken cancellationToken) + { + try + { + var delay = ChatService.VoiceAutoSendDelay; + await Task.Delay(delay < TimeSpan.Zero ? TimeSpan.Zero : delay, cancellationToken); + await InvokeAsync(async () => + { + if (cancellationToken.IsCancellationRequested || VoiceState != PDChatVoiceState.Listening + || VoiceInput is not { IsFocused: false } input) + { + return; + } + + await input.SendAsync(); + }); + } + catch (Exception ex) when (ex is JSDisconnectedException or OperationCanceledException or ObjectDisposedException) + { + // Cancelled by the speaker or the user, or the circuit is going away. + } + } + + /// + /// Marks a message about to be sent while listening as a spoken exchange: the microphone pauses and the reply is read aloud. + /// + private async Task BeginSpokenExchangeAsync() + { + if (VoiceState != PDChatVoiceState.Listening) + { + return; + } + + CancelAutoSend(); + _hasDictatedSinceTurn = false; + VoiceState = PDChatVoiceState.Thinking; + _isAwaitingSpokenAnswer = true; + + // Half duplex: while Merlin thinks and speaks, the microphone sends nothing, so it never hears itself. + await (_voiceModule?.InvokeVoidAsync("pause", true) ?? ValueTask.CompletedTask); + } + /// Reads the first finished reply to a spoken question aloud, then listens again. private async Task SpeakAnswerIfAwaitedAsync(ChatMessage message) { @@ -192,6 +290,7 @@ private static string ToSpeakableText(ChatMessage message) private async ValueTask DisposeVoiceAsync() { + CancelAutoSend(); if (_voiceModule is not null) { try diff --git a/PanoramicData.Blazor/PDChat.razor b/PanoramicData.Blazor/PDChat.razor index 0ab5fbde1..55244cee2 100644 --- a/PanoramicData.Blazor/PDChat.razor +++ b/PanoramicData.Blazor/PDChat.razor @@ -84,15 +84,6 @@ 🗑️ } - @if (IsVoiceModeOffered) - { - - } @@ -208,6 +199,9 @@ IsInputPermitted="@ChatService.IsInputPermitted" InputDisabledMessage="@ChatService.InputDisabledMessage" OnSendClicked="SendCurrentMessageAsync" + InputAccessories="InputAccessories" + InputFocusChanged="OnInputFocusChangedAsync" + IsInputAutoFocused="@(!IsVoiceModeOn)" UserIconSelector="UserIconSelector" UseFullWidthMessages="@ChatService.UseFullWidthMessages" MessageMetadataDisplayMode="@ChatService.MessageMetadataDisplayMode" @@ -255,6 +249,9 @@ IsInputPermitted="@ChatService.IsInputPermitted" InputDisabledMessage="@ChatService.InputDisabledMessage" OnSendClicked="SendCurrentMessageAsync" + InputAccessories="InputAccessories" + InputFocusChanged="OnInputFocusChangedAsync" + IsInputAutoFocused="@(!IsVoiceModeOn)" UserIconSelector="UserIconSelector" UseFullWidthMessages="@ChatService.UseFullWidthMessages" MessageMetadataDisplayMode="@ChatService.MessageMetadataDisplayMode" @@ -276,6 +273,9 @@ IsInputPermitted="@ChatService.IsInputPermitted" InputDisabledMessage="@ChatService.InputDisabledMessage" OnSendClicked="SendCurrentMessageAsync" + InputAccessories="InputAccessories" + InputFocusChanged="OnInputFocusChangedAsync" + IsInputAutoFocused="@(!IsVoiceModeOn)" UserIconSelector="UserIconSelector" UseFullWidthMessages="@ChatService.UseFullWidthMessages" MessageMetadataDisplayMode="@ChatService.MessageMetadataDisplayMode" @@ -289,3 +289,36 @@ } + +@code { + private RenderFragment InputAccessoriesContent => @ + @if (IsAgentPickerOffered) + { +
+ @if (SelectedAgent?.IconUrl is { } iconUrl) + { + + } + +
+ } + @if (IsVoiceModeOffered) + { + + } + ; +} \ No newline at end of file diff --git a/PanoramicData.Blazor/PDChat.razor.cs b/PanoramicData.Blazor/PDChat.razor.cs index 9af92c3d9..d4f623735 100644 --- a/PanoramicData.Blazor/PDChat.razor.cs +++ b/PanoramicData.Blazor/PDChat.razor.cs @@ -537,6 +537,9 @@ private async Task SendCurrentMessageAsync() Timestamp = DateTime.UtcNow }; + // Before sending, so a reply that arrives at once is still the one spoken. + await BeginSpokenExchangeAsync(); + // Fire it off ChatService.SendMessage(message); diff --git a/PanoramicData.Blazor/PDChat.razor.css b/PanoramicData.Blazor/PDChat.razor.css index 71610bde4..79c330574 100644 --- a/PanoramicData.Blazor/PDChat.razor.css +++ b/PanoramicData.Blazor/PDChat.razor.css @@ -543,18 +543,34 @@ body:has(.pdchat-container.dock-fullscreen) { outline-offset: 1px; } -/* Voice Mode: the switch carries a word as well as an icon, so it says what it does. */ +/* Input-area controls: the Voice switch carries a word as well as an icon, so it says what it does. */ .pdchat-voice-toggle { - font-size: 14px; - min-width: auto; white-space: nowrap; } .pdchat-voice-toggle.active { - background: rgba(255,255,255,0.25); - box-shadow: inset 0 0 0 1px rgba(255,255,255,0.6); + color: var(--pd-chat-bg, #fff); + background: #dc3545; + border-color: #dc3545; } +.pdchat-agent-picker { + display: flex; + align-items: center; + gap: 4px; + min-width: 0; +} + +.pdchat-agent-picker select { + min-width: 0; +} + +.pdchat-agent-icon { + width: 20px; + height: 20px; + flex-shrink: 0; + border-radius: 50%; +} .pdchat-voice-status { padding: 6px 12px; font-size: 13px; diff --git a/PanoramicData.Blazor/PDMessages.Interop.cs b/PanoramicData.Blazor/PDMessages.Interop.cs index ba61599da..21cf841c7 100644 --- a/PanoramicData.Blazor/PDMessages.Interop.cs +++ b/PanoramicData.Blazor/PDMessages.Interop.cs @@ -38,7 +38,10 @@ await JSRuntime.InvokeAsync( _dotNetRef ??= DotNetObjectReference.Create(this); await _module.InvokeVoidAsync("attachEnterHandler", InputRef, _dotNetRef); _enterHandlerAttached = true; - await InputRef.FocusAsync(); + if (IsInputAutoFocused) + { + await InputRef.FocusAsync(); + } } catch (Exception) { diff --git a/PanoramicData.Blazor/PDMessages.razor b/PanoramicData.Blazor/PDMessages.razor index ff112e3fa..030ca2291 100644 --- a/PanoramicData.Blazor/PDMessages.razor +++ b/PanoramicData.Blazor/PDMessages.razor @@ -25,9 +25,15 @@ @ref="InputRef" value="@_localInput" @oninput="OnInputChanged" + @onfocus="() => SetInputFocusedAsync(true)" + @onblur="() => SetInputFocusedAsync(false)" placeholder="Type a message..." rows="3" disabled="@(!IsLive)"> + @if (InputAccessories is not null) + { +
@InputAccessories
+ } } diff --git a/PanoramicData.Blazor/PDMessages.razor.cs b/PanoramicData.Blazor/PDMessages.razor.cs index 9dc6366f5..3450cb2bf 100644 --- a/PanoramicData.Blazor/PDMessages.razor.cs +++ b/PanoramicData.Blazor/PDMessages.razor.cs @@ -3,7 +3,7 @@ namespace PanoramicData.Blazor; /// /// A Blazor component that displays a list of chat messages with a send input area. /// -public partial class PDMessages +public partial class PDMessages : IChatInput { /// /// Gets or sets the injected JavaScript runtime. @@ -99,11 +99,51 @@ public partial class PDMessages /// internal ElementReference InputRef { get; set; } + /// + /// Gets or sets optional controls shown in the input row beside the Send button, such as a voice or agent control. + /// + [Parameter] public RenderFragment? InputAccessories { get; set; } + + /// + /// An event callback that is invoked when the text box gains (true) or loses (false) focus. + /// + [Parameter] public EventCallback InputFocusChanged { get; set; } + + /// + /// Gets or sets whether the text box takes focus when first shown and after each send. Defaults to true. + /// + [Parameter] public bool IsInputAutoFocused { get; set; } = true; + private string _localInput = string.Empty; private string _inputKey = Guid.NewGuid().ToString(); private bool CanSendLocal => IsInputPermitted && IsLive && !string.IsNullOrWhiteSpace(_localInput); + /// + public string Text => _localInput; + + /// + public bool IsFocused { get; private set; } + + /// + public async Task AppendAsync(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return; + } + + var addition = text.Trim(); + _localInput = _localInput.Length == 0 || char.IsWhiteSpace(_localInput[^1]) + ? _localInput + addition + : $"{_localInput} {addition}"; + await CurrentInputChanged.InvokeAsync(_localInput); + StateHasChanged(); + } + + /// + public Task SendAsync() => OnSendClickedInternal(); + /// /// Clears the textarea. Called by the parent after a message is sent. /// @@ -111,10 +151,19 @@ public void ClearInput() { _localInput = string.Empty; _inputKey = Guid.NewGuid().ToString(); + + // The text box is replaced, and a removed element does not reliably report losing focus. + IsFocused = false; ResetEnterHandler(); StateHasChanged(); } + private async Task SetInputFocusedAsync(bool isFocused) + { + IsFocused = isFocused; + await InputFocusChanged.InvokeAsync(isFocused); + } + /// /// Called from JavaScript when Enter is pressed in the textarea. /// diff --git a/PanoramicData.Blazor/PDMessages.razor.css b/PanoramicData.Blazor/PDMessages.razor.css index 103468262..eb5d65e56 100644 --- a/PanoramicData.Blazor/PDMessages.razor.css +++ b/PanoramicData.Blazor/PDMessages.razor.css @@ -113,6 +113,17 @@ flex-shrink: 0; } +/* Host-supplied controls (voice, agent) stacked beside Send. */ +.chat-input-accessories { + display: flex; + flex-direction: column; + align-items: stretch; + gap: 4px; + flex-shrink: 0; + min-width: 0; + max-width: 40%; +} + /* Message shown in place of the input when input is not permitted (IsInputPermitted = false). Theme-aware via the --pd-chat-* variables. */ .chat-input-disabled { From 2c58d34e6667a2924193ed9e5621cbca926a1150 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 5 Oct 2026 23:21:42 +0100 Subject: [PATCH 2/2] MS-26473 PDChat: simulated Voice Mode and agents in the demo PDChatVoiceEndpoints takes an optional ModulePath replacing the browser voice module, and PDChatVoiceEndpoints.Simulated uses one that hears that the user is talking, makes the words up and answers with the browser's own speech. DumbChatService offers it by default, plus two agents, so the demo shows Voice Mode and the agent picker with no speech service. Co-Authored-By: Claude Opus 5.5 --- .../Components/PDChatTests.Voice.cs | 29 ++++ .../Models/PDChatVoiceEndpoints.cs | 13 +- PanoramicData.Blazor/PDChat.Voice.cs | 2 +- .../Services/DumbChatService.Voice.cs | 17 +++ .../wwwroot/js/pdchat-voice-simulated.js | 129 ++++++++++++++++++ 5 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 PanoramicData.Blazor/Services/DumbChatService.Voice.cs create mode 100644 PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js diff --git a/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs b/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs index dd6794004..220e58702 100644 --- a/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs +++ b/PanoramicData.Blazor.Test/Components/PDChatTests.Voice.cs @@ -347,6 +347,35 @@ private static async Task AskAsync(IRenderedComponent component, FakeCha component.WaitForAssertion(() => service.Sent.Should().ContainSingle(), Patience); } + /// Verifies that endpoints naming their own module have Voice Mode driven by it, not the standard one. + [Fact] + public async Task Simulated_Voice_Mode_uses_its_own_module() + { + var simulated = PDChatVoiceEndpoints.Simulated; + var module = JSInterop.SetupModule(simulated.ModulePath!); + module.Mode = JSRuntimeMode.Loose; + var service = VoiceService(); + service.VoiceEndpoints = simulated; + var component = RenderChat(service); + + await component.Find(".pdchat-voice-toggle").ClickAsync(new()); + + VoiceModuleImports().Should().BeEmpty(); + module.Invocations["start"].Should().ContainSingle(); + component.Instance.VoiceState.Should().Be(PDChatVoiceState.Listening); + } + + /// Verifies that the demo service shows Voice Mode and the agent picker with no speech service behind it. + [Fact] + public void The_demo_service_offers_simulated_Voice_Mode_and_agents() + { + using var service = new PanoramicData.Blazor.Services.DumbChatService(); + IChatService chat = service; + + chat.VoiceEndpoints.Should().Be(PDChatVoiceEndpoints.Simulated); + chat.Agents.Should().HaveCountGreaterThan(1); + } + private BunitJSModuleInterop SetUpVoiceModule() { var module = JSInterop.SetupModule(VoiceModulePath); diff --git a/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs b/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs index 3b1e36fcd..4e5bf24be 100644 --- a/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs +++ b/PanoramicData.Blazor/Models/PDChatVoiceEndpoints.cs @@ -12,5 +12,16 @@ namespace PanoramicData.Blazor.Models; /// Receives one JSON message, {"text":...}, and answers with 24 kHz mono float32 audio as binary messages, /// then {"type":"done"}. /// +/// +/// An optional replacement for the browser module that drives the microphone and plays answers, exporting the same +/// start, pause, speak and stop functions; null uses the standard one. +/// /// Relative URLs are resolved against the page. Speech is processed wherever the host's endpoints send it. -public sealed record PDChatVoiceEndpoints(string ListenUrl, string SpeakUrl); +public sealed record PDChatVoiceEndpoints(string ListenUrl, string SpeakUrl, string? ModulePath = null) +{ + /// + /// Voice Mode with no speech service, for demos: it hears that the user is talking but makes the words up, and + /// answers in the browser's own voice. No audio leaves the browser. + /// + public static PDChatVoiceEndpoints Simulated { get; } = new("simulated", "simulated", "./_content/PanoramicData.Blazor/js/pdchat-voice-simulated.js"); +} diff --git a/PanoramicData.Blazor/PDChat.Voice.cs b/PanoramicData.Blazor/PDChat.Voice.cs index ab6d6065e..106411db7 100644 --- a/PanoramicData.Blazor/PDChat.Voice.cs +++ b/PanoramicData.Blazor/PDChat.Voice.cs @@ -88,7 +88,7 @@ private async Task ToggleVoiceModeAsync() VoiceState = PDChatVoiceState.Starting; try { - _voiceModule ??= await JSRuntime.InvokeAsync("import", _voiceModulePath); + _voiceModule ??= await JSRuntime.InvokeAsync("import", endpoints.ModulePath ?? _voiceModulePath); _voiceReference ??= DotNetObjectReference.Create(this); await _voiceModule.InvokeVoidAsync("start", endpoints.ListenUrl, _voiceReference); VoiceState = PDChatVoiceState.Listening; diff --git a/PanoramicData.Blazor/Services/DumbChatService.Voice.cs b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs new file mode 100644 index 000000000..50116c496 --- /dev/null +++ b/PanoramicData.Blazor/Services/DumbChatService.Voice.cs @@ -0,0 +1,17 @@ +namespace PanoramicData.Blazor.Services; + +/// +/// The Voice Mode and agent picker of , so a demo shows both without a speech service. +/// +public partial class DumbChatService +{ + /// Gets or sets Voice Mode's endpoints; simulated by default, and null hides Voice Mode. + public PDChatVoiceEndpoints? VoiceEndpoints { get; set; } = PDChatVoiceEndpoints.Simulated; + + /// Gets the agents offered in the input area. + public IReadOnlyList? Agents { get; } = + [ + new("dumbbot", "DumbBot", "Answers by keyword, and not very well."), + new("parrot", "Parrot", "Exactly as helpful as DumbBot, with more feathers."), + ]; +} diff --git a/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js new file mode 100644 index 000000000..5557b86df --- /dev/null +++ b/PanoramicData.Blazor/wwwroot/js/pdchat-voice-simulated.js @@ -0,0 +1,129 @@ +// PDChat Voice Mode, simulated: hears THAT you are talking, never WHAT, and makes the words up. +// Same exports as pdchat-voice.js. Audio never leaves the browser. + +const PHRASES = [ + "I know this is not what you are actually saying, but to use real voice recognition you need to hook in a streaming speech service.", + "blah blah blah blah blah blah", + "This is a simulated transcript, and none of your words were sent anywhere.", + "Pretend I just asked something really clever.", + "Supply VoiceEndpoints from your chat service to hear the real thing.", + "Testing, testing, one two three. Is this thing on?", +]; + +const LOUD = 0.02; +const WORD_MS = 220; +const PAUSE_MS = 700; + +let listening = null; +let phraseIndex = 0; +let wordIndex = 0; + +function nextWord() { + const words = PHRASES[phraseIndex].split(" "); + const word = words[wordIndex]; + wordIndex++; + if (wordIndex >= words.length) { + wordIndex = 0; + phraseIndex = (phraseIndex + 1) % PHRASES.length; + } + return word; +} + +function level(analyser, samples) { + analyser.getFloatTimeDomainData(samples); + let sum = 0; + for (let i = 0; i < samples.length; i++) { + sum += samples[i] * samples[i]; + } + return Math.sqrt(sum / samples.length); +} + +function tick(state, dotNet) { + if (state.paused) { + return; + } + const now = Date.now(); + if (level(state.analyser, state.samples) > LOUD) { + state.lastLoud = now; + if (now - state.lastWord >= WORD_MS) { + state.lastWord = now; + const word = nextWord(); + state.heard.push(word); + dotNet.invokeMethodAsync("OnVoiceWord", word); + } + } else if (state.heard.length > 0 && now - state.lastLoud >= PAUSE_MS) { + const text = state.heard.join(" "); + state.heard = []; + dotNet.invokeMethodAsync("OnVoiceTurn", text); + } +} + +/** Starts listening. Rejects when the microphone is refused or unavailable. */ +export async function start(listenUrl, dotNet) { + stop(); + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const context = new AudioContext(); + const analyser = context.createAnalyser(); + analyser.fftSize = 1024; + context.createMediaStreamSource(stream).connect(analyser); + const state = { + stream, + context, + analyser, + samples: new Float32Array(analyser.fftSize), + paused: false, + heard: [], + lastLoud: 0, + lastWord: 0, + }; + state.timer = setInterval(() => tick(state, dotNet), 50); + listening = state; +} + +/** While paused nothing is "heard", so the assistant does not hear itself. */ +export function pause(paused) { + if (listening) { + listening.paused = paused; + listening.heard = []; + } +} + +/** Stops listening and speaking, and releases the microphone. */ +export function stop() { + stopSpeaking(); + if (!listening) { + return; + } + const state = listening; + listening = null; + clearInterval(state.timer); + state.stream.getTracks().forEach((track) => track.stop()); + state.context.close(); +} + +/** Stops an answer part-way through. */ +export function stopSpeaking() { + if (globalThis.speechSynthesis) { + globalThis.speechSynthesis.cancel(); + } +} + +/** Speaks an answer with the browser's own voice, and calls OnVoiceSpoken when it ends. */ +export function speak(speakUrl, text, dotNet) { + stopSpeaking(); + let finished = false; + const finish = () => { + if (!finished) { + finished = true; + dotNet.invokeMethodAsync("OnVoiceSpoken"); + } + }; + if (!globalThis.speechSynthesis) { + setTimeout(finish, 60 * text.length); + return; + } + const utterance = new SpeechSynthesisUtterance(text); + utterance.onend = finish; + utterance.onerror = finish; + globalThis.speechSynthesis.speak(utterance); +}