diff --git a/src/Schuly.API/Helply/HelplyPrompt.cs b/src/Schuly.API/Helply/HelplyPrompt.cs index cebd0c15..2aa02dd8 100644 --- a/src/Schuly.API/Helply/HelplyPrompt.cs +++ b/src/Schuly.API/Helply/HelplyPrompt.cs @@ -37,6 +37,7 @@ How you work - Presentations, videos, projects, hand-ins and oral exams are assessments too. When the user asks about "tests", include them or ask. - Then check each source for that subject and teacher, cheapest first: OneNote (search_pages for the objectives term; if that finds nothing, walk list_notebooks, list_sections and list_pages for the subject and read_page, where teachers often keep a page per test or a page with assessment dates); search_files and search with the subject and the term; search with the class team's teamId (from list_teams) together with search_channel_messages, because an unscoped search can miss team files; list_team_files for the subject channel and its subfolders (often sorted by year, then topic; open the newest folders first); list_mail from the teacher's surname, then read_mail. - Search in the school's language and with the words teachers actually use: Lernziele, Prüfung, Test, Lernkontrolle, Stoff, Prüfungsstoff, Auftrag; objectifs, examen, épreuve, contrôle; obiettivi, verifica, esame. Also search for the topic itself and the test number ("Test 3", "3. Prüfung"). + - Prefer one search call over many: pass the subject and all the word variants above as terms (for example Prüfung, Test, Lernkontrolle, LK, Masterplan) together with a from/to date range. It searches files, OneNote page titles and, if allowed, channel posts, mail and chats at once and says which sources it skipped; use its readWith and ids to open a hit. If it fails or is partial, fall back to search_files, search_pages, search_channel_messages and list_mail one by one. - Read long documents in parts. Check the date: learning objectives are often reused from last year. Check the topic matches what is being taught now (the latest notebook section, the latest files). - If you find nothing, say so plainly, give a clearly labelled guess (the current chapter, the order of topics from past grades) and name where it probably is instead (Moodle or another platform Schuly can't reach, a group chat, the lesson itself). Offer to work from a file the user attaches. - Always say where you found something: the file or page name, author, date, where it lives (team > channel > folder, or notebook > section > page) and the link if there is one. diff --git a/src/Schuly.API/Helply/HelplyToolLabels.cs b/src/Schuly.API/Helply/HelplyToolLabels.cs index fa6d6ce1..e9ccf22d 100644 --- a/src/Schuly.API/Helply/HelplyToolLabels.cs +++ b/src/Schuly.API/Helply/HelplyToolLabels.cs @@ -48,7 +48,7 @@ public static class HelplyToolLabels ["list_shared_with_me"] = ("Checking files shared with you", "Checked files shared with you"), ["list_recent_files"] = ("Checking your recent files", "Checked your recent files"), ["search_files"] = ("Searching your files", "Searched your files"), - ["search"] = ("Searching your files, mail and chats", "Searched your files, mail and chats"), + ["search"] = ("Searching your files, notes, mail and chats", "Searched your files, notes, mail and chats"), ["read_file"] = ("Reading a file", "Read a file"), ["list_channel_messages"] = ("Reading a Teams channel", "Read a Teams channel"), ["search_channel_messages"] = ("Searching Teams channels", "Searched Teams channels"), diff --git a/src/Schuly.API/Mcp/Tools/MicrosoftFileTools.cs b/src/Schuly.API/Mcp/Tools/MicrosoftFileTools.cs index c06d5978..c9ed0050 100644 --- a/src/Schuly.API/Mcp/Tools/MicrosoftFileTools.cs +++ b/src/Schuly.API/Mcp/Tools/MicrosoftFileTools.cs @@ -102,23 +102,6 @@ public async Task ListRecentFiles(Guid? accountId = null, [Description(" return Serialize(SearchOutput(account, query, data)); } - [McpServerTool(Name = "search", Title = "Search everything", ReadOnly = true, Destructive = false, Idempotent = true, OpenWorld = true)] - [Description("Searches across files (OneDrive, Teams, SharePoint), and, when this connection also has the permission, mail (needs Mail read) and Teams chats and channel posts (needs Teams read). The result lists which sources were searched. Use it for 'find the Masterplan of class X'; then read_file for files or read_mail / list_channel_messages for the rest.")] - [McpAccess(McpArea.Drive, McpAccessLevel.Read)] - public async Task Search([Description("Search text, 1-200 characters.")] string query, [Description("Limit files and channel posts to this team (teamId from list_teams).")] string? teamId = null, Guid? accountId = null, [Description("Hits to skip, for paging.")] int offset = 0, [Description("Hits per call, 1-50.")] int limit = 25, CancellationToken ct = default) - { - var account = await AccountAsync(accountId, ct); - var permissions = Caller(accessor).Permissions; - var types = new List { "files" }; - if (permissions.Allows(McpArea.Mail, McpAccessLevel.Read)) - types.Add("mail"); - if (permissions.Allows(McpArea.Teams, McpAccessLevel.Read)) - types.AddRange(["chats", "channels"]); - var (from, size) = (Math.Max(0, offset), Math.Clamp(limit, 1, 50)); - var data = await WriteAsync(HttpMethods.Post, $"{Base}/accounts/{account}/search", new { q = query.Trim(), types, from, size, teamId = string.IsNullOrWhiteSpace(teamId) ? null : teamId }, ct) ?? default; - return Serialize(SearchOutput(account, query, data, types)); - } - [McpServerTool(Name = "read_file", Title = "Read file contents", ReadOnly = true, Destructive = false, Idempotent = true, OpenWorld = true)] [Description("Reads a file as text: Word, Excel (one sheet at a time, paged by rows), PowerPoint, PDF and plain text files. Give itemId and, for Teams / SharePoint / shared files, the driveId from the listing or search; without driveId the file is in the user's own OneDrive. The answer says how to get the next page or other sheets.")] [McpAccess(McpArea.Drive, McpAccessLevel.Read)] diff --git a/src/Schuly.API/Mcp/Tools/MicrosoftSearchTools.cs b/src/Schuly.API/Mcp/Tools/MicrosoftSearchTools.cs new file mode 100644 index 00000000..675163e4 --- /dev/null +++ b/src/Schuly.API/Mcp/Tools/MicrosoftSearchTools.cs @@ -0,0 +1,328 @@ +using System.Collections.Concurrent; +using System.ComponentModel; +using System.Globalization; +using System.Text; +using System.Text.Json; +using ModelContextProtocol; +using ModelContextProtocol.Server; +using Schuly.Domain.Mcp; +using static Schuly.API.Mcp.Tools.McpToolSupport; + +namespace Schuly.API.Mcp.Tools +{ + /// + /// The cross-source search. One call fans out over several terms and several sources, merges what comes back into one deduplicated, ranked list and says which sources it searched or skipped. + /// Files, mail, chats and channel posts come from the plugin's search endpoint (one request per term and page), OneNote page titles from the plugin's OneNote search. + /// + public sealed partial class MicrosoftTools + { + public const int MaxSearchTerms = 8; + public const int MaxMergedHits = 200; + private const int SearchConcurrency = 4; + private const int SearchPageSize = 50; + private const int MaxPagesPerTerm = 4; + private static readonly TimeSpan SearchTimeout = TimeSpan.FromSeconds(25); + + private static readonly string[] SearchSourceNames = ["files", "channels", "mail", "chats", "pages"]; + + [McpServerTool(Name = "search", Title = "Search everything", ReadOnly = true, Destructive = false, Idempotent = true, OpenWorld = true)] + [Description("Searches across files (OneDrive, Teams, SharePoint), OneNote page titles and, when this connection also has the permission, mail (needs Mail read) and Teams chats and channel posts (needs Teams read), and returns ONE merged, ranked list. Put every synonym in a single call: query is the main text and terms are alternatives that are OR'd (for a test: terms=['Prüfung','Test','Lernkontrolle','Masterplan','LK']), and use from/to (yyyy-MM-dd, modified or received date) to cut old results, instead of calling search several times. Each hit says which source it came from, where it lives (location.label), who wrote it, when it changed, its link and which terms matched, plus the ids for the read tool named in readWith (file: driveId + itemId for read_file; page: pageId for read_page; mail: messageId for read_mail; channel: teamId, channelId and messageId from location for list_channel_messages). The answer lists the sources searched and skipped and sets partial when something could not be searched completely. OneNote matches page titles only.")] + [McpAccess(McpArea.Drive, McpAccessLevel.Read)] + public async Task Search([Description("Main search text, 1-200 characters. Optional when terms is given.")] string? query = null, [Description("Alternative search terms (synonyms, other languages), at most 8, each 1-200 characters. A hit matches if any term matches; the hit lists matchedTerms.")] string[]? terms = null, [Description("Only results modified or received on or after this date, yyyy-MM-dd.")] DateOnly? from = null, [Description("Only results modified or received on or before this date, yyyy-MM-dd.")] DateOnly? to = null, [Description("Limit the sources: any of files, channels (channel posts), mail, chats, pages (OneNote page titles). Default: every source this connection may read.")] string[]? sources = null, [Description("Limit files and channel posts to this team (teamId from list_teams). OneNote is not searched then.")] string? teamId = null, Guid? accountId = null, [Description("Merged hits to skip, for paging.")] int offset = 0, [Description("Merged hits per call, 1-50.")] int limit = 25, CancellationToken ct = default) + { + var searchTerms = SearchTerms(query, terms); + if (from is { } f && to is { } t && f > t) + throw new McpException("from must not be after to."); + var wanted = ParseSearchSources(sources); + var account = await AccountAsync(accountId, ct); + var permissions = Caller(accessor).Permissions; + offset = Math.Max(0, offset); + limit = Math.Clamp(limit, 1, 50); + + var skipped = new List(); + var types = new List(); + var searchPages = false; + foreach (var source in SearchSourceNames) + { + var reason = !wanted.Contains(source) ? "not requested" + : source == "mail" && !permissions.Allows(McpArea.Mail, McpAccessLevel.Read) ? "needs Mail read" + : source is "chats" or "channels" && !permissions.Allows(McpArea.Teams, McpAccessLevel.Read) ? "needs Teams read" + : source == "pages" && !permissions.Allows(McpArea.OneNote, McpAccessLevel.Read) ? "needs OneNote read" + : source == "pages" && !string.IsNullOrWhiteSpace(teamId) ? "teamId limits the search to Teams files and channel posts" + : null; + if (reason is not null) + skipped.Add(new { source, reason }); + else if (source == "pages") + searchPages = true; + else + types.Add(source); + } + var searched = searchPages ? [.. types, "pages"] : types.ToList(); + if (searched.Count == 0) + throw new McpException("None of the requested sources can be searched with this connection's permissions: " + string.Join("; ", skipped.Select(s => JsonSerializer.Serialize(s, Json)))); + + var pagesPerTerm = Math.Clamp((int)Math.Ceiling((offset + limit) / (double)SearchPageSize) + (from is not null || to is not null ? 1 : 0), 1, MaxPagesPerTerm); + var run = new SearchRun(); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(SearchTimeout); + using var gate = new SemaphoreSlim(SearchConcurrency); + + async Task GuardedAsync(string label, Func work) + { + await gate.WaitAsync(timeout.Token); + try + { + await work(timeout.Token); + Interlocked.Increment(ref run.Succeeded); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + run.Errors.Add($"{label}: timed out"); + } + catch (McpException ex) + { + run.Errors.Add($"{label}: {ex.Message}"); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + run.Errors.Add($"{label}: failed"); + } + finally + { + gate.Release(); + } + } + + var tasks = new List(); + foreach (var term in searchTerms) + { + if (types.Count > 0) + tasks.Add(GuardedAsync(string.Join("/", types), c => SearchPluginAsync(account, term, types, teamId, pagesPerTerm, run, c))); + if (searchPages) + tasks.Add(GuardedAsync("pages", c => SearchPageTitlesAsync(account, term, run, c))); + } + try + { + await Task.WhenAll(tasks); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + run.Errors.Add("search: timed out"); + } + ct.ThrowIfCancellationRequested(); + if (run.Succeeded == 0 && run.Errors.Count > 0) + throw new McpException(run.Errors.First()); + + var merged = MergeSearchHits(run.Hits.ToList(), searchTerms, from, to); + var truncated = merged.Count > MaxMergedHits; + if (truncated) + merged = merged.Take(MaxMergedHits).ToList(); + var page = merged.Skip(offset).Take(limit).ToList(); + var next = offset + page.Count < merged.Count ? offset + page.Count : (int?)null; + var notes = new List(); + if (run.MoreAtBackend) + notes.Add($"Some sources have more results than were fetched ({pagesPerTerm * SearchPageSize} per term and source). Narrow the search with from/to, sources or more specific terms."); + if (truncated) + notes.Add($"Only the best {MaxMergedHits} merged hits are kept."); + + return Serialize(new + { + accountId = account, + terms = searchTerms, + from, + to, + searched, + skipped, + partial = run.Errors.Count > 0 || run.MoreAtBackend || truncated, + total = merged.Count, + offset, + nextOffset = next, + hits = page.Select(SearchHitOutput).ToList(), + notes = notes.Count == 0 ? null : notes, + errors = run.Errors.Count == 0 ? null : run.Errors.Take(10).ToList(), + }); + } + + private async Task SearchPluginAsync(Guid account, string term, List types, string? teamId, int pages, SearchRun run, CancellationToken ct) + { + for (var p = 0; p < pages; p++) + { + var data = await WriteAsync(HttpMethods.Post, $"{Base}/accounts/{account}/search", new { q = term, types, from = p * SearchPageSize, size = SearchPageSize, teamId = string.IsNullOrWhiteSpace(teamId) ? null : teamId }, ct) ?? default; + var hits = Items(Prop(data, "hits")).ToList(); + var rank = p * SearchPageSize; + foreach (var h in hits) + if (PluginHit(h, term, rank++) is { } hit) + run.Hits.Add(hit); + if (Prop(data, "errors") is { ValueKind: JsonValueKind.Array } errors) + foreach (var e in errors.EnumerateArray().Select(x => x.GetString()).OfType().Take(5)) + run.Errors.Add(e); + var more = Bool(data, "moreAvailable") == true; + if (!more || hits.Count == 0) + return; + if (p == pages - 1) + run.MoreAtBackend = true; + } + } + + private async Task SearchPageTitlesAsync(Guid account, string term, SearchRun run, CancellationToken ct) + { + var data = await SearchPagesRawAsync(account, term.Length > 100 ? term[..100] : term, null, SearchPageSize, ct) ?? default; + var rank = 0; + foreach (var h in Items(Prop(data, "items"))) + { + var id = Str(h, "id"); + if (id is null) + continue; + var notebook = Prop(h, "notebook") is { } n ? Str(n, "displayName") : null; + var section = Prop(h, "section") is { } s ? Str(s, "displayName") : null; + run.Hits.Add(new RawSearchHit("pages", id, null, Str(h, "title"), null, null, Str(h, "lastModifiedDateTime"), string.Join(" > ", new[] { notebook, section }.Where(x => !string.IsNullOrEmpty(x))), new SearchLocation("onenote", null, null, null, null), Str(h, "webUrl"), null, null, null, term, rank++)); + } + if (Bool(data, "partial") == true) + run.MoreAtBackend = true; + } + + private static RawSearchHit? PluginHit(JsonElement h, string term, int rank) + { + var source = Str(h, "type") switch { "file" => "files", "mail" => "mail", "chat" => "chats", "channel" => "channels", _ => null }; + var id = Str(h, "id"); + if (source is null || id is null) + return null; + var location = Prop(h, "location"); + var loc = location is { } l ? new SearchLocation(Str(l, "kind"), Str(l, "teamId"), Str(l, "channelId"), Str(l, "chatId"), Str(l, "path")) : new SearchLocation(null, null, null, null, null); + return new RawSearchHit(source, id, Str(h, "driveId"), Str(h, "title"), Str(h, "snippet"), Person(Prop(h, "from")), Str(h, "date"), location is { } l2 ? Str(l2, "label") : null, loc, Str(h, "webUrl"), Long(h, "size"), Str(h, "mimeType"), Bool(h, "isFolder") == true ? true : null, term, rank); + } + + /// The distinct, trimmed search texts: query first, then terms, at most . + public static List SearchTerms(string? query, string[]? terms) + { + var result = new List(); + foreach (var raw in new[] { query }.Concat(terms ?? [])) + { + var text = raw?.Trim(); + if (string.IsNullOrEmpty(text)) + continue; + if (text.Length > 200 || text.Any(char.IsControl)) + throw new McpException("Every search text must be 1 to 200 characters without control characters."); + if (!result.Contains(text, StringComparer.OrdinalIgnoreCase)) + result.Add(text); + } + if (result.Count == 0) + throw new McpException("Give query, terms or both."); + if (result.Count > MaxSearchTerms) + throw new McpException($"At most {MaxSearchTerms} search texts (query plus terms) per call. Keep the most distinctive synonyms."); + return result; + } + + private static HashSet ParseSearchSources(string[]? sources) + { + if (sources is not { Length: > 0 }) + return [.. SearchSourceNames]; + var result = new HashSet(); + foreach (var raw in sources) + { + var name = raw?.Trim().ToLowerInvariant() switch + { + "file" or "files" or "drive" => "files", + "channel" or "channels" or "posts" or "channel posts" => "channels", + "mail" or "email" or "emails" => "mail", + "chat" or "chats" => "chats", + "page" or "pages" or "onenote" => "pages", + _ => throw new McpException($"Unknown source '{raw}'. Use files, channels, mail, chats or pages."), + }; + result.Add(name); + } + return result; + } + + /// + /// Merges per-term results: one entry per item (source + drive + id) with the terms that found it, filtered to the date range, ordered by + /// a title match first, then the number of terms matched, then the best rank the source gave it, then the newest date. + /// + public static List MergeSearchHits(IEnumerable raw, IReadOnlyList terms, DateOnly? from, DateOnly? to) + { + var merged = new Dictionary(); + foreach (var hit in raw) + { + var key = $"{hit.Source}|{hit.DriveId}|{hit.Id}"; + if (!merged.TryGetValue(key, out var entry)) + merged[key] = entry = new MergedSearchHit(hit); + if (!entry.MatchedTerms.Contains(hit.Term)) + entry.MatchedTerms.Add(hit.Term); + entry.BestRank = Math.Min(entry.BestRank, hit.Rank); + if (string.IsNullOrEmpty(entry.Hit.Snippet) && !string.IsNullOrEmpty(hit.Snippet)) + entry.Hit = entry.Hit with { Snippet = hit.Snippet }; + } + var folded = terms.ToDictionary(t => t, Fold); + foreach (var entry in merged.Values) + { + var title = Fold(entry.Hit.Title ?? string.Empty); + entry.TitleMatch = entry.MatchedTerms.Any(t => folded.TryGetValue(t, out var f) && f.Length > 0 && title.Contains(f, StringComparison.Ordinal)); + entry.Date = DateTimeOffset.TryParse(entry.Hit.Date, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var d) ? d : null; + } + return merged.Values + .Where(e => e.Date is not { } d || ((from is null || DateOnly.FromDateTime(d.UtcDateTime) >= from) && (to is null || DateOnly.FromDateTime(d.UtcDateTime) <= to))) + .OrderByDescending(e => e.TitleMatch) + .ThenByDescending(e => e.MatchedTerms.Count) + .ThenBy(e => e.BestRank) + .ThenByDescending(e => e.Date ?? DateTimeOffset.MinValue) + .ToList(); + } + + private static string Fold(string text) + { + var decomposed = text.Normalize(NormalizationForm.FormD); + var sb = new StringBuilder(decomposed.Length); + foreach (var c in decomposed) + if (CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark) + sb.Append(char.ToLowerInvariant(c)); + return sb.ToString().Trim(); + } + + private static object SearchHitOutput(MergedSearchHit e) + { + var h = e.Hit; + var loc = h.Location; + return new + { + source = h.Source, + itemId = h.Source == "files" ? h.Id : null, + driveId = h.DriveId, + messageId = h.Source is "mail" or "chats" or "channels" ? h.Id : null, + pageId = h.Source == "pages" ? h.Id : null, + readWith = h.Source switch { "files" => h.IsFolder == true ? "list_folder" : "read_file", "mail" => "read_mail", "channels" => "list_channel_messages", "chats" => "read_chat_messages", _ => "read_page" }, + title = h.Title is { } title ? Cap(title, 200) : null, + snippet = h.Snippet is { } snippet ? Cap(snippet, 300) : null, + author = h.Author, + modified = h.Date, + location = new { label = h.Label, kind = loc.Kind, teamId = loc.TeamId, channelId = loc.ChannelId, chatId = loc.ChatId, path = loc.Path }, + webUrl = h.WebUrl, + size = h.Size, + mimeType = h.MimeType, + isFolder = h.IsFolder, + matchedTerms = e.MatchedTerms.ToList(), + }; + } + + private sealed class SearchRun + { + public ConcurrentBag Hits { get; } = []; + public ConcurrentBag Errors { get; } = []; + public volatile bool MoreAtBackend; + public int Succeeded; + } + } + + public sealed record SearchLocation(string? Kind, string? TeamId, string? ChannelId, string? ChatId, string? Path); + + /// One hit as a source returned it for one search text. is its position in that source's own list (0 is best). + public sealed record RawSearchHit(string Source, string Id, string? DriveId, string? Title, string? Snippet, string? Author, string? Date, string? Label, SearchLocation Location, string? WebUrl, long? Size, string? MimeType, bool? IsFolder, string Term, int Rank); + + public sealed class MergedSearchHit(RawSearchHit hit) + { + public RawSearchHit Hit { get; set; } = hit; + public List MatchedTerms { get; } = []; + public int BestRank { get; set; } = int.MaxValue; + public bool TitleMatch { get; set; } + public DateTimeOffset? Date { get; set; } + } +} diff --git a/src/Schuly.API/Mcp/Tools/MicrosoftTools.cs b/src/Schuly.API/Mcp/Tools/MicrosoftTools.cs index 605f2726..76ffde0f 100644 --- a/src/Schuly.API/Mcp/Tools/MicrosoftTools.cs +++ b/src/Schuly.API/Mcp/Tools/MicrosoftTools.cs @@ -307,8 +307,7 @@ public async Task ReadPage([Description("pageId from list_pages or searc public async Task SearchPages([Description("Text to find in page titles, 1-100 characters.")] string query, [Description("Only search this notebook (notebookId from list_notebooks, exactly as returned).")] string? notebookId = null, Guid? accountId = null, [Description("Hits to return, 1-50.")] int limit = 25, CancellationToken ct = default) { var account = await AccountAsync(accountId, ct); - var url = $"{Base}/accounts/{account}/onenote/search?q={Uri.EscapeDataString(query)}&top={Math.Clamp(limit, 1, 50)}" + (string.IsNullOrWhiteSpace(notebookId) ? "" : $"¬ebook={Uri.EscapeDataString(notebookId)}"); - var data = await ReadAsync(url, ct) ?? default; + var data = await SearchPagesRawAsync(account, query, notebookId, limit, ct) ?? default; var hits = Items(Prop(data, "items")).Select(h => new { pageId = Str(h, "id"), @@ -365,6 +364,9 @@ public async Task ListCalendarEvents([Description("Start, ISO 8601 date return Serialize(new { from = start, to = end, events }); } + private Task SearchPagesRawAsync(Guid account, string query, string? notebookId, int limit, CancellationToken ct) => + ReadAsync($"{Base}/accounts/{account}/onenote/search?q={Uri.EscapeDataString(query)}&top={Math.Clamp(limit, 1, 50)}" + (string.IsNullOrWhiteSpace(notebookId) ? "" : $"¬ebook={Uri.EscapeDataString(notebookId)}"), ct); + private static object MailSummary(JsonElement m) => new { id = Str(m, "id"), diff --git a/src/Schuly.Tests/Mcp/McpFileToolsTests.cs b/src/Schuly.Tests/Mcp/McpFileToolsTests.cs index c2a396ea..2184e95a 100644 --- a/src/Schuly.Tests/Mcp/McpFileToolsTests.cs +++ b/src/Schuly.Tests/Mcp/McpFileToolsTests.cs @@ -23,7 +23,7 @@ namespace Schuly.Tests.Mcp { /// Teams / SharePoint file tools, unified search, file text and channel messages against fake plugin endpoints that record what they receive. [NotInParallel] - public class McpFileToolsTests + public partial class McpFileToolsTests { private static readonly Guid AccountId = Guid.Parse("11111111-2222-3333-4444-555555555555"); @@ -37,7 +37,9 @@ private static McpPermissionSet Only(params (McpArea Area, McpAccessLevel Level) return set; } - private static async Task<(McpHttpHost Host, McpClient Client, ConcurrentQueue Seen)> StartAsync(string name, McpPermissionSet permissions, Func answer) + private static Task<(McpHttpHost Host, McpClient Client, ConcurrentQueue Seen)> StartAsync(string name, McpPermissionSet permissions, Func answer) => StartAsync(name, permissions, seen => answer(seen.Method, seen.Path)); + + private static async Task<(McpHttpHost Host, McpClient Client, ConcurrentQueue Seen)> StartAsync(string name, McpPermissionSet permissions, Func answer) { var seen = new ConcurrentQueue(); var host = await McpHttpHost.StartAsync( @@ -50,8 +52,9 @@ private static McpPermissionSet Only(params (McpArea Area, McpAccessLevel Level) app => app.MapMethods("/api/plugins/microsoft/accounts/{accountId}/{**rest}", ["GET", "POST"], async (HttpContext context, string rest) => { var body = context.Request.ContentLength > 0 ? await new StreamReader(context.Request.Body).ReadToEndAsync() : string.Empty; - seen.Enqueue(new Seen(context.Request.Method, rest, context.Request.QueryString.Value ?? string.Empty, body)); - var json = answer(context.Request.Method, rest); + var request = new Seen(context.Request.Method, rest, context.Request.QueryString.Value ?? string.Empty, body); + seen.Enqueue(request); + var json = answer(request); return json is null ? Results.NotFound() : Results.Content(json, "application/json"); })); var userId = host.Scoped(sp => NewUser(sp.GetRequiredService()).User.Id); @@ -130,7 +133,15 @@ public async Task Unified_search_only_asks_for_mail_and_chats_when_the_connectio await Assert.That(hit.GetProperty("itemId").GetString()).IsEqualTo("item-1"); await Assert.That(hit.GetProperty("driveId").GetString()).IsEqualTo("b!drive"); await Assert.That(hit.GetProperty("location").GetProperty("label").GetString()).IsEqualTo("Team BM23d > General"); - await Assert.That(result.GetProperty("nextOffset").GetInt32()).IsEqualTo(25); + await Assert.That(hit.GetProperty("source").GetString()).IsEqualTo("files"); + await Assert.That(hit.GetProperty("readWith").GetString()).IsEqualTo("read_file"); + await Assert.That(hit.GetProperty("matchedTerms").EnumerateArray().Select(t => t.GetString()!)).IsEquivalentTo(new[] { "Masterplan" }); + await Assert.That(result.GetProperty("partial").GetBoolean()).IsTrue(); + await Assert.That(result.GetProperty("searched").EnumerateArray().Select(t => t.GetString() ?? "")).IsEquivalentTo(cases[i].Types); + var skipped = result.GetProperty("skipped").EnumerateArray().Select(t => t.GetProperty("source").GetString() + ":" + t.GetProperty("reason").GetString()).ToList(); + await Assert.That(skipped.Contains("mail:needs Mail read")).IsEqualTo(!cases[i].Types.Contains("mail")); + await Assert.That(skipped.Contains("chats:needs Teams read")).IsEqualTo(!cases[i].Types.Contains("chats")); + await Assert.That(skipped.Contains("pages:needs OneNote read")).IsTrue(); } } diff --git a/src/Schuly.Tests/Mcp/McpSearchTests.cs b/src/Schuly.Tests/Mcp/McpSearchTests.cs new file mode 100644 index 00000000..dda21f8e --- /dev/null +++ b/src/Schuly.Tests/Mcp/McpSearchTests.cs @@ -0,0 +1,185 @@ +using System.Text.Json; +using System.Web; +using TUnit.Assertions.Enums; +using Schuly.API.Helply; +using Schuly.Domain.Mcp; +using static Schuly.Tests.Mcp.McpTestSupport; + +namespace Schuly.Tests.Mcp +{ + /// The upgraded cross-source search: terms, date range, sources, OneNote titles, merging, ranking and partial results. + public partial class McpFileToolsTests + { + private static string Hits(params string[] hits) => "{\"hits\":[" + string.Join(",", hits) + "]}"; + + private static string FileHit(string id, string title, string date) + { + var when = date.Length == 0 ? "null" : "\"" + date + "\""; + return "{\"type\":\"file\",\"id\":\"" + id + "\",\"driveId\":\"b!d\",\"title\":\"" + title + "\",\"snippet\":\"s-" + id + "\",\"date\":" + when + ",\"from\":{\"displayName\":\"Anna\",\"email\":\"a@x\"},\"location\":{\"kind\":\"team\",\"teamId\":\"t1\",\"label\":\"Team BM23d > General\"},\"webUrl\":\"https://x/" + id + "\"}"; + } + + private static string MailHit(string id, string title, string date) => + "{\"type\":\"mail\",\"id\":\"" + id + "\",\"title\":\"" + title + "\",\"snippet\":\"m\",\"date\":\"" + date + "\",\"from\":{\"displayName\":\"Bob\",\"email\":\"b@x\"},\"location\":{\"kind\":\"mailbox\",\"label\":\"Mailbox\"}}"; + + private static string QueryOf(Seen seen) => seen.Method == "POST" ? JsonDocument.Parse(seen.Body).RootElement.GetProperty("q").GetString()! : HttpUtility.ParseQueryString(seen.Query)["q"]!; + + private static List Ids(JsonElement result) => result.GetProperty("hits").EnumerateArray().Select(h => (h.TryGetProperty("itemId", out var i) ? i : h.TryGetProperty("messageId", out var m) ? m : h.GetProperty("pageId")).GetString()!).ToList(); + + [Test] + public async Task Search_fans_out_over_all_terms_in_one_call_and_merges_duplicates_by_item() + { + var (host, client, seen) = await StartAsync(nameof(Search_fans_out_over_all_terms_in_one_call_and_merges_duplicates_by_item), Only((McpArea.Drive, McpAccessLevel.Read), (McpArea.Mail, McpAccessLevel.Read)), request => QueryOf(request) switch + { + "Pruefung" => Hits(FileHit("A", "Pruefung Plan.xlsx", "2026-10-01T08:00:00Z"), FileHit("B", "Notizen.docx", "2026-09-01T08:00:00Z")), + "Test" => Hits(FileHit("B", "Notizen.docx", "2026-09-01T08:00:00Z"), MailHit("C", "Test 3 Termin", "2026-10-20T08:00:00Z")), + "Lernkontrolle" => Hits(FileHit("B", "Notizen.docx", "2026-09-01T08:00:00Z")), + _ => Hits(), + }); + await using var _h = host; + await using var _c = client; + + var result = await CallAsync(client, "search", new() { ["query"] = "Pruefung", ["terms"] = new[] { "Test", "test", "Lernkontrolle", " " } }); + + var requests = seen.ToList(); + await Assert.That(requests.Count).IsEqualTo(3); + await Assert.That(requests.Select(QueryOf)).IsEquivalentTo(new[] { "Pruefung", "Test", "Lernkontrolle" }); + await Assert.That(result.GetProperty("terms").EnumerateArray().Select(t => t.GetString()!)).IsEquivalentTo(new[] { "Pruefung", "Test", "Lernkontrolle" }); + + // Title matches come first (A, then C by its rank); B was found by all three terms but has no title match, so it follows. It appears once. + var hits = result.GetProperty("hits").EnumerateArray().ToList(); + await Assert.That(hits.Count).IsEqualTo(3); + await Assert.That(hits[0].GetProperty("itemId").GetString()).IsEqualTo("A"); + await Assert.That(hits[1].GetProperty("messageId").GetString()).IsEqualTo("C"); + await Assert.That(hits[1].GetProperty("source").GetString()).IsEqualTo("mail"); + await Assert.That(hits[1].GetProperty("readWith").GetString()).IsEqualTo("read_mail"); + await Assert.That(hits[2].GetProperty("itemId").GetString()).IsEqualTo("B"); + await Assert.That(hits[2].GetProperty("matchedTerms").EnumerateArray().Select(t => t.GetString()!)).IsEquivalentTo(new[] { "Pruefung", "Test", "Lernkontrolle" }); + await Assert.That(hits[0].GetProperty("author").GetString()).IsEqualTo("Anna "); + await Assert.That(hits[0].GetProperty("modified").GetString()).IsEqualTo("2026-10-01T08:00:00Z"); + await Assert.That(hits[0].GetProperty("location").GetProperty("label").GetString()).IsEqualTo("Team BM23d > General"); + await Assert.That(result.GetProperty("total").GetInt32()).IsEqualTo(3); + } + + [Test] + public async Task Search_ranks_more_matched_terms_before_recency_and_pages_the_merged_list() + { + var (host, client, _) = await StartAsync(nameof(Search_ranks_more_matched_terms_before_recency_and_pages_the_merged_list), Only((McpArea.Drive, McpAccessLevel.Read)), request => QueryOf(request) switch + { + "alpha" => Hits(FileHit("OLD", "x1", "2025-01-01T00:00:00Z"), FileHit("BOTH", "x2", "2025-02-01T00:00:00Z")), + "beta" => Hits(FileHit("NEW", "x3", "2026-05-01T00:00:00Z"), FileHit("BOTH", "x2", "2025-02-01T00:00:00Z")), + _ => Hits(), + }); + await using var _h = host; + await using var _c = client; + + // BOTH matched two terms. OLD and NEW tie on terms and rank, so the newer one comes first. + var all = await CallAsync(client, "search", new() { ["terms"] = new[] { "alpha", "beta" } }); + await Assert.That(Ids(all)).IsEquivalentTo(new[] { "BOTH", "NEW", "OLD" }, CollectionOrdering.Matching); + + var second = await CallAsync(client, "search", new() { ["terms"] = new[] { "alpha", "beta" }, ["limit"] = 1, ["offset"] = 1 }); + await Assert.That(Ids(second)).IsEquivalentTo(new[] { "NEW" }); + await Assert.That(second.GetProperty("nextOffset").GetInt32()).IsEqualTo(2); + await Assert.That(second.GetProperty("total").GetInt32()).IsEqualTo(3); + } + + [Test] + public async Task Search_filters_by_the_modified_date_range_and_keeps_undated_hits() + { + var (host, client, _) = await StartAsync(nameof(Search_filters_by_the_modified_date_range_and_keeps_undated_hits), Only((McpArea.Drive, McpAccessLevel.Read)), _ => Hits(FileHit("OLD", "a", "2026-08-31T23:00:00Z"), FileHit("IN1", "b", "2026-09-01T00:00:00Z"), FileHit("IN2", "c", "2026-10-31T22:00:00Z"), FileHit("LATE", "d", "2026-11-01T01:00:00Z"), FileHit("NODATE", "e", ""))); + await using var _h = host; + await using var _c = client; + + var result = await CallAsync(client, "search", new() { ["query"] = "plan", ["from"] = "2026-09-01", ["to"] = "2026-10-31" }); + await Assert.That(Ids(result)).IsEquivalentTo(new[] { "IN1", "IN2", "NODATE" }); + + var bad = await client.CallToolAsync("search", new Dictionary { ["query"] = "plan", ["from"] = "2026-10-31", ["to"] = "2026-09-01" }); + await Assert.That(bad.IsError ?? false).IsTrue(); + } + + [Test] + public async Task Search_adds_OneNote_page_titles_only_with_OneNote_read_and_honours_the_sources_filter() + { + const string pages = """{"items":[{"id":"P1","title":"Pruefungstermine","notebook":{"displayName":"BM23d"},"section":{"displayName":"Deutsch"},"lastModifiedDateTime":"2026-10-02T08:00:00Z","webUrl":"https://x/p1"}],"partial":false}"""; + var (host, client, seen) = await StartAsync(nameof(Search_adds_OneNote_page_titles_only_with_OneNote_read_and_honours_the_sources_filter), Only((McpArea.Drive, McpAccessLevel.Read), (McpArea.OneNote, McpAccessLevel.Read)), request => request.Path == "onenote/search" ? pages : Hits(FileHit("F1", "Plan", "2026-10-01T00:00:00Z"))); + await using var _h = host; + await using var _c = client; + + var both = await CallAsync(client, "search", new() { ["query"] = "Pruefung" }); + await Assert.That(both.GetProperty("searched").EnumerateArray().Select(t => t.GetString() ?? "")).IsEquivalentTo(new[] { "files", "pages" }); + var page = both.GetProperty("hits").EnumerateArray().Single(h => h.GetProperty("source").GetString() == "pages"); + await Assert.That(page.GetProperty("pageId").GetString()).IsEqualTo("P1"); + await Assert.That(page.GetProperty("readWith").GetString()).IsEqualTo("read_page"); + await Assert.That(page.GetProperty("location").GetProperty("label").GetString()).IsEqualTo("BM23d > Deutsch"); + await Assert.That(page.GetProperty("title").GetString()).IsEqualTo("Pruefungstermine"); + + seen.Clear(); + var onlyPages = await CallAsync(client, "search", new() { ["query"] = "Pruefung", ["sources"] = new[] { "OneNote" } }); + await Assert.That(seen.Select(s => s.Path)).IsEquivalentTo(new[] { "onenote/search" }); + await Assert.That(onlyPages.GetProperty("searched").EnumerateArray().Select(t => t.GetString() ?? "")).IsEquivalentTo(new[] { "pages" }); + await Assert.That(onlyPages.GetProperty("skipped").EnumerateArray().All(s => s.GetProperty("reason").GetString() == "not requested")).IsTrue(); + + seen.Clear(); + var scoped = await CallAsync(client, "search", new() { ["query"] = "Pruefung", ["teamId"] = "t1" }); + await Assert.That(seen.Select(s => s.Path)).IsEquivalentTo(new[] { "search" }); + await Assert.That(scoped.GetProperty("skipped").EnumerateArray().Any(s => s.GetProperty("source").GetString() == "pages")).IsTrue(); + } + + [Test] + public async Task Search_without_OneNote_read_never_calls_the_OneNote_endpoint_and_a_requested_source_that_is_not_allowed_is_an_error() + { + var (host, client, seen) = await StartAsync(nameof(Search_without_OneNote_read_never_calls_the_OneNote_endpoint_and_a_requested_source_that_is_not_allowed_is_an_error), Only((McpArea.Drive, McpAccessLevel.Read)), _ => Hits()); + await using var _h = host; + await using var _c = client; + + await CallAsync(client, "search", new() { ["query"] = "x" }); + await Assert.That(seen.Any(s => s.Path.StartsWith("onenote"))).IsFalse(); + + var denied = await client.CallToolAsync("search", new Dictionary { ["query"] = "x", ["sources"] = new[] { "mail", "pages" } }); + await Assert.That(denied.IsError ?? false).IsTrue(); + await Assert.That(seen.Count).IsEqualTo(1); + } + + [Test] + public async Task Search_reports_a_failing_source_as_partial_and_still_returns_the_others() + { + var (host, client, _) = await StartAsync(nameof(Search_reports_a_failing_source_as_partial_and_still_returns_the_others), Only((McpArea.Drive, McpAccessLevel.Read), (McpArea.OneNote, McpAccessLevel.Read)), request => request.Path == "onenote/search" ? null : Hits(FileHit("F1", "Plan", "2026-10-01T00:00:00Z"))); + await using var _h = host; + await using var _c = client; + + var result = await CallAsync(client, "search", new() { ["query"] = "Plan" }); + await Assert.That(Ids(result)).IsEquivalentTo(new[] { "F1" }); + await Assert.That(result.GetProperty("partial").GetBoolean()).IsTrue(); + await Assert.That(result.GetProperty("errors").EnumerateArray().Any(e => e.GetString()!.StartsWith("pages:"))).IsTrue(); + + var (host2, client2, _) = await StartAsync(nameof(Search_reports_a_failing_source_as_partial_and_still_returns_the_others) + "B", Only((McpArea.Drive, McpAccessLevel.Read)), _ => null); + await using var _h2 = host2; + await using var _c2 = client2; + var failed = await client2.CallToolAsync("search", new Dictionary { ["query"] = "Plan" }); + await Assert.That(failed.IsError ?? false).IsTrue(); + } + + [Test] + public async Task Search_needs_a_text_and_caps_the_number_of_terms() + { + var (host, client, seen) = await StartAsync(nameof(Search_needs_a_text_and_caps_the_number_of_terms), Only((McpArea.Drive, McpAccessLevel.Read)), _ => Hits()); + await using var _h = host; + await using var _c = client; + + var none = await client.CallToolAsync("search", new Dictionary { ["terms"] = new[] { " " } }); + await Assert.That(none.IsError ?? false).IsTrue(); + var many = await client.CallToolAsync("search", new Dictionary { ["query"] = "a", ["terms"] = Enumerable.Range(0, 8).Select(i => "t" + i).ToArray() }); + await Assert.That(many.IsError ?? false).IsTrue(); + await Assert.That(seen.IsEmpty).IsTrue(); + } + + [Test] + public async Task The_prompt_teaches_one_search_call_with_terms_and_a_date_range() + { + var prompt = HelplyPrompt.System(DateTimeOffset.UtcNow); + + await Assert.That(prompt).Contains("as terms"); + await Assert.That(prompt).Contains("from/to date range"); + await Assert.That(prompt).Contains("fall back to search_files"); + } + } +}